src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 494                                       u32 thread_index)
 495 {
 496   nat_ed_ses_key_t key;
 497   clib_bihash_kv_16_8_t kv, value;
 498   udp_header_t *udp;
 499   snat_session_t *s = 0;
 500   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 501
 502   if (!sm->forwarding_enabled)
 503     return 0;
 504
 505   if (ip->protocol == IP_PROTOCOL_ICMP)
 506     {
 507       if (icmp_get_ed_key (ip, &key))
 508         return 0;
 509     }
 510   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 511     {
 512       udp = ip4_next_header(ip);
 513       key.l_addr = ip->src_address;
 514       key.r_addr = ip->dst_address;
 515       key.proto = ip->protocol;
 516       key.r_port = udp->dst_port;
 517       key.l_port = udp->src_port;
 518     }
 519   else
 520     {
 521       key.l_addr = ip->src_address;
 522       key.r_addr = ip->dst_address;
 523       key.proto = ip->protocol;
 524       key.l_port = key.r_port = 0;
 525     }
 526   key.fib_index = 0;
 527   kv.key[0] = key.as_u64[0];
 528   kv.key[1] = key.as_u64[1];
 529
 530   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 531     {
 532       s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value.value);
 533       if (is_fwd_bypass_session (s))
 534         {
 535           /* Per-user LRU list maintenance */
 536           clib_dlist_remove (tsm->list_pool, s->per_user_index);
 537           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 538                               s->per_user_index);
 539           return 1;
 540         }
 541       else
 542         return 0;
 543     }
 544
 545   return 0;
 546 }
 547
 548 /**
 549  * Get address and port values to be used for ICMP packet translation
 550  * and create session if needed
 551  *
 552  * @param[in,out] sm             NAT main
 553  * @param[in,out] node           NAT node runtime
 554  * @param[in] thread_index       thread index
 555  * @param[in,out] b0             buffer containing packet to be translated
 556  * @param[out] p_proto           protocol used for matching
 557  * @param[out] p_value           address and port after NAT translation
 558  * @param[out] p_dont_translate  if packet should not be translated
 559  * @param d                      optional parameter
 560  * @param e                      optional parameter
 561  */
 562 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 563                            u32 thread_index, vlib_buffer_t *b0,
 564                            ip4_header_t *ip0, u8 *p_proto,
 565                            snat_session_key_t *p_value,
 566                            u8 *p_dont_translate, void *d, void *e)
 567 {
 568   icmp46_header_t *icmp0;
 569   u32 sw_if_index0;
 570   u32 rx_fib_index0;
 571   snat_session_key_t key0;
 572   snat_session_t *s0 = 0;
 573   u8 dont_translate = 0;
 574   clib_bihash_kv_8_8_t kv0, value0;
 575   u32 next0 = ~0;
 576   int err;
 577
 578   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 579   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 580   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 581
 582   err = icmp_get_key (ip0, &key0);
 583   if (err != -1)
 584     {
 585       b0->error = node->errors[err];
 586       next0 = SNAT_IN2OUT_NEXT_DROP;
 587       goto out;
 588     }
 589   key0.fib_index = rx_fib_index0;
 590
 591   kv0.key = key0.as_u64;
 592
 593   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 594                               &value0))
 595     {
 596       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 597         {
 598           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 599               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 600             {
 601               dont_translate = 1;
 602               goto out;
 603             }
 604         }
 605       else
 606         {
 607           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 608               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 609             {
 610               dont_translate = 1;
 611               goto out;
 612             }
 613         }
 614
 615       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 616         {
 617           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 618           next0 = SNAT_IN2OUT_NEXT_DROP;
 619           goto out;
 620         }
 621
 622       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 623                          &s0, node, next0, thread_index);
 624
 625       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 626         goto out;
 627     }
 628   else
 629     {
 630       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 631                         icmp0->type != ICMP4_echo_reply &&
 632                         !icmp_is_error_message (icmp0)))
 633         {
 634           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 635           next0 = SNAT_IN2OUT_NEXT_DROP;
 636           goto out;
 637         }
 638
 639       if (PREDICT_FALSE (value0.value == ~0ULL))
 640         {
 641           nat_ed_ses_key_t key;
 642           clib_bihash_kv_16_8_t s_kv, s_value;
 643
 644           key.as_u64[0] = 0;
 645           key.as_u64[1] = 0;
 646           if (icmp_get_ed_key (ip0, &key))
 647             {
 648               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 649               next0 = SNAT_IN2OUT_NEXT_DROP;
 650               goto out;
 651             }
 652           key.fib_index = rx_fib_index0;
 653           s_kv.key[0] = key.as_u64[0];
 654           s_kv.key[1] = key.as_u64[1];
 655           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 656             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 657                                     s_value.value);
 658           else
 659            {
 660               next0 = SNAT_IN2OUT_NEXT_DROP;
 661               goto out;
 662            }
 663         }
 664       else
 665         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 666                                 value0.value);
 667     }
 668
 669 out:
 670   *p_proto = key0.protocol;
 671   if (s0)
 672     *p_value = s0->out2in;
 673   *p_dont_translate = dont_translate;
 674   if (d)
 675     *(snat_session_t**)d = s0;
 676   return next0;
 677 }
 678
 679 /**
 680  * Get address and port values to be used for ICMP packet translation
 681  *
 682  * @param[in] sm                 NAT main
 683  * @param[in,out] node           NAT node runtime
 684  * @param[in] thread_index       thread index
 685  * @param[in,out] b0             buffer containing packet to be translated
 686  * @param[out] p_proto           protocol used for matching
 687  * @param[out] p_value           address and port after NAT translation
 688  * @param[out] p_dont_translate  if packet should not be translated
 689  * @param d                      optional parameter
 690  * @param e                      optional parameter
 691  */
 692 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 693                            u32 thread_index, vlib_buffer_t *b0,
 694                            ip4_header_t *ip0, u8 *p_proto,
 695                            snat_session_key_t *p_value,
 696                            u8 *p_dont_translate, void *d, void *e)
 697 {
 698   icmp46_header_t *icmp0;
 699   u32 sw_if_index0;
 700   u32 rx_fib_index0;
 701   snat_session_key_t key0;
 702   snat_session_key_t sm0;
 703   u8 dont_translate = 0;
 704   u8 is_addr_only;
 705   u32 next0 = ~0;
 706   int err;
 707
 708   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 709   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 710   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 711
 712   err = icmp_get_key (ip0, &key0);
 713   if (err != -1)
 714     {
 715       b0->error = node->errors[err];
 716       next0 = SNAT_IN2OUT_NEXT_DROP;
 717       goto out2;
 718     }
 719   key0.fib_index = rx_fib_index0;
 720
 721   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 722     {
 723       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 724           IP_PROTOCOL_ICMP, rx_fib_index0)))
 725         {
 726           dont_translate = 1;
 727           goto out;
 728         }
 729
 730       if (icmp_is_error_message (icmp0))
 731         {
 732           next0 = SNAT_IN2OUT_NEXT_DROP;
 733           goto out;
 734         }
 735
 736       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 737       next0 = SNAT_IN2OUT_NEXT_DROP;
 738       goto out;
 739     }
 740
 741   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 742                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 743                     !icmp_is_error_message (icmp0)))
 744     {
 745       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 746       next0 = SNAT_IN2OUT_NEXT_DROP;
 747       goto out;
 748     }
 749
 750 out:
 751   *p_value = sm0;
 752 out2:
 753   *p_proto = key0.protocol;
 754   *p_dont_translate = dont_translate;
 755   return next0;
 756 }
 757
 758 static inline u32 icmp_in2out (snat_main_t *sm,
 759                                vlib_buffer_t * b0,
 760                                ip4_header_t * ip0,
 761                                icmp46_header_t * icmp0,
 762                                u32 sw_if_index0,
 763                                u32 rx_fib_index0,
 764                                vlib_node_runtime_t * node,
 765                                u32 next0,
 766                                u32 thread_index,
 767                                void *d,
 768                                void *e)
 769 {
 770   snat_session_key_t sm0;
 771   u8 protocol;
 772   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 773   ip4_header_t *inner_ip0;
 774   void *l4_header = 0;
 775   icmp46_header_t *inner_icmp0;
 776   u8 dont_translate;
 777   u32 new_addr0, old_addr0;
 778   u16 old_id0, new_id0;
 779   ip_csum_t sum0;
 780   u16 checksum0;
 781   u32 next0_tmp;
 782
 783   echo0 = (icmp_echo_header_t *)(icmp0+1);
 784
 785   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 786                                        &protocol, &sm0, &dont_translate, d, e);
 787   if (next0_tmp != ~0)
 788     next0 = next0_tmp;
 789   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 790     goto out;
 791
 792   sum0 = ip_incremental_checksum (0, icmp0,
 793                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 794   checksum0 = ~ip_csum_fold (sum0);
 795   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 796     {
 797       next0 = SNAT_IN2OUT_NEXT_DROP;
 798       goto out;
 799     }
 800
 801   old_addr0 = ip0->src_address.as_u32;
 802   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 803   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 804     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 805
 806   sum0 = ip0->checksum;
 807   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 808                          src_address /* changed member */);
 809   ip0->checksum = ip_csum_fold (sum0);
 810
 811   if (icmp0->checksum == 0)
 812     icmp0->checksum = 0xffff;
 813
 814   if (!icmp_is_error_message (icmp0))
 815     {
 816       new_id0 = sm0.port;
 817       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 818         {
 819           old_id0 = echo0->identifier;
 820           new_id0 = sm0.port;
 821           echo0->identifier = new_id0;
 822
 823           sum0 = icmp0->checksum;
 824           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 825                                  identifier);
 826           icmp0->checksum = ip_csum_fold (sum0);
 827         }
 828     }
 829   else
 830     {
 831       inner_ip0 = (ip4_header_t *)(echo0+1);
 832       l4_header = ip4_next_header (inner_ip0);
 833
 834       if (!ip4_header_checksum_is_valid (inner_ip0))
 835         {
 836           next0 = SNAT_IN2OUT_NEXT_DROP;
 837           goto out;
 838         }
 839
 840       old_addr0 = inner_ip0->dst_address.as_u32;
 841       inner_ip0->dst_address = sm0.addr;
 842       new_addr0 = inner_ip0->dst_address.as_u32;
 843
 844       sum0 = icmp0->checksum;
 845       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 846                              dst_address /* changed member */);
 847       icmp0->checksum = ip_csum_fold (sum0);
 848
 849       switch (protocol)
 850         {
 851           case SNAT_PROTOCOL_ICMP:
 852             inner_icmp0 = (icmp46_header_t*)l4_header;
 853             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 854
 855             old_id0 = inner_echo0->identifier;
 856             new_id0 = sm0.port;
 857             inner_echo0->identifier = new_id0;
 858
 859             sum0 = icmp0->checksum;
 860             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 861                                    identifier);
 862             icmp0->checksum = ip_csum_fold (sum0);
 863             break;
 864           case SNAT_PROTOCOL_UDP:
 865           case SNAT_PROTOCOL_TCP:
 866             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 867             new_id0 = sm0.port;
 868             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 869
 870             sum0 = icmp0->checksum;
 871             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 872                                    dst_port);
 873             icmp0->checksum = ip_csum_fold (sum0);
 874             break;
 875           default:
 876             ASSERT(0);
 877         }
 878     }
 879
 880 out:
 881   return next0;
 882 }
 883
 884 /**
 885  * @brief Hairpinning
 886  *
 887  * Hairpinning allows two endpoints on the internal side of the NAT to
 888  * communicate even if they only use each other's external IP addresses
 889  * and ports.
 890  *
 891  * @param sm     NAT main.
 892  * @param b0     Vlib buffer.
 893  * @param ip0    IP header.
 894  * @param udp0   UDP header.
 895  * @param tcp0   TCP header.
 896  * @param proto0 NAT protocol.
 897  */
 898 static inline int
 899 snat_hairpinning (snat_main_t *sm,
 900                   vlib_buffer_t * b0,
 901                   ip4_header_t * ip0,
 902                   udp_header_t * udp0,
 903                   tcp_header_t * tcp0,
 904                   u32 proto0)
 905 {
 906   snat_session_key_t key0, sm0;
 907   snat_session_t * s0;
 908   clib_bihash_kv_8_8_t kv0, value0;
 909   ip_csum_t sum0;
 910   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 911   u16 new_dst_port0, old_dst_port0;
 912
 913   key0.addr = ip0->dst_address;
 914   key0.port = udp0->dst_port;
 915   key0.protocol = proto0;
 916   key0.fib_index = sm->outside_fib_index;
 917   kv0.key = key0.as_u64;
 918
 919   /* Check if destination is static mappings */
 920   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 921     {
 922       new_dst_addr0 = sm0.addr.as_u32;
 923       new_dst_port0 = sm0.port;
 924       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 925     }
 926   /* or active session */
 927   else
 928     {
 929       if (sm->num_workers > 1)
 930         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 931       else
 932         ti = sm->num_workers;
 933
 934       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 935         {
 936           si = value0.value;
 937
 938           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 939           new_dst_addr0 = s0->in2out.addr.as_u32;
 940           new_dst_port0 = s0->in2out.port;
 941           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 942         }
 943     }
 944
 945   /* Destination is behind the same NAT, use internal address and port */
 946   if (new_dst_addr0)
 947     {
 948       old_dst_addr0 = ip0->dst_address.as_u32;
 949       ip0->dst_address.as_u32 = new_dst_addr0;
 950       sum0 = ip0->checksum;
 951       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 952                              ip4_header_t, dst_address);
 953       ip0->checksum = ip_csum_fold (sum0);
 954
 955       old_dst_port0 = tcp0->dst;
 956       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 957         {
 958           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 959             {
 960               tcp0->dst = new_dst_port0;
 961               sum0 = tcp0->checksum;
 962               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 963                                      ip4_header_t, dst_address);
 964               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 965                                      ip4_header_t /* cheat */, length);
 966               tcp0->checksum = ip_csum_fold(sum0);
 967             }
 968           else
 969             {
 970               udp0->dst_port = new_dst_port0;
 971               udp0->checksum = 0;
 972             }
 973         }
 974       else
 975         {
 976           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 977             {
 978               sum0 = tcp0->checksum;
 979               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 980                                      ip4_header_t, dst_address);
 981               tcp0->checksum = ip_csum_fold(sum0);
 982             }
 983         }
 984       return 1;
 985     }
 986   return 0;
 987 }
 988
 989 static inline void
 990 snat_icmp_hairpinning (snat_main_t *sm,
 991                        vlib_buffer_t * b0,
 992                        ip4_header_t * ip0,
 993                        icmp46_header_t * icmp0)
 994 {
 995   snat_session_key_t key0, sm0;
 996   clib_bihash_kv_8_8_t kv0, value0;
 997   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 998   ip_csum_t sum0;
 999   snat_session_t *s0;
1000
1001   if (!icmp_is_error_message (icmp0))
1002     {
1003       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1004       u16 icmp_id0 = echo0->identifier;
1005       key0.addr = ip0->dst_address;
1006       key0.port = icmp_id0;
1007       key0.protocol = SNAT_PROTOCOL_ICMP;
1008       key0.fib_index = sm->outside_fib_index;
1009       kv0.key = key0.as_u64;
1010
1011       if (sm->num_workers > 1)
1012         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1013       else
1014         ti = sm->num_workers;
1015
1016       /* Check if destination is in active sessions */
1017       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1018                                   &value0))
1019         {
1020           /* or static mappings */
1021           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1022             {
1023               new_dst_addr0 = sm0.addr.as_u32;
1024               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1025             }
1026         }
1027       else
1028         {
1029           si = value0.value;
1030
1031           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1032           new_dst_addr0 = s0->in2out.addr.as_u32;
1033           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1034           echo0->identifier = s0->in2out.port;
1035           sum0 = icmp0->checksum;
1036           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1037                                  icmp_echo_header_t, identifier);
1038           icmp0->checksum = ip_csum_fold (sum0);
1039         }
1040
1041       /* Destination is behind the same NAT, use internal address and port */
1042       if (new_dst_addr0)
1043         {
1044           old_dst_addr0 = ip0->dst_address.as_u32;
1045           ip0->dst_address.as_u32 = new_dst_addr0;
1046           sum0 = ip0->checksum;
1047           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1048                                  ip4_header_t, dst_address);
1049           ip0->checksum = ip_csum_fold (sum0);
1050         }
1051     }
1052
1053 }
1054
1055 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1056                                          vlib_buffer_t * b0,
1057                                          ip4_header_t * ip0,
1058                                          icmp46_header_t * icmp0,
1059                                          u32 sw_if_index0,
1060                                          u32 rx_fib_index0,
1061                                          vlib_node_runtime_t * node,
1062                                          u32 next0,
1063                                          f64 now,
1064                                          u32 thread_index,
1065                                          snat_session_t ** p_s0)
1066 {
1067   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1068                       next0, thread_index, p_s0, 0);
1069   snat_session_t * s0 = *p_s0;
1070   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1071     {
1072       /* Hairpinning */
1073       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1074         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1075       /* Accounting */
1076       s0->last_heard = now;
1077       s0->total_pkts++;
1078       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1079       /* Per-user LRU list maintenance */
1080       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1081                          s0->per_user_index);
1082       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1083                           s0->per_user_list_head_index,
1084                           s0->per_user_index);
1085     }
1086   return next0;
1087 }
1088 static inline void
1089 snat_hairpinning_unknown_proto (snat_main_t *sm,
1090                                 vlib_buffer_t * b,
1091                                 ip4_header_t * ip)
1092 {
1093   u32 old_addr, new_addr = 0, ti = 0;
1094   clib_bihash_kv_8_8_t kv, value;
1095   clib_bihash_kv_16_8_t s_kv, s_value;
1096   nat_ed_ses_key_t key;
1097   snat_session_key_t m_key;
1098   snat_static_mapping_t *m;
1099   ip_csum_t sum;
1100   snat_session_t *s;
1101
1102   old_addr = ip->dst_address.as_u32;
1103   key.l_addr.as_u32 = ip->dst_address.as_u32;
1104   key.r_addr.as_u32 = ip->src_address.as_u32;
1105   key.fib_index = sm->outside_fib_index;
1106   key.proto = ip->protocol;
1107   key.r_port = 0;
1108   key.l_port = 0;
1109   s_kv.key[0] = key.as_u64[0];
1110   s_kv.key[1] = key.as_u64[1];
1111   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1112     {
1113       m_key.addr = ip->dst_address;
1114       m_key.fib_index = sm->outside_fib_index;
1115       m_key.port = 0;
1116       m_key.protocol = 0;
1117       kv.key = m_key.as_u64;
1118       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1119         return;
1120
1121       m = pool_elt_at_index (sm->static_mappings, value.value);
1122       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1123         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1124       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1125     }
1126   else
1127     {
1128       if (sm->num_workers > 1)
1129         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1130       else
1131         ti = sm->num_workers;
1132
1133       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1134       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1135         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1136       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1137     }
1138   sum = ip->checksum;
1139   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1140   ip->checksum = ip_csum_fold (sum);
1141 }
1142
1143 static snat_session_t *
1144 snat_in2out_unknown_proto (snat_main_t *sm,
1145                            vlib_buffer_t * b,
1146                            ip4_header_t * ip,
1147                            u32 rx_fib_index,
1148                            u32 thread_index,
1149                            f64 now,
1150                            vlib_main_t * vm,
1151                            vlib_node_runtime_t * node)
1152 {
1153   clib_bihash_kv_8_8_t kv, value;
1154   clib_bihash_kv_16_8_t s_kv, s_value;
1155   snat_static_mapping_t *m;
1156   snat_session_key_t m_key;
1157   u32 old_addr, new_addr = 0;
1158   ip_csum_t sum;
1159   snat_user_t *u;
1160   dlist_elt_t *head, *elt;
1161   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1162   u32 elt_index, head_index, ses_index;
1163   snat_session_t * s;
1164   nat_ed_ses_key_t key;
1165   u32 address_index = ~0;
1166   int i;
1167   u8 is_sm = 0;
1168
1169   old_addr = ip->src_address.as_u32;
1170
1171   key.l_addr = ip->src_address;
1172   key.r_addr = ip->dst_address;
1173   key.fib_index = rx_fib_index;
1174   key.proto = ip->protocol;
1175   key.l_port = 0;
1176   key.r_port = 0;
1177   s_kv.key[0] = key.as_u64[0];
1178   s_kv.key[1] = key.as_u64[1];
1179
1180   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1181     {
1182       s = pool_elt_at_index (tsm->sessions, s_value.value);
1183       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1184     }
1185   else
1186     {
1187       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1188         {
1189           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1190           nat_ipfix_logging_max_sessions(sm->max_translations);
1191           return 0;
1192         }
1193
1194       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1195                                   thread_index);
1196       if (!u)
1197         {
1198           clib_warning ("create NAT user failed");
1199           return 0;
1200         }
1201
1202       m_key.addr = ip->src_address;
1203       m_key.port = 0;
1204       m_key.protocol = 0;
1205       m_key.fib_index = rx_fib_index;
1206       kv.key = m_key.as_u64;
1207
1208       /* Try to find static mapping first */
1209       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1210         {
1211           m = pool_elt_at_index (sm->static_mappings, value.value);
1212           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1213           is_sm = 1;
1214           goto create_ses;
1215         }
1216       /* Fallback to 3-tuple key */
1217       else
1218         {
1219           /* Choose same out address as for TCP/UDP session to same destination */
1220           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1221             {
1222               head_index = u->sessions_per_user_list_head_index;
1223               head = pool_elt_at_index (tsm->list_pool, head_index);
1224               elt_index = head->next;
1225               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1226               ses_index = elt->value;
1227               while (ses_index != ~0)
1228                 {
1229                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1230                   elt_index = elt->next;
1231                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1232                   ses_index = elt->value;
1233
1234                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1235                     {
1236                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1237                       address_index = s->outside_address_index;
1238
1239                       key.fib_index = sm->outside_fib_index;
1240                       key.l_addr.as_u32 = new_addr;
1241                       s_kv.key[0] = key.as_u64[0];
1242                       s_kv.key[1] = key.as_u64[1];
1243                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1244                         break;
1245
1246                       goto create_ses;
1247                     }
1248                 }
1249             }
1250           key.fib_index = sm->outside_fib_index;
1251           for (i = 0; i < vec_len (sm->addresses); i++)
1252             {
1253               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1254               s_kv.key[0] = key.as_u64[0];
1255               s_kv.key[1] = key.as_u64[1];
1256               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1257                 {
1258                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1259                   address_index = i;
1260                   goto create_ses;
1261                 }
1262             }
1263           return 0;
1264         }
1265
1266 create_ses:
1267       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1268       if (!s)
1269         {
1270           clib_warning ("create NAT session failed");
1271           return 0;
1272         }
1273
1274       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1275       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1276       s->outside_address_index = address_index;
1277       s->out2in.addr.as_u32 = new_addr;
1278       s->out2in.fib_index = sm->outside_fib_index;
1279       s->in2out.addr.as_u32 = old_addr;
1280       s->in2out.fib_index = rx_fib_index;
1281       s->in2out.port = s->out2in.port = ip->protocol;
1282       if (is_sm)
1283         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1284       user_session_increment (sm, u, is_sm);
1285
1286       /* Add to lookup tables */
1287       key.l_addr.as_u32 = old_addr;
1288       key.r_addr = ip->dst_address;
1289       key.proto = ip->protocol;
1290       key.fib_index = rx_fib_index;
1291       s_kv.key[0] = key.as_u64[0];
1292       s_kv.key[1] = key.as_u64[1];
1293       s_kv.value = s - tsm->sessions;
1294       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1295         clib_warning ("in2out key add failed");
1296
1297       key.l_addr.as_u32 = new_addr;
1298       key.fib_index = sm->outside_fib_index;
1299       s_kv.key[0] = key.as_u64[0];
1300       s_kv.key[1] = key.as_u64[1];
1301       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1302         clib_warning ("out2in key add failed");
1303   }
1304
1305   /* Update IP checksum */
1306   sum = ip->checksum;
1307   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1308   ip->checksum = ip_csum_fold (sum);
1309
1310   /* Accounting */
1311   s->last_heard = now;
1312   s->total_pkts++;
1313   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1314   /* Per-user LRU list maintenance */
1315   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1316   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1317                       s->per_user_index);
1318
1319   /* Hairpinning */
1320   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1321     snat_hairpinning_unknown_proto(sm, b, ip);
1322
1323   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1324     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1325
1326   return s;
1327 }
1328
1329 static snat_session_t *
1330 snat_in2out_lb (snat_main_t *sm,
1331                 vlib_buffer_t * b,
1332                 ip4_header_t * ip,
1333                 u32 rx_fib_index,
1334                 u32 thread_index,
1335                 f64 now,
1336                 vlib_main_t * vm,
1337                 vlib_node_runtime_t * node)
1338 {
1339   nat_ed_ses_key_t key;
1340   clib_bihash_kv_16_8_t s_kv, s_value;
1341   udp_header_t *udp = ip4_next_header (ip);
1342   tcp_header_t *tcp = (tcp_header_t *) udp;
1343   snat_session_t *s = 0;
1344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1345   u32 old_addr, new_addr;
1346   u16 new_port, old_port;
1347   ip_csum_t sum;
1348   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1349   snat_session_key_t e_key, l_key;
1350   snat_user_t *u;
1351   u8 lb;
1352
1353   old_addr = ip->src_address.as_u32;
1354
1355   key.l_addr = ip->src_address;
1356   key.r_addr = ip->dst_address;
1357   key.fib_index = rx_fib_index;
1358   key.proto = ip->protocol;
1359   key.r_port = udp->dst_port;
1360   key.l_port = udp->src_port;
1361   s_kv.key[0] = key.as_u64[0];
1362   s_kv.key[1] = key.as_u64[1];
1363
1364   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1365     {
1366       s = pool_elt_at_index (tsm->sessions, s_value.value);
1367       if (is_fwd_bypass_session (s))
1368         return 0;
1369     }
1370   else
1371     {
1372       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1373         {
1374           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1375           nat_ipfix_logging_max_sessions(sm->max_translations);
1376           return 0;
1377         }
1378
1379       l_key.addr = ip->src_address;
1380       l_key.port = udp->src_port;
1381       l_key.protocol = proto;
1382       l_key.fib_index = rx_fib_index;
1383       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1384         return 0;
1385
1386       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1387                                   thread_index);
1388       if (!u)
1389         {
1390           clib_warning ("create NAT user failed");
1391           return 0;
1392         }
1393
1394       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1395       if (!s)
1396         {
1397           clib_warning ("create NAT session failed");
1398           return 0;
1399         }
1400
1401       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1402       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1403       if (lb)
1404         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1405       s->outside_address_index = ~0;
1406       s->in2out = l_key;
1407       s->out2in = e_key;
1408       s->out2in.protocol = l_key.protocol;
1409       user_session_increment (sm, u, 1 /* static */);
1410
1411       /* Add to lookup tables */
1412       s_kv.value = s - tsm->sessions;
1413       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1414         clib_warning ("in2out-ed key add failed");
1415
1416       key.l_addr = e_key.addr;
1417       key.fib_index = e_key.fib_index;
1418       key.l_port = e_key.port;
1419       s_kv.key[0] = key.as_u64[0];
1420       s_kv.key[1] = key.as_u64[1];
1421       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1422         clib_warning ("out2in-ed key add failed");
1423     }
1424
1425   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1426
1427   /* Update IP checksum */
1428   sum = ip->checksum;
1429   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1430   if (is_twice_nat_session (s))
1431     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1432                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1433   ip->checksum = ip_csum_fold (sum);
1434
1435   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1436     {
1437       old_port = tcp->src_port;
1438       tcp->src_port = s->out2in.port;
1439       new_port = tcp->src_port;
1440
1441       sum = tcp->checksum;
1442       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1443       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1444       if (is_twice_nat_session (s))
1445         {
1446           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1447                                 s->ext_host_addr.as_u32, ip4_header_t,
1448                                 dst_address);
1449           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1450                                 ip4_header_t, length);
1451           tcp->dst_port = s->ext_host_port;
1452           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1453         }
1454       tcp->checksum = ip_csum_fold(sum);
1455     }
1456   else
1457     {
1458       udp->src_port = s->out2in.port;
1459       if (is_twice_nat_session (s))
1460         {
1461           udp->dst_port = s->ext_host_port;
1462           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1463         }
1464       udp->checksum = 0;
1465     }
1466
1467   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1468     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1469
1470   /* Accounting */
1471   s->last_heard = now;
1472   s->total_pkts++;
1473   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1474   /* Per-user LRU list maintenance */
1475   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1476   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1477                       s->per_user_index);
1478   return s;
1479 }
1480
1481 static inline uword
1482 snat_in2out_node_fn_inline (vlib_main_t * vm,
1483                             vlib_node_runtime_t * node,
1484                             vlib_frame_t * frame, int is_slow_path,
1485                             int is_output_feature)
1486 {
1487   u32 n_left_from, * from, * to_next;
1488   snat_in2out_next_t next_index;
1489   u32 pkts_processed = 0;
1490   snat_main_t * sm = &snat_main;
1491   f64 now = vlib_time_now (vm);
1492   u32 stats_node_index;
1493   u32 thread_index = vlib_get_thread_index ();
1494
1495   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1496     snat_in2out_node.index;
1497
1498   from = vlib_frame_vector_args (frame);
1499   n_left_from = frame->n_vectors;
1500   next_index = node->cached_next_index;
1501
1502   while (n_left_from > 0)
1503     {
1504       u32 n_left_to_next;
1505
1506       vlib_get_next_frame (vm, node, next_index,
1507                            to_next, n_left_to_next);
1508
1509       while (n_left_from >= 4 && n_left_to_next >= 2)
1510         {
1511           u32 bi0, bi1;
1512           vlib_buffer_t * b0, * b1;
1513           u32 next0, next1;
1514           u32 sw_if_index0, sw_if_index1;
1515           ip4_header_t * ip0, * ip1;
1516           ip_csum_t sum0, sum1;
1517           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1518           u16 old_port0, new_port0, old_port1, new_port1;
1519           udp_header_t * udp0, * udp1;
1520           tcp_header_t * tcp0, * tcp1;
1521           icmp46_header_t * icmp0, * icmp1;
1522           snat_session_key_t key0, key1;
1523           u32 rx_fib_index0, rx_fib_index1;
1524           u32 proto0, proto1;
1525           snat_session_t * s0 = 0, * s1 = 0;
1526           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1527           u32 iph_offset0 = 0, iph_offset1 = 0;
1528
1529           /* Prefetch next iteration. */
1530           {
1531             vlib_buffer_t * p2, * p3;
1532
1533             p2 = vlib_get_buffer (vm, from[2]);
1534             p3 = vlib_get_buffer (vm, from[3]);
1535
1536             vlib_prefetch_buffer_header (p2, LOAD);
1537             vlib_prefetch_buffer_header (p3, LOAD);
1538
1539             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1540             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1541           }
1542
1543           /* speculatively enqueue b0 and b1 to the current next frame */
1544           to_next[0] = bi0 = from[0];
1545           to_next[1] = bi1 = from[1];
1546           from += 2;
1547           to_next += 2;
1548           n_left_from -= 2;
1549           n_left_to_next -= 2;
1550
1551           b0 = vlib_get_buffer (vm, bi0);
1552           b1 = vlib_get_buffer (vm, bi1);
1553
1554           if (is_output_feature)
1555             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1556
1557           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1558                  iph_offset0);
1559
1560           udp0 = ip4_next_header (ip0);
1561           tcp0 = (tcp_header_t *) udp0;
1562           icmp0 = (icmp46_header_t *) udp0;
1563
1564           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1565           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1566                                    sw_if_index0);
1567
1568           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1569
1570           if (PREDICT_FALSE(ip0->ttl == 1))
1571             {
1572               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1573               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1574                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1575                                            0);
1576               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1577               goto trace00;
1578             }
1579
1580           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1581
1582           /* Next configured feature, probably ip4-lookup */
1583           if (is_slow_path)
1584             {
1585               if (PREDICT_FALSE (proto0 == ~0))
1586                 {
1587                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1588                                                   thread_index, now, vm, node);
1589                   if (!s0)
1590                     next0 = SNAT_IN2OUT_NEXT_DROP;
1591                   goto trace00;
1592                 }
1593
1594               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1595                 {
1596                   next0 = icmp_in2out_slow_path
1597                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1598                      node, next0, now, thread_index, &s0);
1599                   goto trace00;
1600                 }
1601             }
1602           else
1603             {
1604               if (is_output_feature)
1605                 {
1606                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
1607                     goto trace00;
1608                 }
1609
1610               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1611                 {
1612                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1613                   goto trace00;
1614                 }
1615
1616               if (ip4_is_fragment (ip0))
1617                 {
1618                   next0 = SNAT_IN2OUT_NEXT_REASS;
1619                   goto trace00;
1620                 }
1621             }
1622
1623           key0.addr = ip0->src_address;
1624           key0.port = udp0->src_port;
1625           key0.protocol = proto0;
1626           key0.fib_index = rx_fib_index0;
1627
1628           kv0.key = key0.as_u64;
1629
1630           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1631               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1632             {
1633               if (is_slow_path)
1634                 {
1635                   if (is_output_feature)
1636                     {
1637                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1638                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1639                         goto trace00;
1640                     }
1641                   else
1642                     {
1643                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1644                           ip0, proto0, rx_fib_index0, thread_index)))
1645                         goto trace00;
1646                     }
1647
1648                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1649                                      &s0, node, next0, thread_index);
1650                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1651                     goto trace00;
1652                 }
1653               else
1654                 {
1655                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1656                   goto trace00;
1657                 }
1658             }
1659           else
1660             {
1661               if (PREDICT_FALSE (value0.value == ~0ULL))
1662                 {
1663                   if (is_slow_path)
1664                     {
1665                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1666                                           thread_index, now, vm, node);
1667                       if (!s0 && !sm->forwarding_enabled)
1668                         next0 = SNAT_IN2OUT_NEXT_DROP;
1669                       goto trace00;
1670                     }
1671                   else
1672                     {
1673                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1674                       goto trace00;
1675                     }
1676                 }
1677               else
1678                 {
1679                   s0 = pool_elt_at_index (
1680                     sm->per_thread_data[thread_index].sessions,
1681                     value0.value);
1682                 }
1683             }
1684
1685           b0->flags |= VNET_BUFFER_F_IS_NATED;
1686
1687           old_addr0 = ip0->src_address.as_u32;
1688           ip0->src_address = s0->out2in.addr;
1689           new_addr0 = ip0->src_address.as_u32;
1690           if (!is_output_feature)
1691             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1692
1693           sum0 = ip0->checksum;
1694           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1695                                  ip4_header_t,
1696                                  src_address /* changed member */);
1697           ip0->checksum = ip_csum_fold (sum0);
1698
1699           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1700             {
1701               old_port0 = tcp0->src_port;
1702               tcp0->src_port = s0->out2in.port;
1703               new_port0 = tcp0->src_port;
1704
1705               sum0 = tcp0->checksum;
1706               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1707                                      ip4_header_t,
1708                                      dst_address /* changed member */);
1709               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1710                                      ip4_header_t /* cheat */,
1711                                      length /* changed member */);
1712               tcp0->checksum = ip_csum_fold(sum0);
1713             }
1714           else
1715             {
1716               old_port0 = udp0->src_port;
1717               udp0->src_port = s0->out2in.port;
1718               udp0->checksum = 0;
1719             }
1720
1721           /* Accounting */
1722           s0->last_heard = now;
1723           s0->total_pkts++;
1724           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1725           /* Per-user LRU list maintenance */
1726           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1727                              s0->per_user_index);
1728           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1729                               s0->per_user_list_head_index,
1730                               s0->per_user_index);
1731         trace00:
1732
1733           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1734                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1735             {
1736               snat_in2out_trace_t *t =
1737                  vlib_add_trace (vm, node, b0, sizeof (*t));
1738               t->is_slow_path = is_slow_path;
1739               t->sw_if_index = sw_if_index0;
1740               t->next_index = next0;
1741                   t->session_index = ~0;
1742               if (s0)
1743                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1744             }
1745
1746           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1747
1748           if (is_output_feature)
1749             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1750
1751           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1752                  iph_offset1);
1753
1754           udp1 = ip4_next_header (ip1);
1755           tcp1 = (tcp_header_t *) udp1;
1756           icmp1 = (icmp46_header_t *) udp1;
1757
1758           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1759           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1760                                    sw_if_index1);
1761
1762           if (PREDICT_FALSE(ip1->ttl == 1))
1763             {
1764               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1765               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1766                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1767                                            0);
1768               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1769               goto trace01;
1770             }
1771
1772           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1773
1774           /* Next configured feature, probably ip4-lookup */
1775           if (is_slow_path)
1776             {
1777               if (PREDICT_FALSE (proto1 == ~0))
1778                 {
1779                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1780                                                   thread_index, now, vm, node);
1781                   if (!s1)
1782                     next1 = SNAT_IN2OUT_NEXT_DROP;
1783                   goto trace01;
1784                 }
1785
1786               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1787                 {
1788                   next1 = icmp_in2out_slow_path
1789                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1790                      next1, now, thread_index, &s1);
1791                   goto trace01;
1792                 }
1793             }
1794           else
1795             {
1796               if (is_output_feature)
1797                 {
1798                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1, thread_index)))
1799                     goto trace01;
1800                 }
1801
1802               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1803                 {
1804                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1805                   goto trace01;
1806                 }
1807
1808               if (ip4_is_fragment (ip1))
1809                 {
1810                   next1 = SNAT_IN2OUT_NEXT_REASS;
1811                   goto trace01;
1812                 }
1813             }
1814
1815           key1.addr = ip1->src_address;
1816           key1.port = udp1->src_port;
1817           key1.protocol = proto1;
1818           key1.fib_index = rx_fib_index1;
1819
1820           kv1.key = key1.as_u64;
1821
1822             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1823                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1824             {
1825               if (is_slow_path)
1826                 {
1827                   if (is_output_feature)
1828                     {
1829                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1830                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1831                         goto trace01;
1832                     }
1833                   else
1834                     {
1835                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1836                           ip1, proto1, rx_fib_index1, thread_index)))
1837                         goto trace01;
1838                     }
1839
1840                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1841                                      &s1, node, next1, thread_index);
1842                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1843                     goto trace01;
1844                 }
1845               else
1846                 {
1847                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1848                   goto trace01;
1849                 }
1850             }
1851           else
1852             {
1853               if (PREDICT_FALSE (value1.value == ~0ULL))
1854                 {
1855                   if (is_slow_path)
1856                     {
1857                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1858                                           thread_index, now, vm, node);
1859                       if (!s1 && !sm->forwarding_enabled)
1860                         next1 = SNAT_IN2OUT_NEXT_DROP;
1861                       goto trace01;
1862                     }
1863                   else
1864                     {
1865                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1866                       goto trace01;
1867                     }
1868                 }
1869               else
1870                 {
1871                   s1 = pool_elt_at_index (
1872                     sm->per_thread_data[thread_index].sessions,
1873                     value1.value);
1874                 }
1875             }
1876
1877           b1->flags |= VNET_BUFFER_F_IS_NATED;
1878
1879           old_addr1 = ip1->src_address.as_u32;
1880           ip1->src_address = s1->out2in.addr;
1881           new_addr1 = ip1->src_address.as_u32;
1882           if (!is_output_feature)
1883             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1884
1885           sum1 = ip1->checksum;
1886           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1887                                  ip4_header_t,
1888                                  src_address /* changed member */);
1889           ip1->checksum = ip_csum_fold (sum1);
1890
1891           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1892             {
1893               old_port1 = tcp1->src_port;
1894               tcp1->src_port = s1->out2in.port;
1895               new_port1 = tcp1->src_port;
1896
1897               sum1 = tcp1->checksum;
1898               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1899                                      ip4_header_t,
1900                                      dst_address /* changed member */);
1901               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1902                                      ip4_header_t /* cheat */,
1903                                      length /* changed member */);
1904               tcp1->checksum = ip_csum_fold(sum1);
1905             }
1906           else
1907             {
1908               old_port1 = udp1->src_port;
1909               udp1->src_port = s1->out2in.port;
1910               udp1->checksum = 0;
1911             }
1912
1913           /* Accounting */
1914           s1->last_heard = now;
1915           s1->total_pkts++;
1916           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1917           /* Per-user LRU list maintenance */
1918           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1919                              s1->per_user_index);
1920           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1921                               s1->per_user_list_head_index,
1922                               s1->per_user_index);
1923         trace01:
1924
1925           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1926                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1927             {
1928               snat_in2out_trace_t *t =
1929                  vlib_add_trace (vm, node, b1, sizeof (*t));
1930               t->sw_if_index = sw_if_index1;
1931               t->next_index = next1;
1932               t->session_index = ~0;
1933               if (s1)
1934                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1935             }
1936
1937           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1938
1939           /* verify speculative enqueues, maybe switch current next frame */
1940           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1941                                            to_next, n_left_to_next,
1942                                            bi0, bi1, next0, next1);
1943         }
1944
1945       while (n_left_from > 0 && n_left_to_next > 0)
1946         {
1947           u32 bi0;
1948           vlib_buffer_t * b0;
1949           u32 next0;
1950           u32 sw_if_index0;
1951           ip4_header_t * ip0;
1952           ip_csum_t sum0;
1953           u32 new_addr0, old_addr0;
1954           u16 old_port0, new_port0;
1955           udp_header_t * udp0;
1956           tcp_header_t * tcp0;
1957           icmp46_header_t * icmp0;
1958           snat_session_key_t key0;
1959           u32 rx_fib_index0;
1960           u32 proto0;
1961           snat_session_t * s0 = 0;
1962           clib_bihash_kv_8_8_t kv0, value0;
1963           u32 iph_offset0 = 0;
1964
1965           /* speculatively enqueue b0 to the current next frame */
1966           bi0 = from[0];
1967           to_next[0] = bi0;
1968           from += 1;
1969           to_next += 1;
1970           n_left_from -= 1;
1971           n_left_to_next -= 1;
1972
1973           b0 = vlib_get_buffer (vm, bi0);
1974           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1975
1976           if (is_output_feature)
1977             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1978
1979           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1980                  iph_offset0);
1981
1982           udp0 = ip4_next_header (ip0);
1983           tcp0 = (tcp_header_t *) udp0;
1984           icmp0 = (icmp46_header_t *) udp0;
1985
1986           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1987           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1988                                    sw_if_index0);
1989
1990           if (PREDICT_FALSE(ip0->ttl == 1))
1991             {
1992               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1993               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1994                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1995                                            0);
1996               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1997               goto trace0;
1998             }
1999
2000           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2001
2002           /* Next configured feature, probably ip4-lookup */
2003           if (is_slow_path)
2004             {
2005               if (PREDICT_FALSE (proto0 == ~0))
2006                 {
2007                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2008                                                   thread_index, now, vm, node);
2009                   if (!s0)
2010                     next0 = SNAT_IN2OUT_NEXT_DROP;
2011                   goto trace0;
2012                 }
2013
2014               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2015                 {
2016                   next0 = icmp_in2out_slow_path
2017                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2018                      next0, now, thread_index, &s0);
2019                   goto trace0;
2020                 }
2021             }
2022           else
2023             {
2024                if (is_output_feature)
2025                 {
2026                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
2027                     goto trace0;
2028                 }
2029
2030               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2031                 {
2032                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2033                   goto trace0;
2034                 }
2035
2036               if (ip4_is_fragment (ip0))
2037                 {
2038                   next0 = SNAT_IN2OUT_NEXT_REASS;
2039                   goto trace0;
2040                 }
2041             }
2042
2043           key0.addr = ip0->src_address;
2044           key0.port = udp0->src_port;
2045           key0.protocol = proto0;
2046           key0.fib_index = rx_fib_index0;
2047
2048           kv0.key = key0.as_u64;
2049
2050           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2051                                       &kv0, &value0))
2052             {
2053               if (is_slow_path)
2054                 {
2055                   if (is_output_feature)
2056                     {
2057                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2058                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2059                         goto trace0;
2060                     }
2061                   else
2062                     {
2063                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2064                           ip0, proto0, rx_fib_index0, thread_index)))
2065                         goto trace0;
2066                     }
2067
2068                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2069                                      &s0, node, next0, thread_index);
2070
2071                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2072                     goto trace0;
2073                 }
2074               else
2075                 {
2076                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2077                   goto trace0;
2078                 }
2079             }
2080           else
2081             {
2082               if (PREDICT_FALSE (value0.value == ~0ULL))
2083                 {
2084                   if (is_slow_path)
2085                     {
2086                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2087                                           thread_index, now, vm, node);
2088                       if (!s0 && !sm->forwarding_enabled)
2089                         next0 = SNAT_IN2OUT_NEXT_DROP;
2090                       goto trace0;
2091                     }
2092                   else
2093                     {
2094                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2095                       goto trace0;
2096                     }
2097                 }
2098               else
2099                 {
2100                   s0 = pool_elt_at_index (
2101                     sm->per_thread_data[thread_index].sessions,
2102                     value0.value);
2103                 }
2104             }
2105
2106           b0->flags |= VNET_BUFFER_F_IS_NATED;
2107
2108           old_addr0 = ip0->src_address.as_u32;
2109           ip0->src_address = s0->out2in.addr;
2110           new_addr0 = ip0->src_address.as_u32;
2111           if (!is_output_feature)
2112             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2113
2114           sum0 = ip0->checksum;
2115           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2116                                  ip4_header_t,
2117                                  src_address /* changed member */);
2118           ip0->checksum = ip_csum_fold (sum0);
2119
2120           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2121             {
2122               old_port0 = tcp0->src_port;
2123               tcp0->src_port = s0->out2in.port;
2124               new_port0 = tcp0->src_port;
2125
2126               sum0 = tcp0->checksum;
2127               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2128                                      ip4_header_t,
2129                                      dst_address /* changed member */);
2130               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2131                                      ip4_header_t /* cheat */,
2132                                      length /* changed member */);
2133               tcp0->checksum = ip_csum_fold(sum0);
2134             }
2135           else
2136             {
2137               old_port0 = udp0->src_port;
2138               udp0->src_port = s0->out2in.port;
2139               udp0->checksum = 0;
2140             }
2141
2142           /* Accounting */
2143           s0->last_heard = now;
2144           s0->total_pkts++;
2145           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2146           /* Per-user LRU list maintenance */
2147           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2148                              s0->per_user_index);
2149           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2150                               s0->per_user_list_head_index,
2151                               s0->per_user_index);
2152
2153         trace0:
2154           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2155                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2156             {
2157               snat_in2out_trace_t *t =
2158                  vlib_add_trace (vm, node, b0, sizeof (*t));
2159               t->is_slow_path = is_slow_path;
2160               t->sw_if_index = sw_if_index0;
2161               t->next_index = next0;
2162                   t->session_index = ~0;
2163               if (s0)
2164                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2165             }
2166
2167           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2168
2169           /* verify speculative enqueue, maybe switch current next frame */
2170           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2171                                            to_next, n_left_to_next,
2172                                            bi0, next0);
2173         }
2174
2175       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2176     }
2177
2178   vlib_node_increment_counter (vm, stats_node_index,
2179                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2180                                pkts_processed);
2181   return frame->n_vectors;
2182 }
2183
2184 static uword
2185 snat_in2out_fast_path_fn (vlib_main_t * vm,
2186                           vlib_node_runtime_t * node,
2187                           vlib_frame_t * frame)
2188 {
2189   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2190 }
2191
2192 VLIB_REGISTER_NODE (snat_in2out_node) = {
2193   .function = snat_in2out_fast_path_fn,
2194   .name = "nat44-in2out",
2195   .vector_size = sizeof (u32),
2196   .format_trace = format_snat_in2out_trace,
2197   .type = VLIB_NODE_TYPE_INTERNAL,
2198
2199   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2200   .error_strings = snat_in2out_error_strings,
2201
2202   .runtime_data_bytes = sizeof (snat_runtime_t),
2203
2204   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2205
2206   /* edit / add dispositions here */
2207   .next_nodes = {
2208     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2209     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2210     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2211     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2212     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2213   },
2214 };
2215
2216 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2217
2218 static uword
2219 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2220                                  vlib_node_runtime_t * node,
2221                                  vlib_frame_t * frame)
2222 {
2223   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2224 }
2225
2226 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2227   .function = snat_in2out_output_fast_path_fn,
2228   .name = "nat44-in2out-output",
2229   .vector_size = sizeof (u32),
2230   .format_trace = format_snat_in2out_trace,
2231   .type = VLIB_NODE_TYPE_INTERNAL,
2232
2233   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2234   .error_strings = snat_in2out_error_strings,
2235
2236   .runtime_data_bytes = sizeof (snat_runtime_t),
2237
2238   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2239
2240   /* edit / add dispositions here */
2241   .next_nodes = {
2242     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2243     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2244     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2245     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2246     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2247   },
2248 };
2249
2250 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2251                               snat_in2out_output_fast_path_fn);
2252
2253 static uword
2254 snat_in2out_slow_path_fn (vlib_main_t * vm,
2255                           vlib_node_runtime_t * node,
2256                           vlib_frame_t * frame)
2257 {
2258   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2259 }
2260
2261 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2262   .function = snat_in2out_slow_path_fn,
2263   .name = "nat44-in2out-slowpath",
2264   .vector_size = sizeof (u32),
2265   .format_trace = format_snat_in2out_trace,
2266   .type = VLIB_NODE_TYPE_INTERNAL,
2267
2268   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2269   .error_strings = snat_in2out_error_strings,
2270
2271   .runtime_data_bytes = sizeof (snat_runtime_t),
2272
2273   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2274
2275   /* edit / add dispositions here */
2276   .next_nodes = {
2277     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2278     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2279     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2280     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2281     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2282   },
2283 };
2284
2285 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2286                               snat_in2out_slow_path_fn);
2287
2288 static uword
2289 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2290                                  vlib_node_runtime_t * node,
2291                                  vlib_frame_t * frame)
2292 {
2293   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2294 }
2295
2296 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2297   .function = snat_in2out_output_slow_path_fn,
2298   .name = "nat44-in2out-output-slowpath",
2299   .vector_size = sizeof (u32),
2300   .format_trace = format_snat_in2out_trace,
2301   .type = VLIB_NODE_TYPE_INTERNAL,
2302
2303   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2304   .error_strings = snat_in2out_error_strings,
2305
2306   .runtime_data_bytes = sizeof (snat_runtime_t),
2307
2308   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2309
2310   /* edit / add dispositions here */
2311   .next_nodes = {
2312     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2313     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2314     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2315     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2316     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2317   },
2318 };
2319
2320 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2321                               snat_in2out_output_slow_path_fn);
2322
2323 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2324
2325 static uword
2326 nat44_hairpinning_fn (vlib_main_t * vm,
2327                       vlib_node_runtime_t * node,
2328                       vlib_frame_t * frame)
2329 {
2330   u32 n_left_from, * from, * to_next;
2331   snat_in2out_next_t next_index;
2332   u32 pkts_processed = 0;
2333   snat_main_t * sm = &snat_main;
2334   vnet_feature_main_t *fm = &feature_main;
2335   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2336   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2337
2338   from = vlib_frame_vector_args (frame);
2339   n_left_from = frame->n_vectors;
2340   next_index = node->cached_next_index;
2341
2342   while (n_left_from > 0)
2343     {
2344       u32 n_left_to_next;
2345
2346       vlib_get_next_frame (vm, node, next_index,
2347                            to_next, n_left_to_next);
2348
2349       while (n_left_from > 0 && n_left_to_next > 0)
2350         {
2351           u32 bi0;
2352           vlib_buffer_t * b0;
2353           u32 next0;
2354           ip4_header_t * ip0;
2355           u32 proto0;
2356           udp_header_t * udp0;
2357           tcp_header_t * tcp0;
2358
2359           /* speculatively enqueue b0 to the current next frame */
2360           bi0 = from[0];
2361           to_next[0] = bi0;
2362           from += 1;
2363           to_next += 1;
2364           n_left_from -= 1;
2365           n_left_to_next -= 1;
2366
2367           b0 = vlib_get_buffer (vm, bi0);
2368           ip0 = vlib_buffer_get_current (b0);
2369           udp0 = ip4_next_header (ip0);
2370           tcp0 = (tcp_header_t *) udp0;
2371
2372           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2373
2374           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2375                                 &next0, 0);
2376
2377           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2378             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2379
2380           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2381
2382           /* verify speculative enqueue, maybe switch current next frame */
2383           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2384                                            to_next, n_left_to_next,
2385                                            bi0, next0);
2386          }
2387
2388       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2389     }
2390
2391   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2392                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2393                                pkts_processed);
2394   return frame->n_vectors;
2395 }
2396
2397 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2398   .function = nat44_hairpinning_fn,
2399   .name = "nat44-hairpinning",
2400   .vector_size = sizeof (u32),
2401   .type = VLIB_NODE_TYPE_INTERNAL,
2402   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2403   .error_strings = snat_in2out_error_strings,
2404   .n_next_nodes = 2,
2405   .next_nodes = {
2406     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2407     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2408   },
2409 };
2410
2411 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2412                               nat44_hairpinning_fn);
2413
2414 static inline void
2415 nat44_reass_hairpinning (snat_main_t *sm,
2416                          vlib_buffer_t * b0,
2417                          ip4_header_t * ip0,
2418                          u16 sport,
2419                          u16 dport,
2420                          u32 proto0)
2421 {
2422   snat_session_key_t key0, sm0;
2423   snat_session_t * s0;
2424   clib_bihash_kv_8_8_t kv0, value0;
2425   ip_csum_t sum0;
2426   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2427   u16 new_dst_port0, old_dst_port0;
2428   udp_header_t * udp0;
2429   tcp_header_t * tcp0;
2430
2431   key0.addr = ip0->dst_address;
2432   key0.port = dport;
2433   key0.protocol = proto0;
2434   key0.fib_index = sm->outside_fib_index;
2435   kv0.key = key0.as_u64;
2436
2437   udp0 = ip4_next_header (ip0);
2438
2439   /* Check if destination is static mappings */
2440   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2441     {
2442       new_dst_addr0 = sm0.addr.as_u32;
2443       new_dst_port0 = sm0.port;
2444       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2445     }
2446   /* or active sessions */
2447   else
2448     {
2449       if (sm->num_workers > 1)
2450         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2451       else
2452         ti = sm->num_workers;
2453
2454       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2455         {
2456           si = value0.value;
2457           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2458           new_dst_addr0 = s0->in2out.addr.as_u32;
2459           new_dst_port0 = s0->in2out.port;
2460           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2461         }
2462     }
2463
2464   /* Destination is behind the same NAT, use internal address and port */
2465   if (new_dst_addr0)
2466     {
2467       old_dst_addr0 = ip0->dst_address.as_u32;
2468       ip0->dst_address.as_u32 = new_dst_addr0;
2469       sum0 = ip0->checksum;
2470       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2471                              ip4_header_t, dst_address);
2472       ip0->checksum = ip_csum_fold (sum0);
2473
2474       old_dst_port0 = dport;
2475       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2476                        ip4_is_first_fragment (ip0)))
2477         {
2478           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2479             {
2480               tcp0 = ip4_next_header (ip0);
2481               tcp0->dst = new_dst_port0;
2482               sum0 = tcp0->checksum;
2483               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2484                                      ip4_header_t, dst_address);
2485               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2486                                      ip4_header_t /* cheat */, length);
2487               tcp0->checksum = ip_csum_fold(sum0);
2488             }
2489           else
2490             {
2491               udp0->dst_port = new_dst_port0;
2492               udp0->checksum = 0;
2493             }
2494         }
2495       else
2496         {
2497           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2498             {
2499               tcp0 = ip4_next_header (ip0);
2500               sum0 = tcp0->checksum;
2501               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2502                                      ip4_header_t, dst_address);
2503               tcp0->checksum = ip_csum_fold(sum0);
2504             }
2505         }
2506     }
2507 }
2508
2509 static uword
2510 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2511                             vlib_node_runtime_t * node,
2512                             vlib_frame_t * frame)
2513 {
2514   u32 n_left_from, *from, *to_next;
2515   snat_in2out_next_t next_index;
2516   u32 pkts_processed = 0;
2517   snat_main_t *sm = &snat_main;
2518   f64 now = vlib_time_now (vm);
2519   u32 thread_index = vlib_get_thread_index ();
2520   snat_main_per_thread_data_t *per_thread_data =
2521     &sm->per_thread_data[thread_index];
2522   u32 *fragments_to_drop = 0;
2523   u32 *fragments_to_loopback = 0;
2524
2525   from = vlib_frame_vector_args (frame);
2526   n_left_from = frame->n_vectors;
2527   next_index = node->cached_next_index;
2528
2529   while (n_left_from > 0)
2530     {
2531       u32 n_left_to_next;
2532
2533       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2534
2535       while (n_left_from > 0 && n_left_to_next > 0)
2536        {
2537           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2538           vlib_buffer_t *b0;
2539           u32 next0;
2540           u8 cached0 = 0;
2541           ip4_header_t *ip0;
2542           nat_reass_ip4_t *reass0;
2543           udp_header_t * udp0;
2544           tcp_header_t * tcp0;
2545           snat_session_key_t key0;
2546           clib_bihash_kv_8_8_t kv0, value0;
2547           snat_session_t * s0 = 0;
2548           u16 old_port0, new_port0;
2549           ip_csum_t sum0;
2550
2551           /* speculatively enqueue b0 to the current next frame */
2552           bi0 = from[0];
2553           to_next[0] = bi0;
2554           from += 1;
2555           to_next += 1;
2556           n_left_from -= 1;
2557           n_left_to_next -= 1;
2558
2559           b0 = vlib_get_buffer (vm, bi0);
2560           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2561
2562           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2563           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2564                                                                sw_if_index0);
2565
2566           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2567             {
2568               next0 = SNAT_IN2OUT_NEXT_DROP;
2569               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2570               goto trace0;
2571             }
2572
2573           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2574           udp0 = ip4_next_header (ip0);
2575           tcp0 = (tcp_header_t *) udp0;
2576           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2577
2578           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2579                                                  ip0->dst_address,
2580                                                  ip0->fragment_id,
2581                                                  ip0->protocol,
2582                                                  1,
2583                                                  &fragments_to_drop);
2584
2585           if (PREDICT_FALSE (!reass0))
2586             {
2587               next0 = SNAT_IN2OUT_NEXT_DROP;
2588               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2589               goto trace0;
2590             }
2591
2592           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2593             {
2594               key0.addr = ip0->src_address;
2595               key0.port = udp0->src_port;
2596               key0.protocol = proto0;
2597               key0.fib_index = rx_fib_index0;
2598               kv0.key = key0.as_u64;
2599
2600               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2601                 {
2602                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2603                       ip0, proto0, rx_fib_index0, thread_index)))
2604                     goto trace0;
2605
2606                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2607                                      &s0, node, next0, thread_index);
2608
2609                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2610                     goto trace0;
2611
2612                   reass0->sess_index = s0 - per_thread_data->sessions;
2613                 }
2614               else
2615                 {
2616                   s0 = pool_elt_at_index (per_thread_data->sessions,
2617                                           value0.value);
2618                   reass0->sess_index = value0.value;
2619                 }
2620               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2621             }
2622           else
2623             {
2624               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2625                 {
2626                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2627                     {
2628                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2629                       next0 = SNAT_IN2OUT_NEXT_DROP;
2630                       goto trace0;
2631                     }
2632                   cached0 = 1;
2633                   goto trace0;
2634                 }
2635               s0 = pool_elt_at_index (per_thread_data->sessions,
2636                                       reass0->sess_index);
2637             }
2638
2639           old_addr0 = ip0->src_address.as_u32;
2640           ip0->src_address = s0->out2in.addr;
2641           new_addr0 = ip0->src_address.as_u32;
2642           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2643
2644           sum0 = ip0->checksum;
2645           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2646                                  ip4_header_t,
2647                                  src_address /* changed member */);
2648           ip0->checksum = ip_csum_fold (sum0);
2649
2650           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2651             {
2652               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2653                 {
2654                   old_port0 = tcp0->src_port;
2655                   tcp0->src_port = s0->out2in.port;
2656                   new_port0 = tcp0->src_port;
2657
2658                   sum0 = tcp0->checksum;
2659                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2660                                          ip4_header_t,
2661                                          dst_address /* changed member */);
2662                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2663                                          ip4_header_t /* cheat */,
2664                                          length /* changed member */);
2665                   tcp0->checksum = ip_csum_fold(sum0);
2666                 }
2667               else
2668                 {
2669                   old_port0 = udp0->src_port;
2670                   udp0->src_port = s0->out2in.port;
2671                   udp0->checksum = 0;
2672                 }
2673             }
2674
2675           /* Hairpinning */
2676           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2677                                    s0->ext_host_port, proto0);
2678
2679           /* Accounting */
2680           s0->last_heard = now;
2681           s0->total_pkts++;
2682           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2683           /* Per-user LRU list maintenance */
2684           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2685                              s0->per_user_index);
2686           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2687                               s0->per_user_list_head_index,
2688                               s0->per_user_index);
2689
2690         trace0:
2691           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2692                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2693             {
2694               nat44_in2out_reass_trace_t *t =
2695                  vlib_add_trace (vm, node, b0, sizeof (*t));
2696               t->cached = cached0;
2697               t->sw_if_index = sw_if_index0;
2698               t->next_index = next0;
2699             }
2700
2701           if (cached0)
2702             {
2703               n_left_to_next++;
2704               to_next--;
2705             }
2706           else
2707             {
2708               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2709
2710               /* verify speculative enqueue, maybe switch current next frame */
2711               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2712                                                to_next, n_left_to_next,
2713                                                bi0, next0);
2714             }
2715
2716           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2717             {
2718               from = vlib_frame_vector_args (frame);
2719               u32 len = vec_len (fragments_to_loopback);
2720               if (len <= VLIB_FRAME_SIZE)
2721                 {
2722                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2723                   n_left_from = len;
2724                   vec_reset_length (fragments_to_loopback);
2725                 }
2726               else
2727                 {
2728                   clib_memcpy (from,
2729                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2730                                sizeof (u32) * VLIB_FRAME_SIZE);
2731                   n_left_from = VLIB_FRAME_SIZE;
2732                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2733                 }
2734             }
2735        }
2736
2737       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2738     }
2739
2740   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2741                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2742                                pkts_processed);
2743
2744   nat_send_all_to_node (vm, fragments_to_drop, node,
2745                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2746                         SNAT_IN2OUT_NEXT_DROP);
2747
2748   vec_free (fragments_to_drop);
2749   vec_free (fragments_to_loopback);
2750   return frame->n_vectors;
2751 }
2752
2753 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2754   .function = nat44_in2out_reass_node_fn,
2755   .name = "nat44-in2out-reass",
2756   .vector_size = sizeof (u32),
2757   .format_trace = format_nat44_in2out_reass_trace,
2758   .type = VLIB_NODE_TYPE_INTERNAL,
2759
2760   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2761   .error_strings = snat_in2out_error_strings,
2762
2763   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2764   .next_nodes = {
2765     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2766     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2767     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2768     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2769     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2770   },
2771 };
2772
2773 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2774                               nat44_in2out_reass_node_fn);
2775
2776 /**************************/
2777 /*** deterministic mode ***/
2778 /**************************/
2779 static uword
2780 snat_det_in2out_node_fn (vlib_main_t * vm,
2781                          vlib_node_runtime_t * node,
2782                          vlib_frame_t * frame)
2783 {
2784   u32 n_left_from, * from, * to_next;
2785   snat_in2out_next_t next_index;
2786   u32 pkts_processed = 0;
2787   snat_main_t * sm = &snat_main;
2788   u32 now = (u32) vlib_time_now (vm);
2789   u32 thread_index = vlib_get_thread_index ();
2790
2791   from = vlib_frame_vector_args (frame);
2792   n_left_from = frame->n_vectors;
2793   next_index = node->cached_next_index;
2794
2795   while (n_left_from > 0)
2796     {
2797       u32 n_left_to_next;
2798
2799       vlib_get_next_frame (vm, node, next_index,
2800                            to_next, n_left_to_next);
2801
2802       while (n_left_from >= 4 && n_left_to_next >= 2)
2803         {
2804           u32 bi0, bi1;
2805           vlib_buffer_t * b0, * b1;
2806           u32 next0, next1;
2807           u32 sw_if_index0, sw_if_index1;
2808           ip4_header_t * ip0, * ip1;
2809           ip_csum_t sum0, sum1;
2810           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2811           u16 old_port0, new_port0, lo_port0, i0;
2812           u16 old_port1, new_port1, lo_port1, i1;
2813           udp_header_t * udp0, * udp1;
2814           tcp_header_t * tcp0, * tcp1;
2815           u32 proto0, proto1;
2816           snat_det_out_key_t key0, key1;
2817           snat_det_map_t * dm0, * dm1;
2818           snat_det_session_t * ses0 = 0, * ses1 = 0;
2819           u32 rx_fib_index0, rx_fib_index1;
2820           icmp46_header_t * icmp0, * icmp1;
2821
2822           /* Prefetch next iteration. */
2823           {
2824             vlib_buffer_t * p2, * p3;
2825
2826             p2 = vlib_get_buffer (vm, from[2]);
2827             p3 = vlib_get_buffer (vm, from[3]);
2828
2829             vlib_prefetch_buffer_header (p2, LOAD);
2830             vlib_prefetch_buffer_header (p3, LOAD);
2831
2832             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2833             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2834           }
2835
2836           /* speculatively enqueue b0 and b1 to the current next frame */
2837           to_next[0] = bi0 = from[0];
2838           to_next[1] = bi1 = from[1];
2839           from += 2;
2840           to_next += 2;
2841           n_left_from -= 2;
2842           n_left_to_next -= 2;
2843
2844           b0 = vlib_get_buffer (vm, bi0);
2845           b1 = vlib_get_buffer (vm, bi1);
2846
2847           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2848           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2849
2850           ip0 = vlib_buffer_get_current (b0);
2851           udp0 = ip4_next_header (ip0);
2852           tcp0 = (tcp_header_t *) udp0;
2853
2854           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2855
2856           if (PREDICT_FALSE(ip0->ttl == 1))
2857             {
2858               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2859               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2860                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2861                                            0);
2862               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2863               goto trace0;
2864             }
2865
2866           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2867
2868           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2869             {
2870               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2871               icmp0 = (icmp46_header_t *) udp0;
2872
2873               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2874                                   rx_fib_index0, node, next0, thread_index,
2875                                   &ses0, &dm0);
2876               goto trace0;
2877             }
2878
2879           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2880           if (PREDICT_FALSE(!dm0))
2881             {
2882               clib_warning("no match for internal host %U",
2883                            format_ip4_address, &ip0->src_address);
2884               next0 = SNAT_IN2OUT_NEXT_DROP;
2885               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2886               goto trace0;
2887             }
2888
2889           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2890
2891           key0.ext_host_addr = ip0->dst_address;
2892           key0.ext_host_port = tcp0->dst;
2893
2894           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2895           if (PREDICT_FALSE(!ses0))
2896             {
2897               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2898                 {
2899                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2900                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2901
2902                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2903                     continue;
2904
2905                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2906                   break;
2907                 }
2908               if (PREDICT_FALSE(!ses0))
2909                 {
2910                   /* too many sessions for user, send ICMP error packet */
2911
2912                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2913                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2914                                                ICMP4_destination_unreachable_destination_unreachable_host,
2915                                                0);
2916                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2917                   goto trace0;
2918                 }
2919             }
2920
2921           new_port0 = ses0->out.out_port;
2922
2923           old_addr0.as_u32 = ip0->src_address.as_u32;
2924           ip0->src_address.as_u32 = new_addr0.as_u32;
2925           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2926
2927           sum0 = ip0->checksum;
2928           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2929                                  ip4_header_t,
2930                                  src_address /* changed member */);
2931           ip0->checksum = ip_csum_fold (sum0);
2932
2933           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2934             {
2935               if (tcp0->flags & TCP_FLAG_SYN)
2936                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2937               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2938                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2939               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2940                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2941               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2942                 snat_det_ses_close(dm0, ses0);
2943               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2944                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2945               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2946                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2947
2948               old_port0 = tcp0->src;
2949               tcp0->src = new_port0;
2950
2951               sum0 = tcp0->checksum;
2952               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2953                                      ip4_header_t,
2954                                      dst_address /* changed member */);
2955               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2956                                      ip4_header_t /* cheat */,
2957                                      length /* changed member */);
2958               tcp0->checksum = ip_csum_fold(sum0);
2959             }
2960           else
2961             {
2962               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2963               old_port0 = udp0->src_port;
2964               udp0->src_port = new_port0;
2965               udp0->checksum = 0;
2966             }
2967
2968           switch(ses0->state)
2969             {
2970             case SNAT_SESSION_UDP_ACTIVE:
2971                 ses0->expire = now + sm->udp_timeout;
2972                 break;
2973             case SNAT_SESSION_TCP_SYN_SENT:
2974             case SNAT_SESSION_TCP_FIN_WAIT:
2975             case SNAT_SESSION_TCP_CLOSE_WAIT:
2976             case SNAT_SESSION_TCP_LAST_ACK:
2977                 ses0->expire = now + sm->tcp_transitory_timeout;
2978                 break;
2979             case SNAT_SESSION_TCP_ESTABLISHED:
2980                 ses0->expire = now + sm->tcp_established_timeout;
2981                 break;
2982             }
2983
2984         trace0:
2985           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2986                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2987             {
2988               snat_in2out_trace_t *t =
2989                  vlib_add_trace (vm, node, b0, sizeof (*t));
2990               t->is_slow_path = 0;
2991               t->sw_if_index = sw_if_index0;
2992               t->next_index = next0;
2993               t->session_index = ~0;
2994               if (ses0)
2995                 t->session_index = ses0 - dm0->sessions;
2996             }
2997
2998           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2999
3000           ip1 = vlib_buffer_get_current (b1);
3001           udp1 = ip4_next_header (ip1);
3002           tcp1 = (tcp_header_t *) udp1;
3003
3004           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3005
3006           if (PREDICT_FALSE(ip1->ttl == 1))
3007             {
3008               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3009               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3010                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3011                                            0);
3012               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3013               goto trace1;
3014             }
3015
3016           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3017
3018           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3019             {
3020               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3021               icmp1 = (icmp46_header_t *) udp1;
3022
3023               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3024                                   rx_fib_index1, node, next1, thread_index,
3025                                   &ses1, &dm1);
3026               goto trace1;
3027             }
3028
3029           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3030           if (PREDICT_FALSE(!dm1))
3031             {
3032               clib_warning("no match for internal host %U",
3033                            format_ip4_address, &ip0->src_address);
3034               next1 = SNAT_IN2OUT_NEXT_DROP;
3035               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3036               goto trace1;
3037             }
3038
3039           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3040
3041           key1.ext_host_addr = ip1->dst_address;
3042           key1.ext_host_port = tcp1->dst;
3043
3044           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3045           if (PREDICT_FALSE(!ses1))
3046             {
3047               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3048                 {
3049                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3050                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3051
3052                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3053                     continue;
3054
3055                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3056                   break;
3057                 }
3058               if (PREDICT_FALSE(!ses1))
3059                 {
3060                   /* too many sessions for user, send ICMP error packet */
3061
3062                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3063                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3064                                                ICMP4_destination_unreachable_destination_unreachable_host,
3065                                                0);
3066                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3067                   goto trace1;
3068                 }
3069             }
3070
3071           new_port1 = ses1->out.out_port;
3072
3073           old_addr1.as_u32 = ip1->src_address.as_u32;
3074           ip1->src_address.as_u32 = new_addr1.as_u32;
3075           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3076
3077           sum1 = ip1->checksum;
3078           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3079                                  ip4_header_t,
3080                                  src_address /* changed member */);
3081           ip1->checksum = ip_csum_fold (sum1);
3082
3083           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3084             {
3085               if (tcp1->flags & TCP_FLAG_SYN)
3086                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3087               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3088                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3089               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3090                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3091               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3092                 snat_det_ses_close(dm1, ses1);
3093               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3094                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3095               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3096                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3097
3098               old_port1 = tcp1->src;
3099               tcp1->src = new_port1;
3100
3101               sum1 = tcp1->checksum;
3102               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3103                                      ip4_header_t,
3104                                      dst_address /* changed member */);
3105               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3106                                      ip4_header_t /* cheat */,
3107                                      length /* changed member */);
3108               tcp1->checksum = ip_csum_fold(sum1);
3109             }
3110           else
3111             {
3112               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3113               old_port1 = udp1->src_port;
3114               udp1->src_port = new_port1;
3115               udp1->checksum = 0;
3116             }
3117
3118           switch(ses1->state)
3119             {
3120             case SNAT_SESSION_UDP_ACTIVE:
3121                 ses1->expire = now + sm->udp_timeout;
3122                 break;
3123             case SNAT_SESSION_TCP_SYN_SENT:
3124             case SNAT_SESSION_TCP_FIN_WAIT:
3125             case SNAT_SESSION_TCP_CLOSE_WAIT:
3126             case SNAT_SESSION_TCP_LAST_ACK:
3127                 ses1->expire = now + sm->tcp_transitory_timeout;
3128                 break;
3129             case SNAT_SESSION_TCP_ESTABLISHED:
3130                 ses1->expire = now + sm->tcp_established_timeout;
3131                 break;
3132             }
3133
3134         trace1:
3135           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3136                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3137             {
3138               snat_in2out_trace_t *t =
3139                  vlib_add_trace (vm, node, b1, sizeof (*t));
3140               t->is_slow_path = 0;
3141               t->sw_if_index = sw_if_index1;
3142               t->next_index = next1;
3143               t->session_index = ~0;
3144               if (ses1)
3145                 t->session_index = ses1 - dm1->sessions;
3146             }
3147
3148           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3149
3150           /* verify speculative enqueues, maybe switch current next frame */
3151           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3152                                            to_next, n_left_to_next,
3153                                            bi0, bi1, next0, next1);
3154          }
3155
3156       while (n_left_from > 0 && n_left_to_next > 0)
3157         {
3158           u32 bi0;
3159           vlib_buffer_t * b0;
3160           u32 next0;
3161           u32 sw_if_index0;
3162           ip4_header_t * ip0;
3163           ip_csum_t sum0;
3164           ip4_address_t new_addr0, old_addr0;
3165           u16 old_port0, new_port0, lo_port0, i0;
3166           udp_header_t * udp0;
3167           tcp_header_t * tcp0;
3168           u32 proto0;
3169           snat_det_out_key_t key0;
3170           snat_det_map_t * dm0;
3171           snat_det_session_t * ses0 = 0;
3172           u32 rx_fib_index0;
3173           icmp46_header_t * icmp0;
3174
3175           /* speculatively enqueue b0 to the current next frame */
3176           bi0 = from[0];
3177           to_next[0] = bi0;
3178           from += 1;
3179           to_next += 1;
3180           n_left_from -= 1;
3181           n_left_to_next -= 1;
3182
3183           b0 = vlib_get_buffer (vm, bi0);
3184           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3185
3186           ip0 = vlib_buffer_get_current (b0);
3187           udp0 = ip4_next_header (ip0);
3188           tcp0 = (tcp_header_t *) udp0;
3189
3190           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3191
3192           if (PREDICT_FALSE(ip0->ttl == 1))
3193             {
3194               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3195               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3196                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3197                                            0);
3198               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3199               goto trace00;
3200             }
3201
3202           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3203
3204           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3205             {
3206               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3207               icmp0 = (icmp46_header_t *) udp0;
3208
3209               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3210                                   rx_fib_index0, node, next0, thread_index,
3211                                   &ses0, &dm0);
3212               goto trace00;
3213             }
3214
3215           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3216           if (PREDICT_FALSE(!dm0))
3217             {
3218               clib_warning("no match for internal host %U",
3219                            format_ip4_address, &ip0->src_address);
3220               next0 = SNAT_IN2OUT_NEXT_DROP;
3221               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3222               goto trace00;
3223             }
3224
3225           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3226
3227           key0.ext_host_addr = ip0->dst_address;
3228           key0.ext_host_port = tcp0->dst;
3229
3230           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3231           if (PREDICT_FALSE(!ses0))
3232             {
3233               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3234                 {
3235                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3236                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3237
3238                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3239                     continue;
3240
3241                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3242                   break;
3243                 }
3244               if (PREDICT_FALSE(!ses0))
3245                 {
3246                   /* too many sessions for user, send ICMP error packet */
3247
3248                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3249                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3250                                                ICMP4_destination_unreachable_destination_unreachable_host,
3251                                                0);
3252                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3253                   goto trace00;
3254                 }
3255             }
3256
3257           new_port0 = ses0->out.out_port;
3258
3259           old_addr0.as_u32 = ip0->src_address.as_u32;
3260           ip0->src_address.as_u32 = new_addr0.as_u32;
3261           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3262
3263           sum0 = ip0->checksum;
3264           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3265                                  ip4_header_t,
3266                                  src_address /* changed member */);
3267           ip0->checksum = ip_csum_fold (sum0);
3268
3269           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3270             {
3271               if (tcp0->flags & TCP_FLAG_SYN)
3272                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3273               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3274                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3275               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3276                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3277               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3278                 snat_det_ses_close(dm0, ses0);
3279               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3280                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3281               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3282                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3283
3284               old_port0 = tcp0->src;
3285               tcp0->src = new_port0;
3286
3287               sum0 = tcp0->checksum;
3288               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3289                                      ip4_header_t,
3290                                      dst_address /* changed member */);
3291               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3292                                      ip4_header_t /* cheat */,
3293                                      length /* changed member */);
3294               tcp0->checksum = ip_csum_fold(sum0);
3295             }
3296           else
3297             {
3298               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3299               old_port0 = udp0->src_port;
3300               udp0->src_port = new_port0;
3301               udp0->checksum = 0;
3302             }
3303
3304           switch(ses0->state)
3305             {
3306             case SNAT_SESSION_UDP_ACTIVE:
3307                 ses0->expire = now + sm->udp_timeout;
3308                 break;
3309             case SNAT_SESSION_TCP_SYN_SENT:
3310             case SNAT_SESSION_TCP_FIN_WAIT:
3311             case SNAT_SESSION_TCP_CLOSE_WAIT:
3312             case SNAT_SESSION_TCP_LAST_ACK:
3313                 ses0->expire = now + sm->tcp_transitory_timeout;
3314                 break;
3315             case SNAT_SESSION_TCP_ESTABLISHED:
3316                 ses0->expire = now + sm->tcp_established_timeout;
3317                 break;
3318             }
3319
3320         trace00:
3321           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3322                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3323             {
3324               snat_in2out_trace_t *t =
3325                  vlib_add_trace (vm, node, b0, sizeof (*t));
3326               t->is_slow_path = 0;
3327               t->sw_if_index = sw_if_index0;
3328               t->next_index = next0;
3329               t->session_index = ~0;
3330               if (ses0)
3331                 t->session_index = ses0 - dm0->sessions;
3332             }
3333
3334           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3335
3336           /* verify speculative enqueue, maybe switch current next frame */
3337           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3338                                            to_next, n_left_to_next,
3339                                            bi0, next0);
3340         }
3341
3342       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3343     }
3344
3345   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3346                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3347                                pkts_processed);
3348   return frame->n_vectors;
3349 }
3350
3351 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3352   .function = snat_det_in2out_node_fn,
3353   .name = "nat44-det-in2out",
3354   .vector_size = sizeof (u32),
3355   .format_trace = format_snat_in2out_trace,
3356   .type = VLIB_NODE_TYPE_INTERNAL,
3357
3358   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3359   .error_strings = snat_in2out_error_strings,
3360
3361   .runtime_data_bytes = sizeof (snat_runtime_t),
3362
3363   .n_next_nodes = 3,
3364
3365   /* edit / add dispositions here */
3366   .next_nodes = {
3367     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3368     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3369     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3370   },
3371 };
3372
3373 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3374
3375 /**
3376  * Get address and port values to be used for ICMP packet translation
3377  * and create session if needed
3378  *
3379  * @param[in,out] sm             NAT main
3380  * @param[in,out] node           NAT node runtime
3381  * @param[in] thread_index       thread index
3382  * @param[in,out] b0             buffer containing packet to be translated
3383  * @param[out] p_proto           protocol used for matching
3384  * @param[out] p_value           address and port after NAT translation
3385  * @param[out] p_dont_translate  if packet should not be translated
3386  * @param d                      optional parameter
3387  * @param e                      optional parameter
3388  */
3389 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3390                           u32 thread_index, vlib_buffer_t *b0,
3391                           ip4_header_t *ip0, u8 *p_proto,
3392                           snat_session_key_t *p_value,
3393                           u8 *p_dont_translate, void *d, void *e)
3394 {
3395   icmp46_header_t *icmp0;
3396   u32 sw_if_index0;
3397   u32 rx_fib_index0;
3398   u8 protocol;
3399   snat_det_out_key_t key0;
3400   u8 dont_translate = 0;
3401   u32 next0 = ~0;
3402   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3403   ip4_header_t *inner_ip0;
3404   void *l4_header = 0;
3405   icmp46_header_t *inner_icmp0;
3406   snat_det_map_t * dm0 = 0;
3407   ip4_address_t new_addr0;
3408   u16 lo_port0, i0;
3409   snat_det_session_t * ses0 = 0;
3410   ip4_address_t in_addr;
3411   u16 in_port;
3412
3413   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3414   echo0 = (icmp_echo_header_t *)(icmp0+1);
3415   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3416   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3417
3418   if (!icmp_is_error_message (icmp0))
3419     {
3420       protocol = SNAT_PROTOCOL_ICMP;
3421       in_addr = ip0->src_address;
3422       in_port = echo0->identifier;
3423     }
3424   else
3425     {
3426       inner_ip0 = (ip4_header_t *)(echo0+1);
3427       l4_header = ip4_next_header (inner_ip0);
3428       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3429       in_addr = inner_ip0->dst_address;
3430       switch (protocol)
3431         {
3432         case SNAT_PROTOCOL_ICMP:
3433           inner_icmp0 = (icmp46_header_t*)l4_header;
3434           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3435           in_port = inner_echo0->identifier;
3436           break;
3437         case SNAT_PROTOCOL_UDP:
3438         case SNAT_PROTOCOL_TCP:
3439           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3440           break;
3441         default:
3442           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3443           next0 = SNAT_IN2OUT_NEXT_DROP;
3444           goto out;
3445         }
3446     }
3447
3448   dm0 = snat_det_map_by_user(sm, &in_addr);
3449   if (PREDICT_FALSE(!dm0))
3450     {
3451       clib_warning("no match for internal host %U",
3452                    format_ip4_address, &in_addr);
3453       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3454           IP_PROTOCOL_ICMP, rx_fib_index0)))
3455         {
3456           dont_translate = 1;
3457           goto out;
3458         }
3459       next0 = SNAT_IN2OUT_NEXT_DROP;
3460       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3461       goto out;
3462     }
3463
3464   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3465
3466   key0.ext_host_addr = ip0->dst_address;
3467   key0.ext_host_port = 0;
3468
3469   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3470   if (PREDICT_FALSE(!ses0))
3471     {
3472       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3473           IP_PROTOCOL_ICMP, rx_fib_index0)))
3474         {
3475           dont_translate = 1;
3476           goto out;
3477         }
3478       if (icmp0->type != ICMP4_echo_request)
3479         {
3480           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3481           next0 = SNAT_IN2OUT_NEXT_DROP;
3482           goto out;
3483         }
3484       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3485         {
3486           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3487             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3488
3489           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3490             continue;
3491
3492           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3493           break;
3494         }
3495       if (PREDICT_FALSE(!ses0))
3496         {
3497           next0 = SNAT_IN2OUT_NEXT_DROP;
3498           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3499           goto out;
3500         }
3501     }
3502
3503   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3504                     !icmp_is_error_message (icmp0)))
3505     {
3506       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3507       next0 = SNAT_IN2OUT_NEXT_DROP;
3508       goto out;
3509     }
3510
3511   u32 now = (u32) vlib_time_now (sm->vlib_main);
3512
3513   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3514   ses0->expire = now + sm->icmp_timeout;
3515
3516 out:
3517   *p_proto = protocol;
3518   if (ses0)
3519     {
3520       p_value->addr = new_addr0;
3521       p_value->fib_index = sm->outside_fib_index;
3522       p_value->port = ses0->out.out_port;
3523     }
3524   *p_dont_translate = dont_translate;
3525   if (d)
3526     *(snat_det_session_t**)d = ses0;
3527   if (e)
3528     *(snat_det_map_t**)e = dm0;
3529   return next0;
3530 }
3531
3532 /**********************/
3533 /*** worker handoff ***/
3534 /**********************/
3535 static inline uword
3536 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3537                                       vlib_node_runtime_t * node,
3538                                       vlib_frame_t * frame,
3539                                       u8 is_output)
3540 {
3541   snat_main_t *sm = &snat_main;
3542   vlib_thread_main_t *tm = vlib_get_thread_main ();
3543   u32 n_left_from, *from, *to_next = 0;
3544   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3545   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3546     = 0;
3547   vlib_frame_queue_elt_t *hf = 0;
3548   vlib_frame_t *f = 0;
3549   int i;
3550   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3551   u32 next_worker_index = 0;
3552   u32 current_worker_index = ~0;
3553   u32 thread_index = vlib_get_thread_index ();
3554   u32 fq_index;
3555   u32 to_node_index;
3556
3557   ASSERT (vec_len (sm->workers));
3558
3559   if (is_output)
3560     {
3561       fq_index = sm->fq_in2out_output_index;
3562       to_node_index = sm->in2out_output_node_index;
3563     }
3564   else
3565     {
3566       fq_index = sm->fq_in2out_index;
3567       to_node_index = sm->in2out_node_index;
3568     }
3569
3570   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3571     {
3572       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3573
3574       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3575                                sm->first_worker_index + sm->num_workers - 1,
3576                                (vlib_frame_queue_t *) (~0));
3577     }
3578
3579   from = vlib_frame_vector_args (frame);
3580   n_left_from = frame->n_vectors;
3581
3582   while (n_left_from > 0)
3583     {
3584       u32 bi0;
3585       vlib_buffer_t *b0;
3586       u32 sw_if_index0;
3587       u32 rx_fib_index0;
3588       ip4_header_t * ip0;
3589       u8 do_handoff;
3590
3591       bi0 = from[0];
3592       from += 1;
3593       n_left_from -= 1;
3594
3595       b0 = vlib_get_buffer (vm, bi0);
3596
3597       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3598       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3599
3600       ip0 = vlib_buffer_get_current (b0);
3601
3602       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3603
3604       if (PREDICT_FALSE (next_worker_index != thread_index))
3605         {
3606           do_handoff = 1;
3607
3608           if (next_worker_index != current_worker_index)
3609             {
3610               if (hf)
3611                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3612
3613               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3614                                                       next_worker_index,
3615                                                       handoff_queue_elt_by_worker_index);
3616
3617               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3618               to_next_worker = &hf->buffer_index[hf->n_vectors];
3619               current_worker_index = next_worker_index;
3620             }
3621
3622           /* enqueue to correct worker thread */
3623           to_next_worker[0] = bi0;
3624           to_next_worker++;
3625           n_left_to_next_worker--;
3626
3627           if (n_left_to_next_worker == 0)
3628             {
3629               hf->n_vectors = VLIB_FRAME_SIZE;
3630               vlib_put_frame_queue_elt (hf);
3631               current_worker_index = ~0;
3632               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3633               hf = 0;
3634             }
3635         }
3636       else
3637         {
3638           do_handoff = 0;
3639           /* if this is 1st frame */
3640           if (!f)
3641             {
3642               f = vlib_get_frame_to_node (vm, to_node_index);
3643               to_next = vlib_frame_vector_args (f);
3644             }
3645
3646           to_next[0] = bi0;
3647           to_next += 1;
3648           f->n_vectors++;
3649         }
3650
3651       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3652                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3653         {
3654           snat_in2out_worker_handoff_trace_t *t =
3655             vlib_add_trace (vm, node, b0, sizeof (*t));
3656           t->next_worker_index = next_worker_index;
3657           t->do_handoff = do_handoff;
3658         }
3659     }
3660
3661   if (f)
3662     vlib_put_frame_to_node (vm, to_node_index, f);
3663
3664   if (hf)
3665     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3666
3667   /* Ship frames to the worker nodes */
3668   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3669     {
3670       if (handoff_queue_elt_by_worker_index[i])
3671         {
3672           hf = handoff_queue_elt_by_worker_index[i];
3673           /*
3674            * It works better to let the handoff node
3675            * rate-adapt, always ship the handoff queue element.
3676            */
3677           if (1 || hf->n_vectors == hf->last_n_vectors)
3678             {
3679               vlib_put_frame_queue_elt (hf);
3680               handoff_queue_elt_by_worker_index[i] = 0;
3681             }
3682           else
3683             hf->last_n_vectors = hf->n_vectors;
3684         }
3685       congested_handoff_queue_by_worker_index[i] =
3686         (vlib_frame_queue_t *) (~0);
3687     }
3688   hf = 0;
3689   current_worker_index = ~0;
3690   return frame->n_vectors;
3691 }
3692
3693 static uword
3694 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3695                                vlib_node_runtime_t * node,
3696                                vlib_frame_t * frame)
3697 {
3698   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3699 }
3700
3701 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3702   .function = snat_in2out_worker_handoff_fn,
3703   .name = "nat44-in2out-worker-handoff",
3704   .vector_size = sizeof (u32),
3705   .format_trace = format_snat_in2out_worker_handoff_trace,
3706   .type = VLIB_NODE_TYPE_INTERNAL,
3707
3708   .n_next_nodes = 1,
3709
3710   .next_nodes = {
3711     [0] = "error-drop",
3712   },
3713 };
3714
3715 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3716                               snat_in2out_worker_handoff_fn);
3717
3718 static uword
3719 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3720                                       vlib_node_runtime_t * node,
3721                                       vlib_frame_t * frame)
3722 {
3723   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3724 }
3725
3726 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3727   .function = snat_in2out_output_worker_handoff_fn,
3728   .name = "nat44-in2out-output-worker-handoff",
3729   .vector_size = sizeof (u32),
3730   .format_trace = format_snat_in2out_worker_handoff_trace,
3731   .type = VLIB_NODE_TYPE_INTERNAL,
3732
3733   .n_next_nodes = 1,
3734
3735   .next_nodes = {
3736     [0] = "error-drop",
3737   },
3738 };
3739
3740 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3741                               snat_in2out_output_worker_handoff_fn);
3742
3743 static_always_inline int
3744 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3745 {
3746   snat_address_t * ap;
3747   clib_bihash_kv_8_8_t kv, value;
3748   snat_session_key_t m_key;
3749
3750   vec_foreach (ap, sm->addresses)
3751     {
3752       if (ap->addr.as_u32 == dst_addr->as_u32)
3753         return 1;
3754     }
3755
3756   m_key.addr.as_u32 = dst_addr->as_u32;
3757   m_key.fib_index = sm->outside_fib_index;
3758   m_key.port = 0;
3759   m_key.protocol = 0;
3760   kv.key = m_key.as_u64;
3761   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3762     return 1;
3763
3764   return 0;
3765 }
3766
3767 static uword
3768 snat_hairpin_dst_fn (vlib_main_t * vm,
3769                      vlib_node_runtime_t * node,
3770                      vlib_frame_t * frame)
3771 {
3772   u32 n_left_from, * from, * to_next;
3773   snat_in2out_next_t next_index;
3774   u32 pkts_processed = 0;
3775   snat_main_t * sm = &snat_main;
3776
3777   from = vlib_frame_vector_args (frame);
3778   n_left_from = frame->n_vectors;
3779   next_index = node->cached_next_index;
3780
3781   while (n_left_from > 0)
3782     {
3783       u32 n_left_to_next;
3784
3785       vlib_get_next_frame (vm, node, next_index,
3786                            to_next, n_left_to_next);
3787
3788       while (n_left_from > 0 && n_left_to_next > 0)
3789         {
3790           u32 bi0;
3791           vlib_buffer_t * b0;
3792           u32 next0;
3793           ip4_header_t * ip0;
3794           u32 proto0;
3795
3796           /* speculatively enqueue b0 to the current next frame */
3797           bi0 = from[0];
3798           to_next[0] = bi0;
3799           from += 1;
3800           to_next += 1;
3801           n_left_from -= 1;
3802           n_left_to_next -= 1;
3803
3804           b0 = vlib_get_buffer (vm, bi0);
3805           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3806           ip0 = vlib_buffer_get_current (b0);
3807
3808           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3809
3810           vnet_buffer (b0)->snat.flags = 0;
3811           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3812             {
3813               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3814                 {
3815                   udp_header_t * udp0 = ip4_next_header (ip0);
3816                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3817
3818                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3819                 }
3820               else if (proto0 == SNAT_PROTOCOL_ICMP)
3821                 {
3822                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3823
3824                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3825                 }
3826               else
3827                 {
3828                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3829                 }
3830
3831               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3832             }
3833
3834           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3835
3836           /* verify speculative enqueue, maybe switch current next frame */
3837           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3838                                            to_next, n_left_to_next,
3839                                            bi0, next0);
3840          }
3841
3842       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3843     }
3844
3845   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3846                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3847                                pkts_processed);
3848   return frame->n_vectors;
3849 }
3850
3851 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3852   .function = snat_hairpin_dst_fn,
3853   .name = "nat44-hairpin-dst",
3854   .vector_size = sizeof (u32),
3855   .type = VLIB_NODE_TYPE_INTERNAL,
3856   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3857   .error_strings = snat_in2out_error_strings,
3858   .n_next_nodes = 2,
3859   .next_nodes = {
3860     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3861     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3862   },
3863 };
3864
3865 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3866                               snat_hairpin_dst_fn);
3867
3868 static uword
3869 snat_hairpin_src_fn (vlib_main_t * vm,
3870                      vlib_node_runtime_t * node,
3871                      vlib_frame_t * frame)
3872 {
3873   u32 n_left_from, * from, * to_next;
3874   snat_in2out_next_t next_index;
3875   u32 pkts_processed = 0;
3876   snat_main_t *sm = &snat_main;
3877
3878   from = vlib_frame_vector_args (frame);
3879   n_left_from = frame->n_vectors;
3880   next_index = node->cached_next_index;
3881
3882   while (n_left_from > 0)
3883     {
3884       u32 n_left_to_next;
3885
3886       vlib_get_next_frame (vm, node, next_index,
3887                            to_next, n_left_to_next);
3888
3889       while (n_left_from > 0 && n_left_to_next > 0)
3890         {
3891           u32 bi0;
3892           vlib_buffer_t * b0;
3893           u32 next0;
3894           snat_interface_t *i;
3895           u32 sw_if_index0;
3896
3897           /* speculatively enqueue b0 to the current next frame */
3898           bi0 = from[0];
3899           to_next[0] = bi0;
3900           from += 1;
3901           to_next += 1;
3902           n_left_from -= 1;
3903           n_left_to_next -= 1;
3904
3905           b0 = vlib_get_buffer (vm, bi0);
3906           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3907           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3908
3909           pool_foreach (i, sm->output_feature_interfaces,
3910           ({
3911             /* Only packets from NAT inside interface */
3912             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3913               {
3914                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3915                                     SNAT_FLAG_HAIRPINNING))
3916                   {
3917                     if (PREDICT_TRUE (sm->num_workers > 1))
3918                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3919                     else
3920                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3921                   }
3922                 break;
3923               }
3924           }));
3925
3926           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3927
3928           /* verify speculative enqueue, maybe switch current next frame */
3929           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3930                                            to_next, n_left_to_next,
3931                                            bi0, next0);
3932          }
3933
3934       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3935     }
3936
3937   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3938                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3939                                pkts_processed);
3940   return frame->n_vectors;
3941 }
3942
3943 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3944   .function = snat_hairpin_src_fn,
3945   .name = "nat44-hairpin-src",
3946   .vector_size = sizeof (u32),
3947   .type = VLIB_NODE_TYPE_INTERNAL,
3948   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3949   .error_strings = snat_in2out_error_strings,
3950   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3951   .next_nodes = {
3952      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3953      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3954      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3955      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3956   },
3957 };
3958
3959 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3960                               snat_hairpin_src_fn);
3961
3962 static uword
3963 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3964                                 vlib_node_runtime_t * node,
3965                                 vlib_frame_t * frame)
3966 {
3967   u32 n_left_from, * from, * to_next;
3968   snat_in2out_next_t next_index;
3969   u32 pkts_processed = 0;
3970   snat_main_t * sm = &snat_main;
3971   u32 stats_node_index;
3972
3973   stats_node_index = snat_in2out_fast_node.index;
3974
3975   from = vlib_frame_vector_args (frame);
3976   n_left_from = frame->n_vectors;
3977   next_index = node->cached_next_index;
3978
3979   while (n_left_from > 0)
3980     {
3981       u32 n_left_to_next;
3982
3983       vlib_get_next_frame (vm, node, next_index,
3984                            to_next, n_left_to_next);
3985
3986       while (n_left_from > 0 && n_left_to_next > 0)
3987         {
3988           u32 bi0;
3989           vlib_buffer_t * b0;
3990           u32 next0;
3991           u32 sw_if_index0;
3992           ip4_header_t * ip0;
3993           ip_csum_t sum0;
3994           u32 new_addr0, old_addr0;
3995           u16 old_port0, new_port0;
3996           udp_header_t * udp0;
3997           tcp_header_t * tcp0;
3998           icmp46_header_t * icmp0;
3999           snat_session_key_t key0, sm0;
4000           u32 proto0;
4001           u32 rx_fib_index0;
4002
4003           /* speculatively enqueue b0 to the current next frame */
4004           bi0 = from[0];
4005           to_next[0] = bi0;
4006           from += 1;
4007           to_next += 1;
4008           n_left_from -= 1;
4009           n_left_to_next -= 1;
4010
4011           b0 = vlib_get_buffer (vm, bi0);
4012           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4013
4014           ip0 = vlib_buffer_get_current (b0);
4015           udp0 = ip4_next_header (ip0);
4016           tcp0 = (tcp_header_t *) udp0;
4017           icmp0 = (icmp46_header_t *) udp0;
4018
4019           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4020           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4021
4022           if (PREDICT_FALSE(ip0->ttl == 1))
4023             {
4024               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4025               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4026                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4027                                            0);
4028               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4029               goto trace0;
4030             }
4031
4032           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4033
4034           if (PREDICT_FALSE (proto0 == ~0))
4035               goto trace0;
4036
4037           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4038             {
4039               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4040                                   rx_fib_index0, node, next0, ~0, 0, 0);
4041               goto trace0;
4042             }
4043
4044           key0.addr = ip0->src_address;
4045           key0.protocol = proto0;
4046           key0.port = udp0->src_port;
4047           key0.fib_index = rx_fib_index0;
4048
4049           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4050             {
4051               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4052               next0= SNAT_IN2OUT_NEXT_DROP;
4053               goto trace0;
4054             }
4055
4056           new_addr0 = sm0.addr.as_u32;
4057           new_port0 = sm0.port;
4058           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4059           old_addr0 = ip0->src_address.as_u32;
4060           ip0->src_address.as_u32 = new_addr0;
4061
4062           sum0 = ip0->checksum;
4063           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4064                                  ip4_header_t,
4065                                  src_address /* changed member */);
4066           ip0->checksum = ip_csum_fold (sum0);
4067
4068           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4069             {
4070               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4071                 {
4072                   old_port0 = tcp0->src_port;
4073                   tcp0->src_port = new_port0;
4074
4075                   sum0 = tcp0->checksum;
4076                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4077                                          ip4_header_t,
4078                                          dst_address /* changed member */);
4079                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4080                                          ip4_header_t /* cheat */,
4081                                          length /* changed member */);
4082                   tcp0->checksum = ip_csum_fold(sum0);
4083                 }
4084               else
4085                 {
4086                   old_port0 = udp0->src_port;
4087                   udp0->src_port = new_port0;
4088                   udp0->checksum = 0;
4089                 }
4090             }
4091           else
4092             {
4093               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4094                 {
4095                   sum0 = tcp0->checksum;
4096                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4097                                          ip4_header_t,
4098                                          dst_address /* changed member */);
4099                   tcp0->checksum = ip_csum_fold(sum0);
4100                 }
4101             }
4102
4103           /* Hairpinning */
4104           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4105
4106         trace0:
4107           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4108                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4109             {
4110               snat_in2out_trace_t *t =
4111                  vlib_add_trace (vm, node, b0, sizeof (*t));
4112               t->sw_if_index = sw_if_index0;
4113               t->next_index = next0;
4114             }
4115
4116           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4117
4118           /* verify speculative enqueue, maybe switch current next frame */
4119           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4120                                            to_next, n_left_to_next,
4121                                            bi0, next0);
4122         }
4123
4124       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4125     }
4126
4127   vlib_node_increment_counter (vm, stats_node_index,
4128                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4129                                pkts_processed);
4130   return frame->n_vectors;
4131 }
4132
4133
4134 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4135   .function = snat_in2out_fast_static_map_fn,
4136   .name = "nat44-in2out-fast",
4137   .vector_size = sizeof (u32),
4138   .format_trace = format_snat_in2out_fast_trace,
4139   .type = VLIB_NODE_TYPE_INTERNAL,
4140
4141   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4142   .error_strings = snat_in2out_error_strings,
4143
4144   .runtime_data_bytes = sizeof (snat_runtime_t),
4145
4146   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4147
4148   /* edit / add dispositions here */
4149   .next_nodes = {
4150     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4151     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4152     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4153     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4154     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4155   },
4156 };
4157
4158 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);