src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28 #include <nat/nat_inlines.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38   u32 is_slow_path;
  39 } snat_in2out_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_in2out_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  52   char * tag;
  53
  54   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  55
  56   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  57               t->sw_if_index, t->next_index, t->session_index);
  58
  59   return s;
  60 }
  61
  62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  75 {
  76   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  77   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  78   snat_in2out_worker_handoff_trace_t * t =
  79     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  80   char * m;
  81
  82   m = t->do_handoff ? "next worker" : "same worker";
  83   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  84
  85   return s;
  86 }
  87
  88 typedef struct {
  89   u32 sw_if_index;
  90   u32 next_index;
  91   u8 cached;
  92 } nat44_in2out_reass_trace_t;
  93
  94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  95 {
  96   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  97   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  98   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  99
 100   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 101               t->sw_if_index, t->next_index,
 102               t->cached ? "cached" : "translated");
 103
 104   return s;
 105 }
 106
 107 vlib_node_registration_t snat_in2out_node;
 108 vlib_node_registration_t snat_in2out_slowpath_node;
 109 vlib_node_registration_t snat_in2out_fast_node;
 110 vlib_node_registration_t snat_in2out_worker_handoff_node;
 111 vlib_node_registration_t snat_det_in2out_node;
 112 vlib_node_registration_t snat_in2out_output_node;
 113 vlib_node_registration_t snat_in2out_output_slowpath_node;
 114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 115 vlib_node_registration_t snat_hairpin_dst_node;
 116 vlib_node_registration_t snat_hairpin_src_node;
 117 vlib_node_registration_t nat44_hairpinning_node;
 118 vlib_node_registration_t nat44_in2out_reass_node;
 119
 120
 121 #define foreach_snat_in2out_error                       \
 122 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 123 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 124 _(OUT_OF_PORTS, "Out of ports")                         \
 125 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 126 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 127 _(NO_TRANSLATION, "No translation")                     \
 128 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 129 _(DROP_FRAGMENT, "Drop fragment")                       \
 130 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 131 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 132 _(FQ_CONGESTED, "Handoff frame queue congested")
 133
 134 typedef enum {
 135 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 136   foreach_snat_in2out_error
 137 #undef _
 138   SNAT_IN2OUT_N_ERROR,
 139 } snat_in2out_error_t;
 140
 141 static char * snat_in2out_error_strings[] = {
 142 #define _(sym,string) string,
 143   foreach_snat_in2out_error
 144 #undef _
 145 };
 146
 147 typedef enum {
 148   SNAT_IN2OUT_NEXT_LOOKUP,
 149   SNAT_IN2OUT_NEXT_DROP,
 150   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 151   SNAT_IN2OUT_NEXT_SLOW_PATH,
 152   SNAT_IN2OUT_NEXT_REASS,
 153   SNAT_IN2OUT_N_NEXT,
 154 } snat_in2out_next_t;
 155
 156 typedef enum {
 157   SNAT_HAIRPIN_SRC_NEXT_DROP,
 158   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 159   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 160   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 161   SNAT_HAIRPIN_SRC_N_NEXT,
 162 } snat_hairpin_next_t;
 163
 164 /**
 165  * @brief Check if packet should be translated
 166  *
 167  * Packets aimed at outside interface and external address with active session
 168  * should be translated.
 169  *
 170  * @param sm            NAT main
 171  * @param rt            NAT runtime data
 172  * @param sw_if_index0  index of the inside interface
 173  * @param ip0           IPv4 header
 174  * @param proto0        NAT protocol
 175  * @param rx_fib_index0 RX FIB index
 176  *
 177  * @returns 0 if packet should be translated otherwise 1
 178  */
 179 static inline int
 180 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 181                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 182                          u32 rx_fib_index0)
 183 {
 184   if (sm->out2in_dpo)
 185     return 0;
 186
 187   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 188   fib_prefix_t pfx = {
 189     .fp_proto = FIB_PROTOCOL_IP4,
 190     .fp_len = 32,
 191     .fp_addr = {
 192         .ip4.as_u32 = ip0->dst_address.as_u32,
 193     },
 194   };
 195
 196   /* Don't NAT packet aimed at the intfc address */
 197   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 198                                       ip0->dst_address.as_u32)))
 199     return 1;
 200
 201   fei = fib_table_lookup (rx_fib_index0, &pfx);
 202   if (FIB_NODE_INDEX_INVALID != fei)
 203     {
 204       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 205       if (sw_if_index == ~0)
 206         {
 207           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 208           if (FIB_NODE_INDEX_INVALID != fei)
 209             sw_if_index = fib_entry_get_resolving_interface (fei);
 210         }
 211       snat_interface_t *i;
 212       pool_foreach (i, sm->interfaces,
 213       ({
 214         /* NAT packet aimed at outside interface */
 215         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 216           return 0;
 217       }));
 218     }
 219
 220   return 1;
 221 }
 222
 223 static inline int
 224 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 225                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 226                     u32 rx_fib_index0, u32 thread_index)
 227 {
 228   udp_header_t * udp0 = ip4_next_header (ip0);
 229   snat_session_key_t key0, sm0;
 230   clib_bihash_kv_8_8_t kv0, value0;
 231
 232   key0.addr = ip0->dst_address;
 233   key0.port = udp0->dst_port;
 234   key0.protocol = proto0;
 235   key0.fib_index = sm->outside_fib_index;
 236   kv0.key = key0.as_u64;
 237
 238   /* NAT packet aimed at external address if */
 239   /* has active sessions */
 240   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 241                               &value0))
 242     {
 243       /* or is static mappings */
 244       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 245         return 0;
 246     }
 247   else
 248     return 0;
 249
 250   if (sm->forwarding_enabled)
 251     return 1;
 252
 253   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 254                                  rx_fib_index0);
 255 }
 256
 257 static inline int
 258 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 259                                   u32 proto0, u16 src_port, u16 dst_port,
 260                                   u32 thread_index, u32 sw_if_index)
 261 {
 262   snat_session_key_t key0;
 263   clib_bihash_kv_8_8_t kv0, value0;
 264   snat_interface_t *i;
 265
 266   /* src NAT check */
 267   key0.addr = ip0->src_address;
 268   key0.port = src_port;
 269   key0.protocol = proto0;
 270   key0.fib_index = sm->outside_fib_index;
 271   kv0.key = key0.as_u64;
 272
 273   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 274                                &value0))
 275     return 1;
 276
 277   /* dst NAT check */
 278   key0.addr = ip0->dst_address;
 279   key0.port = dst_port;
 280   key0.protocol = proto0;
 281   key0.fib_index = sm->inside_fib_index;
 282   kv0.key = key0.as_u64;
 283   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 284                                &value0))
 285   {
 286     /* hairpinning */
 287     pool_foreach (i, sm->output_feature_interfaces,
 288     ({
 289       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 290         return 0;
 291     }));
 292     return 1;
 293   }
 294
 295   return 0;
 296 }
 297
 298 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 299                       ip4_header_t * ip0,
 300                       u32 rx_fib_index0,
 301                       snat_session_key_t * key0,
 302                       snat_session_t ** sessionp,
 303                       vlib_node_runtime_t * node,
 304                       u32 next0,
 305                       u32 thread_index)
 306 {
 307   snat_user_t *u;
 308   snat_session_t *s;
 309   clib_bihash_kv_8_8_t kv0;
 310   snat_session_key_t key1;
 311   u32 address_index = ~0;
 312   u32 outside_fib_index;
 313   uword * p;
 314   udp_header_t * udp0 = ip4_next_header (ip0);
 315   u8 is_sm = 0;
 316
 317   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 318     {
 319       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 320       nat_ipfix_logging_max_sessions(sm->max_translations);
 321       nat_log_notice ("maximum sessions exceeded");
 322       return SNAT_IN2OUT_NEXT_DROP;
 323     }
 324
 325   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 326   if (! p)
 327     {
 328       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 329       return SNAT_IN2OUT_NEXT_DROP;
 330     }
 331   outside_fib_index = p[0];
 332
 333   key1.protocol = key0->protocol;
 334
 335   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 336                               thread_index);
 337   if (!u)
 338     {
 339       nat_log_warn ("create NAT user failed");
 340       return SNAT_IN2OUT_NEXT_DROP;
 341     }
 342
 343   /* First try to match static mapping by local address and port */
 344   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 345     {
 346       /* Try to create dynamic translation */
 347       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 348                                                thread_index, &key1,
 349                                                &address_index,
 350                                                sm->port_per_thread,
 351                                                sm->per_thread_data[thread_index].snat_thread_index))
 352         {
 353           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 354           return SNAT_IN2OUT_NEXT_DROP;
 355         }
 356     }
 357   else
 358     is_sm = 1;
 359
 360   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 361   if (!s)
 362     {
 363       nat_log_warn ("create NAT session failed");
 364       return SNAT_IN2OUT_NEXT_DROP;
 365     }
 366
 367   if (is_sm)
 368     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 369   user_session_increment (sm, u, is_sm);
 370   s->outside_address_index = address_index;
 371   s->in2out = *key0;
 372   s->out2in = key1;
 373   s->out2in.protocol = key0->protocol;
 374   s->out2in.fib_index = outside_fib_index;
 375   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 376   s->ext_host_port = udp0->dst_port;
 377   *sessionp = s;
 378
 379   /* Add to translation hashes */
 380   kv0.key = s->in2out.as_u64;
 381   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 382   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 383                                1 /* is_add */))
 384       nat_log_notice ("in2out key add failed");
 385
 386   kv0.key = s->out2in.as_u64;
 387   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 388
 389   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 390                                1 /* is_add */))
 391       nat_log_notice ("out2in key add failed");
 392
 393   /* log NAT event */
 394   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 395                                       s->out2in.addr.as_u32,
 396                                       s->in2out.protocol,
 397                                       s->in2out.port,
 398                                       s->out2in.port,
 399                                       s->in2out.fib_index);
 400   return next0;
 401 }
 402
 403 static_always_inline
 404 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 405                                  snat_session_key_t *p_key0)
 406 {
 407   icmp46_header_t *icmp0;
 408   snat_session_key_t key0;
 409   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 410   ip4_header_t *inner_ip0 = 0;
 411   void *l4_header = 0;
 412   icmp46_header_t *inner_icmp0;
 413
 414   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 415   echo0 = (icmp_echo_header_t *)(icmp0+1);
 416
 417   if (!icmp_is_error_message (icmp0))
 418     {
 419       key0.protocol = SNAT_PROTOCOL_ICMP;
 420       key0.addr = ip0->src_address;
 421       key0.port = echo0->identifier;
 422     }
 423   else
 424     {
 425       inner_ip0 = (ip4_header_t *)(echo0+1);
 426       l4_header = ip4_next_header (inner_ip0);
 427       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 428       key0.addr = inner_ip0->dst_address;
 429       switch (key0.protocol)
 430         {
 431         case SNAT_PROTOCOL_ICMP:
 432           inner_icmp0 = (icmp46_header_t*)l4_header;
 433           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 434           key0.port = inner_echo0->identifier;
 435           break;
 436         case SNAT_PROTOCOL_UDP:
 437         case SNAT_PROTOCOL_TCP:
 438           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 439           break;
 440         default:
 441           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 442         }
 443     }
 444   *p_key0 = key0;
 445   return -1; /* success */
 446 }
 447
 448 static_always_inline int
 449 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 450 {
 451   icmp46_header_t *icmp0;
 452   nat_ed_ses_key_t key0;
 453   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 454   ip4_header_t *inner_ip0 = 0;
 455   void *l4_header = 0;
 456   icmp46_header_t *inner_icmp0;
 457
 458   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 459   echo0 = (icmp_echo_header_t *)(icmp0+1);
 460
 461   if (!icmp_is_error_message (icmp0))
 462     {
 463       key0.proto = IP_PROTOCOL_ICMP;
 464       key0.l_addr = ip0->src_address;
 465       key0.r_addr = ip0->dst_address;
 466       key0.l_port = key0.r_port = echo0->identifier;
 467     }
 468   else
 469     {
 470       inner_ip0 = (ip4_header_t *)(echo0+1);
 471       l4_header = ip4_next_header (inner_ip0);
 472       key0.proto = inner_ip0->protocol;
 473       key0.r_addr = inner_ip0->src_address;
 474       key0.l_addr = inner_ip0->dst_address;
 475       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 476         {
 477         case SNAT_PROTOCOL_ICMP:
 478           inner_icmp0 = (icmp46_header_t*)l4_header;
 479           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 480           key0.r_port = key0.l_port = inner_echo0->identifier;
 481           break;
 482         case SNAT_PROTOCOL_UDP:
 483         case SNAT_PROTOCOL_TCP:
 484           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 485           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 486           break;
 487         default:
 488           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 489         }
 490     }
 491   *p_key0 = key0;
 492   return 0;
 493 }
 494
 495 static inline int
 496 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 497                                       u32 thread_index)
 498 {
 499   nat_ed_ses_key_t key;
 500   clib_bihash_kv_16_8_t kv, value;
 501   udp_header_t *udp;
 502   snat_session_t *s = 0;
 503   f64 now = vlib_time_now (sm->vlib_main);
 504
 505   if (!sm->forwarding_enabled)
 506     return 0;
 507
 508   if (ip->protocol == IP_PROTOCOL_ICMP)
 509     {
 510       if (icmp_get_ed_key (ip, &key))
 511         return 0;
 512     }
 513   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 514     {
 515       udp = ip4_next_header(ip);
 516       key.l_addr = ip->src_address;
 517       key.r_addr = ip->dst_address;
 518       key.proto = ip->protocol;
 519       key.r_port = udp->dst_port;
 520       key.l_port = udp->src_port;
 521     }
 522   else
 523     {
 524       key.l_addr = ip->src_address;
 525       key.r_addr = ip->dst_address;
 526       key.proto = ip->protocol;
 527       key.l_port = key.r_port = 0;
 528     }
 529   key.fib_index = 0;
 530   kv.key[0] = key.as_u64[0];
 531   kv.key[1] = key.as_u64[1];
 532
 533   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 534     {
 535       s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value.value);
 536       if (is_fwd_bypass_session (s))
 537         {
 538           if (ip->protocol == IP_PROTOCOL_TCP)
 539             {
 540               tcp_header_t *tcp = ip4_next_header(ip);
 541               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 542                 return 1;
 543             }
 544           /* Per-user LRU list maintenance */
 545           nat44_session_update_lru (sm, s, thread_index);
 546           /* Accounting */
 547           nat44_session_update_counters (s, now, 0);
 548           return 1;
 549         }
 550       else
 551         return 0;
 552     }
 553
 554   return 0;
 555 }
 556
 557 /**
 558  * Get address and port values to be used for ICMP packet translation
 559  * and create session if needed
 560  *
 561  * @param[in,out] sm             NAT main
 562  * @param[in,out] node           NAT node runtime
 563  * @param[in] thread_index       thread index
 564  * @param[in,out] b0             buffer containing packet to be translated
 565  * @param[out] p_proto           protocol used for matching
 566  * @param[out] p_value           address and port after NAT translation
 567  * @param[out] p_dont_translate  if packet should not be translated
 568  * @param d                      optional parameter
 569  * @param e                      optional parameter
 570  */
 571 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 572                            u32 thread_index, vlib_buffer_t *b0,
 573                            ip4_header_t *ip0, u8 *p_proto,
 574                            snat_session_key_t *p_value,
 575                            u8 *p_dont_translate, void *d, void *e)
 576 {
 577   icmp46_header_t *icmp0;
 578   u32 sw_if_index0;
 579   u32 rx_fib_index0;
 580   snat_session_key_t key0;
 581   snat_session_t *s0 = 0;
 582   u8 dont_translate = 0;
 583   clib_bihash_kv_8_8_t kv0, value0;
 584   u32 next0 = ~0;
 585   int err;
 586
 587   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 588   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 589   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 590
 591   err = icmp_get_key (ip0, &key0);
 592   if (err != -1)
 593     {
 594       b0->error = node->errors[err];
 595       next0 = SNAT_IN2OUT_NEXT_DROP;
 596       goto out;
 597     }
 598   key0.fib_index = rx_fib_index0;
 599
 600   kv0.key = key0.as_u64;
 601
 602   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 603                               &value0))
 604     {
 605       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 606         {
 607           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 608               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 609             {
 610               dont_translate = 1;
 611               goto out;
 612             }
 613         }
 614       else
 615         {
 616           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 617               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 618             {
 619               dont_translate = 1;
 620               goto out;
 621             }
 622         }
 623
 624       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 625         {
 626           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 627           next0 = SNAT_IN2OUT_NEXT_DROP;
 628           goto out;
 629         }
 630
 631       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 632                          &s0, node, next0, thread_index);
 633
 634       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 635         goto out;
 636     }
 637   else
 638     {
 639       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 640                         icmp0->type != ICMP4_echo_reply &&
 641                         !icmp_is_error_message (icmp0)))
 642         {
 643           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 644           next0 = SNAT_IN2OUT_NEXT_DROP;
 645           goto out;
 646         }
 647
 648       if (PREDICT_FALSE (value0.value == ~0ULL))
 649         {
 650           nat_ed_ses_key_t key;
 651           clib_bihash_kv_16_8_t s_kv, s_value;
 652
 653           key.as_u64[0] = 0;
 654           key.as_u64[1] = 0;
 655           if (icmp_get_ed_key (ip0, &key))
 656             {
 657               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 658               next0 = SNAT_IN2OUT_NEXT_DROP;
 659               goto out;
 660             }
 661           key.fib_index = rx_fib_index0;
 662           s_kv.key[0] = key.as_u64[0];
 663           s_kv.key[1] = key.as_u64[1];
 664           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 665             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 666                                     s_value.value);
 667           else
 668            {
 669               next0 = SNAT_IN2OUT_NEXT_DROP;
 670               goto out;
 671            }
 672         }
 673       else
 674         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 675                                 value0.value);
 676     }
 677
 678 out:
 679   *p_proto = key0.protocol;
 680   if (s0)
 681     *p_value = s0->out2in;
 682   *p_dont_translate = dont_translate;
 683   if (d)
 684     *(snat_session_t**)d = s0;
 685   return next0;
 686 }
 687
 688 /**
 689  * Get address and port values to be used for ICMP packet translation
 690  *
 691  * @param[in] sm                 NAT main
 692  * @param[in,out] node           NAT node runtime
 693  * @param[in] thread_index       thread index
 694  * @param[in,out] b0             buffer containing packet to be translated
 695  * @param[out] p_proto           protocol used for matching
 696  * @param[out] p_value           address and port after NAT translation
 697  * @param[out] p_dont_translate  if packet should not be translated
 698  * @param d                      optional parameter
 699  * @param e                      optional parameter
 700  */
 701 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 702                            u32 thread_index, vlib_buffer_t *b0,
 703                            ip4_header_t *ip0, u8 *p_proto,
 704                            snat_session_key_t *p_value,
 705                            u8 *p_dont_translate, void *d, void *e)
 706 {
 707   icmp46_header_t *icmp0;
 708   u32 sw_if_index0;
 709   u32 rx_fib_index0;
 710   snat_session_key_t key0;
 711   snat_session_key_t sm0;
 712   u8 dont_translate = 0;
 713   u8 is_addr_only;
 714   u32 next0 = ~0;
 715   int err;
 716
 717   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 718   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 719   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 720
 721   err = icmp_get_key (ip0, &key0);
 722   if (err != -1)
 723     {
 724       b0->error = node->errors[err];
 725       next0 = SNAT_IN2OUT_NEXT_DROP;
 726       goto out2;
 727     }
 728   key0.fib_index = rx_fib_index0;
 729
 730   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 731     {
 732       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 733           IP_PROTOCOL_ICMP, rx_fib_index0)))
 734         {
 735           dont_translate = 1;
 736           goto out;
 737         }
 738
 739       if (icmp_is_error_message (icmp0))
 740         {
 741           next0 = SNAT_IN2OUT_NEXT_DROP;
 742           goto out;
 743         }
 744
 745       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 746       next0 = SNAT_IN2OUT_NEXT_DROP;
 747       goto out;
 748     }
 749
 750   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 751                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 752                     !icmp_is_error_message (icmp0)))
 753     {
 754       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 755       next0 = SNAT_IN2OUT_NEXT_DROP;
 756       goto out;
 757     }
 758
 759 out:
 760   *p_value = sm0;
 761 out2:
 762   *p_proto = key0.protocol;
 763   *p_dont_translate = dont_translate;
 764   return next0;
 765 }
 766
 767 static inline u32 icmp_in2out (snat_main_t *sm,
 768                                vlib_buffer_t * b0,
 769                                ip4_header_t * ip0,
 770                                icmp46_header_t * icmp0,
 771                                u32 sw_if_index0,
 772                                u32 rx_fib_index0,
 773                                vlib_node_runtime_t * node,
 774                                u32 next0,
 775                                u32 thread_index,
 776                                void *d,
 777                                void *e)
 778 {
 779   snat_session_key_t sm0;
 780   u8 protocol;
 781   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 782   ip4_header_t *inner_ip0;
 783   void *l4_header = 0;
 784   icmp46_header_t *inner_icmp0;
 785   u8 dont_translate;
 786   u32 new_addr0, old_addr0;
 787   u16 old_id0, new_id0;
 788   ip_csum_t sum0;
 789   u16 checksum0;
 790   u32 next0_tmp;
 791
 792   echo0 = (icmp_echo_header_t *)(icmp0+1);
 793
 794   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 795                                        &protocol, &sm0, &dont_translate, d, e);
 796   if (next0_tmp != ~0)
 797     next0 = next0_tmp;
 798   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 799     goto out;
 800
 801   sum0 = ip_incremental_checksum (0, icmp0,
 802                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 803   checksum0 = ~ip_csum_fold (sum0);
 804   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 805     {
 806       next0 = SNAT_IN2OUT_NEXT_DROP;
 807       goto out;
 808     }
 809
 810   old_addr0 = ip0->src_address.as_u32;
 811   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 812   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 813     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 814
 815   sum0 = ip0->checksum;
 816   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 817                          src_address /* changed member */);
 818   ip0->checksum = ip_csum_fold (sum0);
 819
 820   if (icmp0->checksum == 0)
 821     icmp0->checksum = 0xffff;
 822
 823   if (!icmp_is_error_message (icmp0))
 824     {
 825       new_id0 = sm0.port;
 826       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 827         {
 828           old_id0 = echo0->identifier;
 829           new_id0 = sm0.port;
 830           echo0->identifier = new_id0;
 831
 832           sum0 = icmp0->checksum;
 833           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 834                                  identifier);
 835           icmp0->checksum = ip_csum_fold (sum0);
 836         }
 837     }
 838   else
 839     {
 840       inner_ip0 = (ip4_header_t *)(echo0+1);
 841       l4_header = ip4_next_header (inner_ip0);
 842
 843       if (!ip4_header_checksum_is_valid (inner_ip0))
 844         {
 845           next0 = SNAT_IN2OUT_NEXT_DROP;
 846           goto out;
 847         }
 848
 849       old_addr0 = inner_ip0->dst_address.as_u32;
 850       inner_ip0->dst_address = sm0.addr;
 851       new_addr0 = inner_ip0->dst_address.as_u32;
 852
 853       sum0 = icmp0->checksum;
 854       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 855                              dst_address /* changed member */);
 856       icmp0->checksum = ip_csum_fold (sum0);
 857
 858       switch (protocol)
 859         {
 860           case SNAT_PROTOCOL_ICMP:
 861             inner_icmp0 = (icmp46_header_t*)l4_header;
 862             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 863
 864             old_id0 = inner_echo0->identifier;
 865             new_id0 = sm0.port;
 866             inner_echo0->identifier = new_id0;
 867
 868             sum0 = icmp0->checksum;
 869             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 870                                    identifier);
 871             icmp0->checksum = ip_csum_fold (sum0);
 872             break;
 873           case SNAT_PROTOCOL_UDP:
 874           case SNAT_PROTOCOL_TCP:
 875             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 876             new_id0 = sm0.port;
 877             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 878
 879             sum0 = icmp0->checksum;
 880             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 881                                    dst_port);
 882             icmp0->checksum = ip_csum_fold (sum0);
 883             break;
 884           default:
 885             ASSERT(0);
 886         }
 887     }
 888
 889 out:
 890   return next0;
 891 }
 892
 893 /**
 894  * @brief Hairpinning
 895  *
 896  * Hairpinning allows two endpoints on the internal side of the NAT to
 897  * communicate even if they only use each other's external IP addresses
 898  * and ports.
 899  *
 900  * @param sm     NAT main.
 901  * @param b0     Vlib buffer.
 902  * @param ip0    IP header.
 903  * @param udp0   UDP header.
 904  * @param tcp0   TCP header.
 905  * @param proto0 NAT protocol.
 906  */
 907 static inline int
 908 snat_hairpinning (snat_main_t *sm,
 909                   vlib_buffer_t * b0,
 910                   ip4_header_t * ip0,
 911                   udp_header_t * udp0,
 912                   tcp_header_t * tcp0,
 913                   u32 proto0)
 914 {
 915   snat_session_key_t key0, sm0;
 916   snat_session_t * s0;
 917   clib_bihash_kv_8_8_t kv0, value0;
 918   ip_csum_t sum0;
 919   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 920   u16 new_dst_port0, old_dst_port0;
 921
 922   key0.addr = ip0->dst_address;
 923   key0.port = udp0->dst_port;
 924   key0.protocol = proto0;
 925   key0.fib_index = sm->outside_fib_index;
 926   kv0.key = key0.as_u64;
 927
 928   /* Check if destination is static mappings */
 929   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 930     {
 931       new_dst_addr0 = sm0.addr.as_u32;
 932       new_dst_port0 = sm0.port;
 933       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 934     }
 935   /* or active session */
 936   else
 937     {
 938       if (sm->num_workers > 1)
 939         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 940       else
 941         ti = sm->num_workers;
 942
 943       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 944         {
 945           si = value0.value;
 946
 947           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 948           new_dst_addr0 = s0->in2out.addr.as_u32;
 949           new_dst_port0 = s0->in2out.port;
 950           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 951         }
 952     }
 953
 954   /* Destination is behind the same NAT, use internal address and port */
 955   if (new_dst_addr0)
 956     {
 957       old_dst_addr0 = ip0->dst_address.as_u32;
 958       ip0->dst_address.as_u32 = new_dst_addr0;
 959       sum0 = ip0->checksum;
 960       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 961                              ip4_header_t, dst_address);
 962       ip0->checksum = ip_csum_fold (sum0);
 963
 964       old_dst_port0 = tcp0->dst;
 965       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 966         {
 967           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 968             {
 969               tcp0->dst = new_dst_port0;
 970               sum0 = tcp0->checksum;
 971               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 972                                      ip4_header_t, dst_address);
 973               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 974                                      ip4_header_t /* cheat */, length);
 975               tcp0->checksum = ip_csum_fold(sum0);
 976             }
 977           else
 978             {
 979               udp0->dst_port = new_dst_port0;
 980               udp0->checksum = 0;
 981             }
 982         }
 983       else
 984         {
 985           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 986             {
 987               sum0 = tcp0->checksum;
 988               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 989                                      ip4_header_t, dst_address);
 990               tcp0->checksum = ip_csum_fold(sum0);
 991             }
 992         }
 993       return 1;
 994     }
 995   return 0;
 996 }
 997
 998 static inline void
 999 snat_icmp_hairpinning (snat_main_t *sm,
1000                        vlib_buffer_t * b0,
1001                        ip4_header_t * ip0,
1002                        icmp46_header_t * icmp0)
1003 {
1004   snat_session_key_t key0, sm0;
1005   clib_bihash_kv_8_8_t kv0, value0;
1006   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
1007   ip_csum_t sum0;
1008   snat_session_t *s0;
1009
1010   if (!icmp_is_error_message (icmp0))
1011     {
1012       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1013       u16 icmp_id0 = echo0->identifier;
1014       key0.addr = ip0->dst_address;
1015       key0.port = icmp_id0;
1016       key0.protocol = SNAT_PROTOCOL_ICMP;
1017       key0.fib_index = sm->outside_fib_index;
1018       kv0.key = key0.as_u64;
1019
1020       if (sm->num_workers > 1)
1021         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1022       else
1023         ti = sm->num_workers;
1024
1025       /* Check if destination is in active sessions */
1026       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1027                                   &value0))
1028         {
1029           /* or static mappings */
1030           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1031             {
1032               new_dst_addr0 = sm0.addr.as_u32;
1033               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1034             }
1035         }
1036       else
1037         {
1038           si = value0.value;
1039
1040           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1041           new_dst_addr0 = s0->in2out.addr.as_u32;
1042           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1043           echo0->identifier = s0->in2out.port;
1044           sum0 = icmp0->checksum;
1045           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1046                                  icmp_echo_header_t, identifier);
1047           icmp0->checksum = ip_csum_fold (sum0);
1048         }
1049
1050       /* Destination is behind the same NAT, use internal address and port */
1051       if (new_dst_addr0)
1052         {
1053           old_dst_addr0 = ip0->dst_address.as_u32;
1054           ip0->dst_address.as_u32 = new_dst_addr0;
1055           sum0 = ip0->checksum;
1056           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1057                                  ip4_header_t, dst_address);
1058           ip0->checksum = ip_csum_fold (sum0);
1059         }
1060     }
1061
1062 }
1063
1064 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1065                                          vlib_buffer_t * b0,
1066                                          ip4_header_t * ip0,
1067                                          icmp46_header_t * icmp0,
1068                                          u32 sw_if_index0,
1069                                          u32 rx_fib_index0,
1070                                          vlib_node_runtime_t * node,
1071                                          u32 next0,
1072                                          f64 now,
1073                                          u32 thread_index,
1074                                          snat_session_t ** p_s0)
1075 {
1076   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1077                       next0, thread_index, p_s0, 0);
1078   snat_session_t * s0 = *p_s0;
1079   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1080     {
1081       /* Hairpinning */
1082       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1083         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1084       /* Accounting */
1085       nat44_session_update_counters (s0, now,
1086                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
1087       /* Per-user LRU list maintenance */
1088       nat44_session_update_lru (sm, s0, thread_index);
1089     }
1090   return next0;
1091 }
1092 static inline void
1093 snat_hairpinning_unknown_proto (snat_main_t *sm,
1094                                 vlib_buffer_t * b,
1095                                 ip4_header_t * ip)
1096 {
1097   u32 old_addr, new_addr = 0, ti = 0;
1098   clib_bihash_kv_8_8_t kv, value;
1099   clib_bihash_kv_16_8_t s_kv, s_value;
1100   nat_ed_ses_key_t key;
1101   snat_session_key_t m_key;
1102   snat_static_mapping_t *m;
1103   ip_csum_t sum;
1104   snat_session_t *s;
1105
1106   old_addr = ip->dst_address.as_u32;
1107   key.l_addr.as_u32 = ip->dst_address.as_u32;
1108   key.r_addr.as_u32 = ip->src_address.as_u32;
1109   key.fib_index = sm->outside_fib_index;
1110   key.proto = ip->protocol;
1111   key.r_port = 0;
1112   key.l_port = 0;
1113   s_kv.key[0] = key.as_u64[0];
1114   s_kv.key[1] = key.as_u64[1];
1115   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1116     {
1117       m_key.addr = ip->dst_address;
1118       m_key.fib_index = sm->outside_fib_index;
1119       m_key.port = 0;
1120       m_key.protocol = 0;
1121       kv.key = m_key.as_u64;
1122       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1123         return;
1124
1125       m = pool_elt_at_index (sm->static_mappings, value.value);
1126       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1127         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1128       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1129     }
1130   else
1131     {
1132       if (sm->num_workers > 1)
1133         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1134       else
1135         ti = sm->num_workers;
1136
1137       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1138       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1139         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1140       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1141     }
1142   sum = ip->checksum;
1143   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1144   ip->checksum = ip_csum_fold (sum);
1145 }
1146
1147 static snat_session_t *
1148 snat_in2out_unknown_proto (snat_main_t *sm,
1149                            vlib_buffer_t * b,
1150                            ip4_header_t * ip,
1151                            u32 rx_fib_index,
1152                            u32 thread_index,
1153                            f64 now,
1154                            vlib_main_t * vm,
1155                            vlib_node_runtime_t * node)
1156 {
1157   clib_bihash_kv_8_8_t kv, value;
1158   clib_bihash_kv_16_8_t s_kv, s_value;
1159   snat_static_mapping_t *m;
1160   snat_session_key_t m_key;
1161   u32 old_addr, new_addr = 0;
1162   ip_csum_t sum;
1163   snat_user_t *u;
1164   dlist_elt_t *head, *elt;
1165   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1166   u32 elt_index, head_index, ses_index;
1167   snat_session_t * s;
1168   nat_ed_ses_key_t key;
1169   u32 address_index = ~0;
1170   int i;
1171   u8 is_sm = 0;
1172
1173   old_addr = ip->src_address.as_u32;
1174
1175   key.l_addr = ip->src_address;
1176   key.r_addr = ip->dst_address;
1177   key.fib_index = rx_fib_index;
1178   key.proto = ip->protocol;
1179   key.l_port = 0;
1180   key.r_port = 0;
1181   s_kv.key[0] = key.as_u64[0];
1182   s_kv.key[1] = key.as_u64[1];
1183
1184   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1185     {
1186       s = pool_elt_at_index (tsm->sessions, s_value.value);
1187       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1188     }
1189   else
1190     {
1191       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1192         {
1193           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1194           nat_ipfix_logging_max_sessions(sm->max_translations);
1195           nat_log_notice ("maximum sessions exceeded");
1196           return 0;
1197         }
1198
1199       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1200                                   thread_index);
1201       if (!u)
1202         {
1203           nat_log_warn ("create NAT user failed");
1204           return 0;
1205         }
1206
1207       m_key.addr = ip->src_address;
1208       m_key.port = 0;
1209       m_key.protocol = 0;
1210       m_key.fib_index = rx_fib_index;
1211       kv.key = m_key.as_u64;
1212
1213       /* Try to find static mapping first */
1214       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1215         {
1216           m = pool_elt_at_index (sm->static_mappings, value.value);
1217           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1218           is_sm = 1;
1219           goto create_ses;
1220         }
1221       /* Fallback to 3-tuple key */
1222       else
1223         {
1224           /* Choose same out address as for TCP/UDP session to same destination */
1225           head_index = u->sessions_per_user_list_head_index;
1226           head = pool_elt_at_index (tsm->list_pool, head_index);
1227           elt_index = head->next;
1228           if (PREDICT_FALSE (elt_index == ~0))
1229             ses_index = ~0;
1230           else
1231             {
1232               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1233               ses_index = elt->value;
1234             }
1235
1236           while (ses_index != ~0)
1237             {
1238               s =  pool_elt_at_index (tsm->sessions, ses_index);
1239               elt_index = elt->next;
1240               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1241               ses_index = elt->value;
1242
1243               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1244                 {
1245                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1246                   address_index = s->outside_address_index;
1247
1248                   key.fib_index = sm->outside_fib_index;
1249                   key.l_addr.as_u32 = new_addr;
1250                   s_kv.key[0] = key.as_u64[0];
1251                   s_kv.key[1] = key.as_u64[1];
1252                   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1253                     break;
1254
1255                   goto create_ses;
1256                 }
1257             }
1258           key.fib_index = sm->outside_fib_index;
1259           for (i = 0; i < vec_len (sm->addresses); i++)
1260             {
1261               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1262               s_kv.key[0] = key.as_u64[0];
1263               s_kv.key[1] = key.as_u64[1];
1264               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1265                 {
1266                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1267                   address_index = i;
1268                   goto create_ses;
1269                 }
1270             }
1271           return 0;
1272         }
1273
1274 create_ses:
1275       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1276       if (!s)
1277         {
1278           nat_log_warn ("create NAT session failed");
1279           return 0;
1280         }
1281
1282       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1283       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1284       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1285       s->outside_address_index = address_index;
1286       s->out2in.addr.as_u32 = new_addr;
1287       s->out2in.fib_index = sm->outside_fib_index;
1288       s->in2out.addr.as_u32 = old_addr;
1289       s->in2out.fib_index = rx_fib_index;
1290       s->in2out.port = s->out2in.port = ip->protocol;
1291       if (is_sm)
1292         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1293       user_session_increment (sm, u, is_sm);
1294
1295       /* Add to lookup tables */
1296       key.l_addr.as_u32 = old_addr;
1297       key.r_addr = ip->dst_address;
1298       key.proto = ip->protocol;
1299       key.fib_index = rx_fib_index;
1300       s_kv.key[0] = key.as_u64[0];
1301       s_kv.key[1] = key.as_u64[1];
1302       s_kv.value = s - tsm->sessions;
1303       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1304         nat_log_notice ("in2out key add failed");
1305
1306       key.l_addr.as_u32 = new_addr;
1307       key.fib_index = sm->outside_fib_index;
1308       s_kv.key[0] = key.as_u64[0];
1309       s_kv.key[1] = key.as_u64[1];
1310       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1311         nat_log_notice ("out2in key add failed");
1312   }
1313
1314   /* Update IP checksum */
1315   sum = ip->checksum;
1316   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1317   ip->checksum = ip_csum_fold (sum);
1318
1319   /* Accounting */
1320   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
1321   /* Per-user LRU list maintenance */
1322   nat44_session_update_lru (sm, s, thread_index);
1323
1324   /* Hairpinning */
1325   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1326     snat_hairpinning_unknown_proto(sm, b, ip);
1327
1328   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1329     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1330
1331   return s;
1332 }
1333
1334 static snat_session_t *
1335 snat_in2out_lb (snat_main_t *sm,
1336                 vlib_buffer_t * b,
1337                 ip4_header_t * ip,
1338                 u32 rx_fib_index,
1339                 u32 thread_index,
1340                 f64 now,
1341                 vlib_main_t * vm,
1342                 vlib_node_runtime_t * node)
1343 {
1344   nat_ed_ses_key_t key;
1345   clib_bihash_kv_16_8_t s_kv, s_value;
1346   udp_header_t *udp = ip4_next_header (ip);
1347   tcp_header_t *tcp = (tcp_header_t *) udp;
1348   snat_session_t *s = 0;
1349   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1350   u32 old_addr, new_addr;
1351   u16 new_port, old_port;
1352   ip_csum_t sum;
1353   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1354   snat_session_key_t e_key, l_key;
1355   snat_user_t *u;
1356   u8 lb;
1357
1358   old_addr = ip->src_address.as_u32;
1359
1360   key.l_addr = ip->src_address;
1361   key.r_addr = ip->dst_address;
1362   key.fib_index = rx_fib_index;
1363   key.proto = ip->protocol;
1364   key.r_port = udp->dst_port;
1365   key.l_port = udp->src_port;
1366   s_kv.key[0] = key.as_u64[0];
1367   s_kv.key[1] = key.as_u64[1];
1368
1369   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1370     {
1371       s = pool_elt_at_index (tsm->sessions, s_value.value);
1372       if (is_fwd_bypass_session (s))
1373         {
1374           if (ip->protocol == IP_PROTOCOL_TCP)
1375             {
1376               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
1377                 return 0;
1378             }
1379           /* Per-user LRU list maintenance */
1380           nat44_session_update_lru (sm, s, thread_index);
1381           return 0;
1382         }
1383     }
1384   else
1385     {
1386       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1387         {
1388           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1389           nat_ipfix_logging_max_sessions(sm->max_translations);
1390           nat_log_notice ("maximum sessions exceeded");
1391           return 0;
1392         }
1393
1394       l_key.addr = ip->src_address;
1395       l_key.port = udp->src_port;
1396       l_key.protocol = proto;
1397       l_key.fib_index = rx_fib_index;
1398       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1399         return 0;
1400
1401       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1402                                   thread_index);
1403       if (!u)
1404         {
1405           nat_log_warn ("create NAT user failed");
1406           return 0;
1407         }
1408
1409       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1410       if (!s)
1411         {
1412           nat_log_warn ("create NAT session failed");
1413           return 0;
1414         }
1415
1416       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1417       s->ext_host_port = udp->dst_port;
1418       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1419       if (lb)
1420         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1421       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1422       s->outside_address_index = ~0;
1423       s->in2out = l_key;
1424       s->out2in = e_key;
1425       s->out2in.protocol = l_key.protocol;
1426       user_session_increment (sm, u, 1 /* static */);
1427
1428       /* Add to lookup tables */
1429       s_kv.value = s - tsm->sessions;
1430       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1431         nat_log_notice ("in2out-ed key add failed");
1432
1433       key.l_addr = e_key.addr;
1434       key.fib_index = e_key.fib_index;
1435       key.l_port = e_key.port;
1436       s_kv.key[0] = key.as_u64[0];
1437       s_kv.key[1] = key.as_u64[1];
1438       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1439         nat_log_notice ("out2in-ed key add failed");
1440     }
1441
1442   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1443
1444   /* Update IP checksum */
1445   sum = ip->checksum;
1446   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1447   if (is_twice_nat_session (s))
1448     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1449                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1450   ip->checksum = ip_csum_fold (sum);
1451
1452   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1453     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1454
1455   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1456     {
1457       old_port = tcp->src_port;
1458       tcp->src_port = s->out2in.port;
1459       new_port = tcp->src_port;
1460
1461       sum = tcp->checksum;
1462       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1463       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1464       if (is_twice_nat_session (s))
1465         {
1466           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1467                                 s->ext_host_addr.as_u32, ip4_header_t,
1468                                 dst_address);
1469           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1470                                 ip4_header_t, length);
1471           tcp->dst_port = s->ext_host_port;
1472           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1473         }
1474       tcp->checksum = ip_csum_fold(sum);
1475       if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
1476         return s;
1477     }
1478   else
1479     {
1480       udp->src_port = s->out2in.port;
1481       if (is_twice_nat_session (s))
1482         {
1483           udp->dst_port = s->ext_host_port;
1484           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1485         }
1486       udp->checksum = 0;
1487     }
1488
1489   /* Accounting */
1490   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
1491   /* Per-user LRU list maintenance */
1492   nat44_session_update_lru (sm, s, thread_index);
1493
1494   return s;
1495 }
1496
1497 static inline uword
1498 snat_in2out_node_fn_inline (vlib_main_t * vm,
1499                             vlib_node_runtime_t * node,
1500                             vlib_frame_t * frame, int is_slow_path,
1501                             int is_output_feature)
1502 {
1503   u32 n_left_from, * from, * to_next;
1504   snat_in2out_next_t next_index;
1505   u32 pkts_processed = 0;
1506   snat_main_t * sm = &snat_main;
1507   f64 now = vlib_time_now (vm);
1508   u32 stats_node_index;
1509   u32 thread_index = vlib_get_thread_index ();
1510
1511   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1512     snat_in2out_node.index;
1513
1514   from = vlib_frame_vector_args (frame);
1515   n_left_from = frame->n_vectors;
1516   next_index = node->cached_next_index;
1517
1518   while (n_left_from > 0)
1519     {
1520       u32 n_left_to_next;
1521
1522       vlib_get_next_frame (vm, node, next_index,
1523                            to_next, n_left_to_next);
1524
1525       while (n_left_from >= 4 && n_left_to_next >= 2)
1526         {
1527           u32 bi0, bi1;
1528           vlib_buffer_t * b0, * b1;
1529           u32 next0, next1;
1530           u32 sw_if_index0, sw_if_index1;
1531           ip4_header_t * ip0, * ip1;
1532           ip_csum_t sum0, sum1;
1533           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1534           u16 old_port0, new_port0, old_port1, new_port1;
1535           udp_header_t * udp0, * udp1;
1536           tcp_header_t * tcp0, * tcp1;
1537           icmp46_header_t * icmp0, * icmp1;
1538           snat_session_key_t key0, key1;
1539           u32 rx_fib_index0, rx_fib_index1;
1540           u32 proto0, proto1;
1541           snat_session_t * s0 = 0, * s1 = 0;
1542           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1543           u32 iph_offset0 = 0, iph_offset1 = 0;
1544
1545           /* Prefetch next iteration. */
1546           {
1547             vlib_buffer_t * p2, * p3;
1548
1549             p2 = vlib_get_buffer (vm, from[2]);
1550             p3 = vlib_get_buffer (vm, from[3]);
1551
1552             vlib_prefetch_buffer_header (p2, LOAD);
1553             vlib_prefetch_buffer_header (p3, LOAD);
1554
1555             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1556             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1557           }
1558
1559           /* speculatively enqueue b0 and b1 to the current next frame */
1560           to_next[0] = bi0 = from[0];
1561           to_next[1] = bi1 = from[1];
1562           from += 2;
1563           to_next += 2;
1564           n_left_from -= 2;
1565           n_left_to_next -= 2;
1566
1567           b0 = vlib_get_buffer (vm, bi0);
1568           b1 = vlib_get_buffer (vm, bi1);
1569
1570           if (is_output_feature)
1571             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1572
1573           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1574                  iph_offset0);
1575
1576           udp0 = ip4_next_header (ip0);
1577           tcp0 = (tcp_header_t *) udp0;
1578           icmp0 = (icmp46_header_t *) udp0;
1579
1580           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1581           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1582                                    sw_if_index0);
1583
1584           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1585
1586           if (PREDICT_FALSE(ip0->ttl == 1))
1587             {
1588               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1589               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1590                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1591                                            0);
1592               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1593               goto trace00;
1594             }
1595
1596           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1597
1598           /* Next configured feature, probably ip4-lookup */
1599           if (is_slow_path)
1600             {
1601               if (PREDICT_FALSE (proto0 == ~0))
1602                 {
1603                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1604                                                   thread_index, now, vm, node);
1605                   if (!s0)
1606                     next0 = SNAT_IN2OUT_NEXT_DROP;
1607                   goto trace00;
1608                 }
1609
1610               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1611                 {
1612                   next0 = icmp_in2out_slow_path
1613                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1614                      node, next0, now, thread_index, &s0);
1615                   goto trace00;
1616                 }
1617             }
1618           else
1619             {
1620               if (is_output_feature)
1621                 {
1622                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
1623                     goto trace00;
1624                 }
1625
1626               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1627                 {
1628                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1629                   goto trace00;
1630                 }
1631
1632               if (ip4_is_fragment (ip0))
1633                 {
1634                   next0 = SNAT_IN2OUT_NEXT_REASS;
1635                   goto trace00;
1636                 }
1637             }
1638
1639           key0.addr = ip0->src_address;
1640           key0.port = udp0->src_port;
1641           key0.protocol = proto0;
1642           key0.fib_index = rx_fib_index0;
1643
1644           kv0.key = key0.as_u64;
1645
1646           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1647               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1648             {
1649               if (is_slow_path)
1650                 {
1651                   if (is_output_feature)
1652                     {
1653                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1654                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1655                         goto trace00;
1656                     }
1657                   else
1658                     {
1659                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1660                           ip0, proto0, rx_fib_index0, thread_index)))
1661                         goto trace00;
1662                     }
1663
1664                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1665                                      &s0, node, next0, thread_index);
1666                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1667                     goto trace00;
1668                 }
1669               else
1670                 {
1671                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1672                   goto trace00;
1673                 }
1674             }
1675           else
1676             {
1677               if (PREDICT_FALSE (value0.value == ~0ULL))
1678                 {
1679                   if (is_slow_path)
1680                     {
1681                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1682                                           thread_index, now, vm, node);
1683                       if (!s0 && !sm->forwarding_enabled)
1684                         next0 = SNAT_IN2OUT_NEXT_DROP;
1685                       goto trace00;
1686                     }
1687                   else
1688                     {
1689                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1690                       goto trace00;
1691                     }
1692                 }
1693               else
1694                 {
1695                   s0 = pool_elt_at_index (
1696                     sm->per_thread_data[thread_index].sessions,
1697                     value0.value);
1698                 }
1699             }
1700
1701           b0->flags |= VNET_BUFFER_F_IS_NATED;
1702
1703           old_addr0 = ip0->src_address.as_u32;
1704           ip0->src_address = s0->out2in.addr;
1705           new_addr0 = ip0->src_address.as_u32;
1706           if (!is_output_feature)
1707             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1708
1709           sum0 = ip0->checksum;
1710           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1711                                  ip4_header_t,
1712                                  src_address /* changed member */);
1713           ip0->checksum = ip_csum_fold (sum0);
1714
1715           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1716             {
1717               old_port0 = tcp0->src_port;
1718               tcp0->src_port = s0->out2in.port;
1719               new_port0 = tcp0->src_port;
1720
1721               sum0 = tcp0->checksum;
1722               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1723                                      ip4_header_t,
1724                                      dst_address /* changed member */);
1725               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1726                                      ip4_header_t /* cheat */,
1727                                      length /* changed member */);
1728               tcp0->checksum = ip_csum_fold(sum0);
1729             }
1730           else
1731             {
1732               old_port0 = udp0->src_port;
1733               udp0->src_port = s0->out2in.port;
1734               udp0->checksum = 0;
1735             }
1736
1737           /* Accounting */
1738           nat44_session_update_counters (s0, now,
1739                                          vlib_buffer_length_in_chain (vm, b0));
1740           /* Per-user LRU list maintenance */
1741           nat44_session_update_lru (sm, s0, thread_index);
1742         trace00:
1743
1744           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1745                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1746             {
1747               snat_in2out_trace_t *t =
1748                  vlib_add_trace (vm, node, b0, sizeof (*t));
1749               t->is_slow_path = is_slow_path;
1750               t->sw_if_index = sw_if_index0;
1751               t->next_index = next0;
1752                   t->session_index = ~0;
1753               if (s0)
1754                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1755             }
1756
1757           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1758
1759           if (is_output_feature)
1760             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1761
1762           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1763                  iph_offset1);
1764
1765           udp1 = ip4_next_header (ip1);
1766           tcp1 = (tcp_header_t *) udp1;
1767           icmp1 = (icmp46_header_t *) udp1;
1768
1769           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1770           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1771                                    sw_if_index1);
1772
1773           if (PREDICT_FALSE(ip1->ttl == 1))
1774             {
1775               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1776               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1777                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1778                                            0);
1779               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1780               goto trace01;
1781             }
1782
1783           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1784
1785           /* Next configured feature, probably ip4-lookup */
1786           if (is_slow_path)
1787             {
1788               if (PREDICT_FALSE (proto1 == ~0))
1789                 {
1790                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1791                                                   thread_index, now, vm, node);
1792                   if (!s1)
1793                     next1 = SNAT_IN2OUT_NEXT_DROP;
1794                   goto trace01;
1795                 }
1796
1797               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1798                 {
1799                   next1 = icmp_in2out_slow_path
1800                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1801                      next1, now, thread_index, &s1);
1802                   goto trace01;
1803                 }
1804             }
1805           else
1806             {
1807               if (is_output_feature)
1808                 {
1809                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1, thread_index)))
1810                     goto trace01;
1811                 }
1812
1813               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1814                 {
1815                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1816                   goto trace01;
1817                 }
1818
1819               if (ip4_is_fragment (ip1))
1820                 {
1821                   next1 = SNAT_IN2OUT_NEXT_REASS;
1822                   goto trace01;
1823                 }
1824             }
1825
1826           key1.addr = ip1->src_address;
1827           key1.port = udp1->src_port;
1828           key1.protocol = proto1;
1829           key1.fib_index = rx_fib_index1;
1830
1831           kv1.key = key1.as_u64;
1832
1833             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1834                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1835             {
1836               if (is_slow_path)
1837                 {
1838                   if (is_output_feature)
1839                     {
1840                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1841                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1842                         goto trace01;
1843                     }
1844                   else
1845                     {
1846                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1847                           ip1, proto1, rx_fib_index1, thread_index)))
1848                         goto trace01;
1849                     }
1850
1851                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1852                                      &s1, node, next1, thread_index);
1853                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1854                     goto trace01;
1855                 }
1856               else
1857                 {
1858                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1859                   goto trace01;
1860                 }
1861             }
1862           else
1863             {
1864               if (PREDICT_FALSE (value1.value == ~0ULL))
1865                 {
1866                   if (is_slow_path)
1867                     {
1868                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1869                                           thread_index, now, vm, node);
1870                       if (!s1 && !sm->forwarding_enabled)
1871                         next1 = SNAT_IN2OUT_NEXT_DROP;
1872                       goto trace01;
1873                     }
1874                   else
1875                     {
1876                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1877                       goto trace01;
1878                     }
1879                 }
1880               else
1881                 {
1882                   s1 = pool_elt_at_index (
1883                     sm->per_thread_data[thread_index].sessions,
1884                     value1.value);
1885                 }
1886             }
1887
1888           b1->flags |= VNET_BUFFER_F_IS_NATED;
1889
1890           old_addr1 = ip1->src_address.as_u32;
1891           ip1->src_address = s1->out2in.addr;
1892           new_addr1 = ip1->src_address.as_u32;
1893           if (!is_output_feature)
1894             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1895
1896           sum1 = ip1->checksum;
1897           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1898                                  ip4_header_t,
1899                                  src_address /* changed member */);
1900           ip1->checksum = ip_csum_fold (sum1);
1901
1902           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1903             {
1904               old_port1 = tcp1->src_port;
1905               tcp1->src_port = s1->out2in.port;
1906               new_port1 = tcp1->src_port;
1907
1908               sum1 = tcp1->checksum;
1909               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1910                                      ip4_header_t,
1911                                      dst_address /* changed member */);
1912               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1913                                      ip4_header_t /* cheat */,
1914                                      length /* changed member */);
1915               tcp1->checksum = ip_csum_fold(sum1);
1916             }
1917           else
1918             {
1919               old_port1 = udp1->src_port;
1920               udp1->src_port = s1->out2in.port;
1921               udp1->checksum = 0;
1922             }
1923
1924           /* Accounting */
1925           nat44_session_update_counters (s1, now,
1926                                          vlib_buffer_length_in_chain (vm, b1));
1927           /* Per-user LRU list maintenance */
1928           nat44_session_update_lru (sm, s1, thread_index);
1929         trace01:
1930
1931           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1932                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1933             {
1934               snat_in2out_trace_t *t =
1935                  vlib_add_trace (vm, node, b1, sizeof (*t));
1936               t->sw_if_index = sw_if_index1;
1937               t->next_index = next1;
1938               t->session_index = ~0;
1939               if (s1)
1940                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1941             }
1942
1943           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1944
1945           /* verify speculative enqueues, maybe switch current next frame */
1946           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1947                                            to_next, n_left_to_next,
1948                                            bi0, bi1, next0, next1);
1949         }
1950
1951       while (n_left_from > 0 && n_left_to_next > 0)
1952         {
1953           u32 bi0;
1954           vlib_buffer_t * b0;
1955           u32 next0;
1956           u32 sw_if_index0;
1957           ip4_header_t * ip0;
1958           ip_csum_t sum0;
1959           u32 new_addr0, old_addr0;
1960           u16 old_port0, new_port0;
1961           udp_header_t * udp0;
1962           tcp_header_t * tcp0;
1963           icmp46_header_t * icmp0;
1964           snat_session_key_t key0;
1965           u32 rx_fib_index0;
1966           u32 proto0;
1967           snat_session_t * s0 = 0;
1968           clib_bihash_kv_8_8_t kv0, value0;
1969           u32 iph_offset0 = 0;
1970
1971           /* speculatively enqueue b0 to the current next frame */
1972           bi0 = from[0];
1973           to_next[0] = bi0;
1974           from += 1;
1975           to_next += 1;
1976           n_left_from -= 1;
1977           n_left_to_next -= 1;
1978
1979           b0 = vlib_get_buffer (vm, bi0);
1980           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1981
1982           if (is_output_feature)
1983             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1984
1985           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1986                  iph_offset0);
1987
1988           udp0 = ip4_next_header (ip0);
1989           tcp0 = (tcp_header_t *) udp0;
1990           icmp0 = (icmp46_header_t *) udp0;
1991
1992           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1993           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1994                                    sw_if_index0);
1995
1996           if (PREDICT_FALSE(ip0->ttl == 1))
1997             {
1998               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1999               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2000                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2001                                            0);
2002               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2003               goto trace0;
2004             }
2005
2006           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2007
2008           /* Next configured feature, probably ip4-lookup */
2009           if (is_slow_path)
2010             {
2011               if (PREDICT_FALSE (proto0 == ~0))
2012                 {
2013                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2014                                                   thread_index, now, vm, node);
2015                   if (!s0)
2016                     next0 = SNAT_IN2OUT_NEXT_DROP;
2017                   goto trace0;
2018                 }
2019
2020               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2021                 {
2022                   next0 = icmp_in2out_slow_path
2023                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2024                      next0, now, thread_index, &s0);
2025                   goto trace0;
2026                 }
2027             }
2028           else
2029             {
2030                if (is_output_feature)
2031                 {
2032                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
2033                     goto trace0;
2034                 }
2035
2036               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2037                 {
2038                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2039                   goto trace0;
2040                 }
2041
2042               if (ip4_is_fragment (ip0))
2043                 {
2044                   next0 = SNAT_IN2OUT_NEXT_REASS;
2045                   goto trace0;
2046                 }
2047             }
2048
2049           key0.addr = ip0->src_address;
2050           key0.port = udp0->src_port;
2051           key0.protocol = proto0;
2052           key0.fib_index = rx_fib_index0;
2053
2054           kv0.key = key0.as_u64;
2055
2056           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2057                                       &kv0, &value0))
2058             {
2059               if (is_slow_path)
2060                 {
2061                   if (is_output_feature)
2062                     {
2063                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2064                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2065                         goto trace0;
2066                     }
2067                   else
2068                     {
2069                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2070                           ip0, proto0, rx_fib_index0, thread_index)))
2071                         goto trace0;
2072                     }
2073
2074                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2075                                      &s0, node, next0, thread_index);
2076
2077                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2078                     goto trace0;
2079                 }
2080               else
2081                 {
2082                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2083                   goto trace0;
2084                 }
2085             }
2086           else
2087             {
2088               if (PREDICT_FALSE (value0.value == ~0ULL))
2089                 {
2090                   if (is_slow_path)
2091                     {
2092                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2093                                           thread_index, now, vm, node);
2094                       if (!s0 && !sm->forwarding_enabled)
2095                         next0 = SNAT_IN2OUT_NEXT_DROP;
2096                       goto trace0;
2097                     }
2098                   else
2099                     {
2100                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2101                       goto trace0;
2102                     }
2103                 }
2104               else
2105                 {
2106                   s0 = pool_elt_at_index (
2107                     sm->per_thread_data[thread_index].sessions,
2108                     value0.value);
2109                 }
2110             }
2111
2112           b0->flags |= VNET_BUFFER_F_IS_NATED;
2113
2114           old_addr0 = ip0->src_address.as_u32;
2115           ip0->src_address = s0->out2in.addr;
2116           new_addr0 = ip0->src_address.as_u32;
2117           if (!is_output_feature)
2118             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2119
2120           sum0 = ip0->checksum;
2121           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2122                                  ip4_header_t,
2123                                  src_address /* changed member */);
2124           ip0->checksum = ip_csum_fold (sum0);
2125
2126           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2127             {
2128               old_port0 = tcp0->src_port;
2129               tcp0->src_port = s0->out2in.port;
2130               new_port0 = tcp0->src_port;
2131
2132               sum0 = tcp0->checksum;
2133               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2134                                      ip4_header_t,
2135                                      dst_address /* changed member */);
2136               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2137                                      ip4_header_t /* cheat */,
2138                                      length /* changed member */);
2139               tcp0->checksum = ip_csum_fold(sum0);
2140             }
2141           else
2142             {
2143               old_port0 = udp0->src_port;
2144               udp0->src_port = s0->out2in.port;
2145               udp0->checksum = 0;
2146             }
2147
2148           /* Accounting */
2149           nat44_session_update_counters (s0, now,
2150                                          vlib_buffer_length_in_chain (vm, b0));
2151           /* Per-user LRU list maintenance */
2152           nat44_session_update_lru (sm, s0, thread_index);
2153
2154         trace0:
2155           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2156                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2157             {
2158               snat_in2out_trace_t *t =
2159                  vlib_add_trace (vm, node, b0, sizeof (*t));
2160               t->is_slow_path = is_slow_path;
2161               t->sw_if_index = sw_if_index0;
2162               t->next_index = next0;
2163                   t->session_index = ~0;
2164               if (s0)
2165                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2166             }
2167
2168           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2169
2170           /* verify speculative enqueue, maybe switch current next frame */
2171           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2172                                            to_next, n_left_to_next,
2173                                            bi0, next0);
2174         }
2175
2176       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2177     }
2178
2179   vlib_node_increment_counter (vm, stats_node_index,
2180                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2181                                pkts_processed);
2182   return frame->n_vectors;
2183 }
2184
2185 static uword
2186 snat_in2out_fast_path_fn (vlib_main_t * vm,
2187                           vlib_node_runtime_t * node,
2188                           vlib_frame_t * frame)
2189 {
2190   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2191 }
2192
2193 VLIB_REGISTER_NODE (snat_in2out_node) = {
2194   .function = snat_in2out_fast_path_fn,
2195   .name = "nat44-in2out",
2196   .vector_size = sizeof (u32),
2197   .format_trace = format_snat_in2out_trace,
2198   .type = VLIB_NODE_TYPE_INTERNAL,
2199
2200   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2201   .error_strings = snat_in2out_error_strings,
2202
2203   .runtime_data_bytes = sizeof (snat_runtime_t),
2204
2205   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2206
2207   /* edit / add dispositions here */
2208   .next_nodes = {
2209     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2210     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2211     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2212     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2213     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2214   },
2215 };
2216
2217 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2218
2219 static uword
2220 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2221                                  vlib_node_runtime_t * node,
2222                                  vlib_frame_t * frame)
2223 {
2224   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2225 }
2226
2227 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2228   .function = snat_in2out_output_fast_path_fn,
2229   .name = "nat44-in2out-output",
2230   .vector_size = sizeof (u32),
2231   .format_trace = format_snat_in2out_trace,
2232   .type = VLIB_NODE_TYPE_INTERNAL,
2233
2234   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2235   .error_strings = snat_in2out_error_strings,
2236
2237   .runtime_data_bytes = sizeof (snat_runtime_t),
2238
2239   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2240
2241   /* edit / add dispositions here */
2242   .next_nodes = {
2243     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2244     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2245     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2246     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2247     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2248   },
2249 };
2250
2251 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2252                               snat_in2out_output_fast_path_fn);
2253
2254 static uword
2255 snat_in2out_slow_path_fn (vlib_main_t * vm,
2256                           vlib_node_runtime_t * node,
2257                           vlib_frame_t * frame)
2258 {
2259   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2260 }
2261
2262 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2263   .function = snat_in2out_slow_path_fn,
2264   .name = "nat44-in2out-slowpath",
2265   .vector_size = sizeof (u32),
2266   .format_trace = format_snat_in2out_trace,
2267   .type = VLIB_NODE_TYPE_INTERNAL,
2268
2269   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2270   .error_strings = snat_in2out_error_strings,
2271
2272   .runtime_data_bytes = sizeof (snat_runtime_t),
2273
2274   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2275
2276   /* edit / add dispositions here */
2277   .next_nodes = {
2278     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2279     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2280     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2281     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2282     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2283   },
2284 };
2285
2286 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2287                               snat_in2out_slow_path_fn);
2288
2289 static uword
2290 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2291                                  vlib_node_runtime_t * node,
2292                                  vlib_frame_t * frame)
2293 {
2294   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2295 }
2296
2297 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2298   .function = snat_in2out_output_slow_path_fn,
2299   .name = "nat44-in2out-output-slowpath",
2300   .vector_size = sizeof (u32),
2301   .format_trace = format_snat_in2out_trace,
2302   .type = VLIB_NODE_TYPE_INTERNAL,
2303
2304   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2305   .error_strings = snat_in2out_error_strings,
2306
2307   .runtime_data_bytes = sizeof (snat_runtime_t),
2308
2309   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2310
2311   /* edit / add dispositions here */
2312   .next_nodes = {
2313     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2314     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2315     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2316     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2317     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2318   },
2319 };
2320
2321 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2322                               snat_in2out_output_slow_path_fn);
2323
2324 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2325
2326 static uword
2327 nat44_hairpinning_fn (vlib_main_t * vm,
2328                       vlib_node_runtime_t * node,
2329                       vlib_frame_t * frame)
2330 {
2331   u32 n_left_from, * from, * to_next;
2332   snat_in2out_next_t next_index;
2333   u32 pkts_processed = 0;
2334   snat_main_t * sm = &snat_main;
2335   vnet_feature_main_t *fm = &feature_main;
2336   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2337   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2338
2339   from = vlib_frame_vector_args (frame);
2340   n_left_from = frame->n_vectors;
2341   next_index = node->cached_next_index;
2342
2343   while (n_left_from > 0)
2344     {
2345       u32 n_left_to_next;
2346
2347       vlib_get_next_frame (vm, node, next_index,
2348                            to_next, n_left_to_next);
2349
2350       while (n_left_from > 0 && n_left_to_next > 0)
2351         {
2352           u32 bi0;
2353           vlib_buffer_t * b0;
2354           u32 next0;
2355           ip4_header_t * ip0;
2356           u32 proto0;
2357           udp_header_t * udp0;
2358           tcp_header_t * tcp0;
2359
2360           /* speculatively enqueue b0 to the current next frame */
2361           bi0 = from[0];
2362           to_next[0] = bi0;
2363           from += 1;
2364           to_next += 1;
2365           n_left_from -= 1;
2366           n_left_to_next -= 1;
2367
2368           b0 = vlib_get_buffer (vm, bi0);
2369           ip0 = vlib_buffer_get_current (b0);
2370           udp0 = ip4_next_header (ip0);
2371           tcp0 = (tcp_header_t *) udp0;
2372
2373           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2374
2375           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2376                                 &next0, 0);
2377
2378           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2379             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2380
2381           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2382
2383           /* verify speculative enqueue, maybe switch current next frame */
2384           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2385                                            to_next, n_left_to_next,
2386                                            bi0, next0);
2387          }
2388
2389       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2390     }
2391
2392   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2393                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2394                                pkts_processed);
2395   return frame->n_vectors;
2396 }
2397
2398 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2399   .function = nat44_hairpinning_fn,
2400   .name = "nat44-hairpinning",
2401   .vector_size = sizeof (u32),
2402   .type = VLIB_NODE_TYPE_INTERNAL,
2403   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2404   .error_strings = snat_in2out_error_strings,
2405   .n_next_nodes = 2,
2406   .next_nodes = {
2407     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2408     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2409   },
2410 };
2411
2412 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2413                               nat44_hairpinning_fn);
2414
2415 static inline void
2416 nat44_reass_hairpinning (snat_main_t *sm,
2417                          vlib_buffer_t * b0,
2418                          ip4_header_t * ip0,
2419                          u16 sport,
2420                          u16 dport,
2421                          u32 proto0)
2422 {
2423   snat_session_key_t key0, sm0;
2424   snat_session_t * s0;
2425   clib_bihash_kv_8_8_t kv0, value0;
2426   ip_csum_t sum0;
2427   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2428   u16 new_dst_port0, old_dst_port0;
2429   udp_header_t * udp0;
2430   tcp_header_t * tcp0;
2431
2432   key0.addr = ip0->dst_address;
2433   key0.port = dport;
2434   key0.protocol = proto0;
2435   key0.fib_index = sm->outside_fib_index;
2436   kv0.key = key0.as_u64;
2437
2438   udp0 = ip4_next_header (ip0);
2439
2440   /* Check if destination is static mappings */
2441   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2442     {
2443       new_dst_addr0 = sm0.addr.as_u32;
2444       new_dst_port0 = sm0.port;
2445       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2446     }
2447   /* or active sessions */
2448   else
2449     {
2450       if (sm->num_workers > 1)
2451         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2452       else
2453         ti = sm->num_workers;
2454
2455       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2456         {
2457           si = value0.value;
2458           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2459           new_dst_addr0 = s0->in2out.addr.as_u32;
2460           new_dst_port0 = s0->in2out.port;
2461           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2462         }
2463     }
2464
2465   /* Destination is behind the same NAT, use internal address and port */
2466   if (new_dst_addr0)
2467     {
2468       old_dst_addr0 = ip0->dst_address.as_u32;
2469       ip0->dst_address.as_u32 = new_dst_addr0;
2470       sum0 = ip0->checksum;
2471       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2472                              ip4_header_t, dst_address);
2473       ip0->checksum = ip_csum_fold (sum0);
2474
2475       old_dst_port0 = dport;
2476       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2477                        ip4_is_first_fragment (ip0)))
2478         {
2479           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2480             {
2481               tcp0 = ip4_next_header (ip0);
2482               tcp0->dst = new_dst_port0;
2483               sum0 = tcp0->checksum;
2484               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2485                                      ip4_header_t, dst_address);
2486               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2487                                      ip4_header_t /* cheat */, length);
2488               tcp0->checksum = ip_csum_fold(sum0);
2489             }
2490           else
2491             {
2492               udp0->dst_port = new_dst_port0;
2493               udp0->checksum = 0;
2494             }
2495         }
2496       else
2497         {
2498           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2499             {
2500               tcp0 = ip4_next_header (ip0);
2501               sum0 = tcp0->checksum;
2502               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2503                                      ip4_header_t, dst_address);
2504               tcp0->checksum = ip_csum_fold(sum0);
2505             }
2506         }
2507     }
2508 }
2509
2510 static uword
2511 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2512                             vlib_node_runtime_t * node,
2513                             vlib_frame_t * frame)
2514 {
2515   u32 n_left_from, *from, *to_next;
2516   snat_in2out_next_t next_index;
2517   u32 pkts_processed = 0;
2518   snat_main_t *sm = &snat_main;
2519   f64 now = vlib_time_now (vm);
2520   u32 thread_index = vlib_get_thread_index ();
2521   snat_main_per_thread_data_t *per_thread_data =
2522     &sm->per_thread_data[thread_index];
2523   u32 *fragments_to_drop = 0;
2524   u32 *fragments_to_loopback = 0;
2525
2526   from = vlib_frame_vector_args (frame);
2527   n_left_from = frame->n_vectors;
2528   next_index = node->cached_next_index;
2529
2530   while (n_left_from > 0)
2531     {
2532       u32 n_left_to_next;
2533
2534       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2535
2536       while (n_left_from > 0 && n_left_to_next > 0)
2537        {
2538           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2539           vlib_buffer_t *b0;
2540           u32 next0;
2541           u8 cached0 = 0;
2542           ip4_header_t *ip0;
2543           nat_reass_ip4_t *reass0;
2544           udp_header_t * udp0;
2545           tcp_header_t * tcp0;
2546           snat_session_key_t key0;
2547           clib_bihash_kv_8_8_t kv0, value0;
2548           snat_session_t * s0 = 0;
2549           u16 old_port0, new_port0;
2550           ip_csum_t sum0;
2551
2552           /* speculatively enqueue b0 to the current next frame */
2553           bi0 = from[0];
2554           to_next[0] = bi0;
2555           from += 1;
2556           to_next += 1;
2557           n_left_from -= 1;
2558           n_left_to_next -= 1;
2559
2560           b0 = vlib_get_buffer (vm, bi0);
2561           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2562
2563           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2564           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2565                                                                sw_if_index0);
2566
2567           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2568             {
2569               next0 = SNAT_IN2OUT_NEXT_DROP;
2570               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2571               goto trace0;
2572             }
2573
2574           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2575           udp0 = ip4_next_header (ip0);
2576           tcp0 = (tcp_header_t *) udp0;
2577           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2578
2579           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2580                                                  ip0->dst_address,
2581                                                  ip0->fragment_id,
2582                                                  ip0->protocol,
2583                                                  1,
2584                                                  &fragments_to_drop);
2585
2586           if (PREDICT_FALSE (!reass0))
2587             {
2588               next0 = SNAT_IN2OUT_NEXT_DROP;
2589               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2590               nat_log_notice ("maximum reassemblies exceeded");
2591               goto trace0;
2592             }
2593
2594           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2595             {
2596               key0.addr = ip0->src_address;
2597               key0.port = udp0->src_port;
2598               key0.protocol = proto0;
2599               key0.fib_index = rx_fib_index0;
2600               kv0.key = key0.as_u64;
2601
2602               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2603                 {
2604                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2605                       ip0, proto0, rx_fib_index0, thread_index)))
2606                     goto trace0;
2607
2608                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2609                                      &s0, node, next0, thread_index);
2610
2611                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2612                     goto trace0;
2613
2614                   reass0->sess_index = s0 - per_thread_data->sessions;
2615                 }
2616               else
2617                 {
2618                   s0 = pool_elt_at_index (per_thread_data->sessions,
2619                                           value0.value);
2620                   reass0->sess_index = value0.value;
2621                 }
2622               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2623             }
2624           else
2625             {
2626               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2627                 {
2628                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2629                     {
2630                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2631                       nat_log_notice ("maximum fragments per reassembly exceeded");
2632                       next0 = SNAT_IN2OUT_NEXT_DROP;
2633                       goto trace0;
2634                     }
2635                   cached0 = 1;
2636                   goto trace0;
2637                 }
2638               s0 = pool_elt_at_index (per_thread_data->sessions,
2639                                       reass0->sess_index);
2640             }
2641
2642           old_addr0 = ip0->src_address.as_u32;
2643           ip0->src_address = s0->out2in.addr;
2644           new_addr0 = ip0->src_address.as_u32;
2645           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2646
2647           sum0 = ip0->checksum;
2648           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2649                                  ip4_header_t,
2650                                  src_address /* changed member */);
2651           ip0->checksum = ip_csum_fold (sum0);
2652
2653           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2654             {
2655               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2656                 {
2657                   old_port0 = tcp0->src_port;
2658                   tcp0->src_port = s0->out2in.port;
2659                   new_port0 = tcp0->src_port;
2660
2661                   sum0 = tcp0->checksum;
2662                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2663                                          ip4_header_t,
2664                                          dst_address /* changed member */);
2665                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2666                                          ip4_header_t /* cheat */,
2667                                          length /* changed member */);
2668                   tcp0->checksum = ip_csum_fold(sum0);
2669                 }
2670               else
2671                 {
2672                   old_port0 = udp0->src_port;
2673                   udp0->src_port = s0->out2in.port;
2674                   udp0->checksum = 0;
2675                 }
2676             }
2677
2678           /* Hairpinning */
2679           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2680                                    s0->ext_host_port, proto0);
2681
2682           /* Accounting */
2683           nat44_session_update_counters (s0, now,
2684                                          vlib_buffer_length_in_chain (vm, b0));
2685           /* Per-user LRU list maintenance */
2686           nat44_session_update_lru (sm, s0, thread_index);
2687
2688         trace0:
2689           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2690                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2691             {
2692               nat44_in2out_reass_trace_t *t =
2693                  vlib_add_trace (vm, node, b0, sizeof (*t));
2694               t->cached = cached0;
2695               t->sw_if_index = sw_if_index0;
2696               t->next_index = next0;
2697             }
2698
2699           if (cached0)
2700             {
2701               n_left_to_next++;
2702               to_next--;
2703             }
2704           else
2705             {
2706               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2707
2708               /* verify speculative enqueue, maybe switch current next frame */
2709               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2710                                                to_next, n_left_to_next,
2711                                                bi0, next0);
2712             }
2713
2714           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2715             {
2716               from = vlib_frame_vector_args (frame);
2717               u32 len = vec_len (fragments_to_loopback);
2718               if (len <= VLIB_FRAME_SIZE)
2719                 {
2720                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2721                   n_left_from = len;
2722                   vec_reset_length (fragments_to_loopback);
2723                 }
2724               else
2725                 {
2726                   clib_memcpy (from,
2727                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2728                                sizeof (u32) * VLIB_FRAME_SIZE);
2729                   n_left_from = VLIB_FRAME_SIZE;
2730                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2731                 }
2732             }
2733        }
2734
2735       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2736     }
2737
2738   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2739                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2740                                pkts_processed);
2741
2742   nat_send_all_to_node (vm, fragments_to_drop, node,
2743                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2744                         SNAT_IN2OUT_NEXT_DROP);
2745
2746   vec_free (fragments_to_drop);
2747   vec_free (fragments_to_loopback);
2748   return frame->n_vectors;
2749 }
2750
2751 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2752   .function = nat44_in2out_reass_node_fn,
2753   .name = "nat44-in2out-reass",
2754   .vector_size = sizeof (u32),
2755   .format_trace = format_nat44_in2out_reass_trace,
2756   .type = VLIB_NODE_TYPE_INTERNAL,
2757
2758   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2759   .error_strings = snat_in2out_error_strings,
2760
2761   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2762   .next_nodes = {
2763     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2764     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2765     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2766     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2767     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2768   },
2769 };
2770
2771 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2772                               nat44_in2out_reass_node_fn);
2773
2774 /**************************/
2775 /*** deterministic mode ***/
2776 /**************************/
2777 static uword
2778 snat_det_in2out_node_fn (vlib_main_t * vm,
2779                          vlib_node_runtime_t * node,
2780                          vlib_frame_t * frame)
2781 {
2782   u32 n_left_from, * from, * to_next;
2783   snat_in2out_next_t next_index;
2784   u32 pkts_processed = 0;
2785   snat_main_t * sm = &snat_main;
2786   u32 now = (u32) vlib_time_now (vm);
2787   u32 thread_index = vlib_get_thread_index ();
2788
2789   from = vlib_frame_vector_args (frame);
2790   n_left_from = frame->n_vectors;
2791   next_index = node->cached_next_index;
2792
2793   while (n_left_from > 0)
2794     {
2795       u32 n_left_to_next;
2796
2797       vlib_get_next_frame (vm, node, next_index,
2798                            to_next, n_left_to_next);
2799
2800       while (n_left_from >= 4 && n_left_to_next >= 2)
2801         {
2802           u32 bi0, bi1;
2803           vlib_buffer_t * b0, * b1;
2804           u32 next0, next1;
2805           u32 sw_if_index0, sw_if_index1;
2806           ip4_header_t * ip0, * ip1;
2807           ip_csum_t sum0, sum1;
2808           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2809           u16 old_port0, new_port0, lo_port0, i0;
2810           u16 old_port1, new_port1, lo_port1, i1;
2811           udp_header_t * udp0, * udp1;
2812           tcp_header_t * tcp0, * tcp1;
2813           u32 proto0, proto1;
2814           snat_det_out_key_t key0, key1;
2815           snat_det_map_t * dm0, * dm1;
2816           snat_det_session_t * ses0 = 0, * ses1 = 0;
2817           u32 rx_fib_index0, rx_fib_index1;
2818           icmp46_header_t * icmp0, * icmp1;
2819
2820           /* Prefetch next iteration. */
2821           {
2822             vlib_buffer_t * p2, * p3;
2823
2824             p2 = vlib_get_buffer (vm, from[2]);
2825             p3 = vlib_get_buffer (vm, from[3]);
2826
2827             vlib_prefetch_buffer_header (p2, LOAD);
2828             vlib_prefetch_buffer_header (p3, LOAD);
2829
2830             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2831             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2832           }
2833
2834           /* speculatively enqueue b0 and b1 to the current next frame */
2835           to_next[0] = bi0 = from[0];
2836           to_next[1] = bi1 = from[1];
2837           from += 2;
2838           to_next += 2;
2839           n_left_from -= 2;
2840           n_left_to_next -= 2;
2841
2842           b0 = vlib_get_buffer (vm, bi0);
2843           b1 = vlib_get_buffer (vm, bi1);
2844
2845           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2846           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2847
2848           ip0 = vlib_buffer_get_current (b0);
2849           udp0 = ip4_next_header (ip0);
2850           tcp0 = (tcp_header_t *) udp0;
2851
2852           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2853
2854           if (PREDICT_FALSE(ip0->ttl == 1))
2855             {
2856               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2857               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2858                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2859                                            0);
2860               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2861               goto trace0;
2862             }
2863
2864           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2865
2866           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2867             {
2868               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2869               icmp0 = (icmp46_header_t *) udp0;
2870
2871               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2872                                   rx_fib_index0, node, next0, thread_index,
2873                                   &ses0, &dm0);
2874               goto trace0;
2875             }
2876
2877           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2878           if (PREDICT_FALSE(!dm0))
2879             {
2880               nat_log_info ("no match for internal host %U",
2881                             format_ip4_address, &ip0->src_address);
2882               next0 = SNAT_IN2OUT_NEXT_DROP;
2883               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2884               goto trace0;
2885             }
2886
2887           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2888
2889           key0.ext_host_addr = ip0->dst_address;
2890           key0.ext_host_port = tcp0->dst;
2891
2892           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2893           if (PREDICT_FALSE(!ses0))
2894             {
2895               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2896                 {
2897                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2898                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2899
2900                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2901                     continue;
2902
2903                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2904                   break;
2905                 }
2906               if (PREDICT_FALSE(!ses0))
2907                 {
2908                   /* too many sessions for user, send ICMP error packet */
2909
2910                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2911                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2912                                                ICMP4_destination_unreachable_destination_unreachable_host,
2913                                                0);
2914                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2915                   goto trace0;
2916                 }
2917             }
2918
2919           new_port0 = ses0->out.out_port;
2920
2921           old_addr0.as_u32 = ip0->src_address.as_u32;
2922           ip0->src_address.as_u32 = new_addr0.as_u32;
2923           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2924
2925           sum0 = ip0->checksum;
2926           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2927                                  ip4_header_t,
2928                                  src_address /* changed member */);
2929           ip0->checksum = ip_csum_fold (sum0);
2930
2931           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2932             {
2933               if (tcp0->flags & TCP_FLAG_SYN)
2934                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2935               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2936                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2937               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2938                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2939               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2940                 snat_det_ses_close(dm0, ses0);
2941               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2942                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2943               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2944                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2945
2946               old_port0 = tcp0->src;
2947               tcp0->src = new_port0;
2948
2949               sum0 = tcp0->checksum;
2950               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2951                                      ip4_header_t,
2952                                      dst_address /* changed member */);
2953               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2954                                      ip4_header_t /* cheat */,
2955                                      length /* changed member */);
2956               tcp0->checksum = ip_csum_fold(sum0);
2957             }
2958           else
2959             {
2960               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2961               old_port0 = udp0->src_port;
2962               udp0->src_port = new_port0;
2963               udp0->checksum = 0;
2964             }
2965
2966           switch(ses0->state)
2967             {
2968             case SNAT_SESSION_UDP_ACTIVE:
2969                 ses0->expire = now + sm->udp_timeout;
2970                 break;
2971             case SNAT_SESSION_TCP_SYN_SENT:
2972             case SNAT_SESSION_TCP_FIN_WAIT:
2973             case SNAT_SESSION_TCP_CLOSE_WAIT:
2974             case SNAT_SESSION_TCP_LAST_ACK:
2975                 ses0->expire = now + sm->tcp_transitory_timeout;
2976                 break;
2977             case SNAT_SESSION_TCP_ESTABLISHED:
2978                 ses0->expire = now + sm->tcp_established_timeout;
2979                 break;
2980             }
2981
2982         trace0:
2983           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2984                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2985             {
2986               snat_in2out_trace_t *t =
2987                  vlib_add_trace (vm, node, b0, sizeof (*t));
2988               t->is_slow_path = 0;
2989               t->sw_if_index = sw_if_index0;
2990               t->next_index = next0;
2991               t->session_index = ~0;
2992               if (ses0)
2993                 t->session_index = ses0 - dm0->sessions;
2994             }
2995
2996           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2997
2998           ip1 = vlib_buffer_get_current (b1);
2999           udp1 = ip4_next_header (ip1);
3000           tcp1 = (tcp_header_t *) udp1;
3001
3002           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3003
3004           if (PREDICT_FALSE(ip1->ttl == 1))
3005             {
3006               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3007               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3008                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3009                                            0);
3010               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3011               goto trace1;
3012             }
3013
3014           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3015
3016           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3017             {
3018               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3019               icmp1 = (icmp46_header_t *) udp1;
3020
3021               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3022                                   rx_fib_index1, node, next1, thread_index,
3023                                   &ses1, &dm1);
3024               goto trace1;
3025             }
3026
3027           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3028           if (PREDICT_FALSE(!dm1))
3029             {
3030               nat_log_info ("no match for internal host %U",
3031                             format_ip4_address, &ip0->src_address);
3032               next1 = SNAT_IN2OUT_NEXT_DROP;
3033               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3034               goto trace1;
3035             }
3036
3037           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3038
3039           key1.ext_host_addr = ip1->dst_address;
3040           key1.ext_host_port = tcp1->dst;
3041
3042           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3043           if (PREDICT_FALSE(!ses1))
3044             {
3045               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3046                 {
3047                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3048                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3049
3050                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3051                     continue;
3052
3053                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3054                   break;
3055                 }
3056               if (PREDICT_FALSE(!ses1))
3057                 {
3058                   /* too many sessions for user, send ICMP error packet */
3059
3060                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3061                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3062                                                ICMP4_destination_unreachable_destination_unreachable_host,
3063                                                0);
3064                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3065                   goto trace1;
3066                 }
3067             }
3068
3069           new_port1 = ses1->out.out_port;
3070
3071           old_addr1.as_u32 = ip1->src_address.as_u32;
3072           ip1->src_address.as_u32 = new_addr1.as_u32;
3073           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3074
3075           sum1 = ip1->checksum;
3076           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3077                                  ip4_header_t,
3078                                  src_address /* changed member */);
3079           ip1->checksum = ip_csum_fold (sum1);
3080
3081           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3082             {
3083               if (tcp1->flags & TCP_FLAG_SYN)
3084                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3085               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3086                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3087               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3088                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3089               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3090                 snat_det_ses_close(dm1, ses1);
3091               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3092                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3093               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3094                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3095
3096               old_port1 = tcp1->src;
3097               tcp1->src = new_port1;
3098
3099               sum1 = tcp1->checksum;
3100               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3101                                      ip4_header_t,
3102                                      dst_address /* changed member */);
3103               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3104                                      ip4_header_t /* cheat */,
3105                                      length /* changed member */);
3106               tcp1->checksum = ip_csum_fold(sum1);
3107             }
3108           else
3109             {
3110               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3111               old_port1 = udp1->src_port;
3112               udp1->src_port = new_port1;
3113               udp1->checksum = 0;
3114             }
3115
3116           switch(ses1->state)
3117             {
3118             case SNAT_SESSION_UDP_ACTIVE:
3119                 ses1->expire = now + sm->udp_timeout;
3120                 break;
3121             case SNAT_SESSION_TCP_SYN_SENT:
3122             case SNAT_SESSION_TCP_FIN_WAIT:
3123             case SNAT_SESSION_TCP_CLOSE_WAIT:
3124             case SNAT_SESSION_TCP_LAST_ACK:
3125                 ses1->expire = now + sm->tcp_transitory_timeout;
3126                 break;
3127             case SNAT_SESSION_TCP_ESTABLISHED:
3128                 ses1->expire = now + sm->tcp_established_timeout;
3129                 break;
3130             }
3131
3132         trace1:
3133           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3134                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3135             {
3136               snat_in2out_trace_t *t =
3137                  vlib_add_trace (vm, node, b1, sizeof (*t));
3138               t->is_slow_path = 0;
3139               t->sw_if_index = sw_if_index1;
3140               t->next_index = next1;
3141               t->session_index = ~0;
3142               if (ses1)
3143                 t->session_index = ses1 - dm1->sessions;
3144             }
3145
3146           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3147
3148           /* verify speculative enqueues, maybe switch current next frame */
3149           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3150                                            to_next, n_left_to_next,
3151                                            bi0, bi1, next0, next1);
3152          }
3153
3154       while (n_left_from > 0 && n_left_to_next > 0)
3155         {
3156           u32 bi0;
3157           vlib_buffer_t * b0;
3158           u32 next0;
3159           u32 sw_if_index0;
3160           ip4_header_t * ip0;
3161           ip_csum_t sum0;
3162           ip4_address_t new_addr0, old_addr0;
3163           u16 old_port0, new_port0, lo_port0, i0;
3164           udp_header_t * udp0;
3165           tcp_header_t * tcp0;
3166           u32 proto0;
3167           snat_det_out_key_t key0;
3168           snat_det_map_t * dm0;
3169           snat_det_session_t * ses0 = 0;
3170           u32 rx_fib_index0;
3171           icmp46_header_t * icmp0;
3172
3173           /* speculatively enqueue b0 to the current next frame */
3174           bi0 = from[0];
3175           to_next[0] = bi0;
3176           from += 1;
3177           to_next += 1;
3178           n_left_from -= 1;
3179           n_left_to_next -= 1;
3180
3181           b0 = vlib_get_buffer (vm, bi0);
3182           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3183
3184           ip0 = vlib_buffer_get_current (b0);
3185           udp0 = ip4_next_header (ip0);
3186           tcp0 = (tcp_header_t *) udp0;
3187
3188           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3189
3190           if (PREDICT_FALSE(ip0->ttl == 1))
3191             {
3192               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3193               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3194                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3195                                            0);
3196               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3197               goto trace00;
3198             }
3199
3200           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3201
3202           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3203             {
3204               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3205               icmp0 = (icmp46_header_t *) udp0;
3206
3207               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3208                                   rx_fib_index0, node, next0, thread_index,
3209                                   &ses0, &dm0);
3210               goto trace00;
3211             }
3212
3213           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3214           if (PREDICT_FALSE(!dm0))
3215             {
3216               nat_log_info ("no match for internal host %U",
3217                             format_ip4_address, &ip0->src_address);
3218               next0 = SNAT_IN2OUT_NEXT_DROP;
3219               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3220               goto trace00;
3221             }
3222
3223           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3224
3225           key0.ext_host_addr = ip0->dst_address;
3226           key0.ext_host_port = tcp0->dst;
3227
3228           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3229           if (PREDICT_FALSE(!ses0))
3230             {
3231               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3232                 {
3233                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3234                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3235
3236                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3237                     continue;
3238
3239                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3240                   break;
3241                 }
3242               if (PREDICT_FALSE(!ses0))
3243                 {
3244                   /* too many sessions for user, send ICMP error packet */
3245
3246                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3247                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3248                                                ICMP4_destination_unreachable_destination_unreachable_host,
3249                                                0);
3250                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3251                   goto trace00;
3252                 }
3253             }
3254
3255           new_port0 = ses0->out.out_port;
3256
3257           old_addr0.as_u32 = ip0->src_address.as_u32;
3258           ip0->src_address.as_u32 = new_addr0.as_u32;
3259           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3260
3261           sum0 = ip0->checksum;
3262           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3263                                  ip4_header_t,
3264                                  src_address /* changed member */);
3265           ip0->checksum = ip_csum_fold (sum0);
3266
3267           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3268             {
3269               if (tcp0->flags & TCP_FLAG_SYN)
3270                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3271               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3272                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3273               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3274                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3275               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3276                 snat_det_ses_close(dm0, ses0);
3277               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3278                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3279               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3280                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3281
3282               old_port0 = tcp0->src;
3283               tcp0->src = new_port0;
3284
3285               sum0 = tcp0->checksum;
3286               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3287                                      ip4_header_t,
3288                                      dst_address /* changed member */);
3289               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3290                                      ip4_header_t /* cheat */,
3291                                      length /* changed member */);
3292               tcp0->checksum = ip_csum_fold(sum0);
3293             }
3294           else
3295             {
3296               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3297               old_port0 = udp0->src_port;
3298               udp0->src_port = new_port0;
3299               udp0->checksum = 0;
3300             }
3301
3302           switch(ses0->state)
3303             {
3304             case SNAT_SESSION_UDP_ACTIVE:
3305                 ses0->expire = now + sm->udp_timeout;
3306                 break;
3307             case SNAT_SESSION_TCP_SYN_SENT:
3308             case SNAT_SESSION_TCP_FIN_WAIT:
3309             case SNAT_SESSION_TCP_CLOSE_WAIT:
3310             case SNAT_SESSION_TCP_LAST_ACK:
3311                 ses0->expire = now + sm->tcp_transitory_timeout;
3312                 break;
3313             case SNAT_SESSION_TCP_ESTABLISHED:
3314                 ses0->expire = now + sm->tcp_established_timeout;
3315                 break;
3316             }
3317
3318         trace00:
3319           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3320                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3321             {
3322               snat_in2out_trace_t *t =
3323                  vlib_add_trace (vm, node, b0, sizeof (*t));
3324               t->is_slow_path = 0;
3325               t->sw_if_index = sw_if_index0;
3326               t->next_index = next0;
3327               t->session_index = ~0;
3328               if (ses0)
3329                 t->session_index = ses0 - dm0->sessions;
3330             }
3331
3332           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3333
3334           /* verify speculative enqueue, maybe switch current next frame */
3335           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3336                                            to_next, n_left_to_next,
3337                                            bi0, next0);
3338         }
3339
3340       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3341     }
3342
3343   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3344                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3345                                pkts_processed);
3346   return frame->n_vectors;
3347 }
3348
3349 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3350   .function = snat_det_in2out_node_fn,
3351   .name = "nat44-det-in2out",
3352   .vector_size = sizeof (u32),
3353   .format_trace = format_snat_in2out_trace,
3354   .type = VLIB_NODE_TYPE_INTERNAL,
3355
3356   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3357   .error_strings = snat_in2out_error_strings,
3358
3359   .runtime_data_bytes = sizeof (snat_runtime_t),
3360
3361   .n_next_nodes = 3,
3362
3363   /* edit / add dispositions here */
3364   .next_nodes = {
3365     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3366     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3367     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3368   },
3369 };
3370
3371 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3372
3373 /**
3374  * Get address and port values to be used for ICMP packet translation
3375  * and create session if needed
3376  *
3377  * @param[in,out] sm             NAT main
3378  * @param[in,out] node           NAT node runtime
3379  * @param[in] thread_index       thread index
3380  * @param[in,out] b0             buffer containing packet to be translated
3381  * @param[out] p_proto           protocol used for matching
3382  * @param[out] p_value           address and port after NAT translation
3383  * @param[out] p_dont_translate  if packet should not be translated
3384  * @param d                      optional parameter
3385  * @param e                      optional parameter
3386  */
3387 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3388                           u32 thread_index, vlib_buffer_t *b0,
3389                           ip4_header_t *ip0, u8 *p_proto,
3390                           snat_session_key_t *p_value,
3391                           u8 *p_dont_translate, void *d, void *e)
3392 {
3393   icmp46_header_t *icmp0;
3394   u32 sw_if_index0;
3395   u32 rx_fib_index0;
3396   u8 protocol;
3397   snat_det_out_key_t key0;
3398   u8 dont_translate = 0;
3399   u32 next0 = ~0;
3400   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3401   ip4_header_t *inner_ip0;
3402   void *l4_header = 0;
3403   icmp46_header_t *inner_icmp0;
3404   snat_det_map_t * dm0 = 0;
3405   ip4_address_t new_addr0;
3406   u16 lo_port0, i0;
3407   snat_det_session_t * ses0 = 0;
3408   ip4_address_t in_addr;
3409   u16 in_port;
3410
3411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3412   echo0 = (icmp_echo_header_t *)(icmp0+1);
3413   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3414   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3415
3416   if (!icmp_is_error_message (icmp0))
3417     {
3418       protocol = SNAT_PROTOCOL_ICMP;
3419       in_addr = ip0->src_address;
3420       in_port = echo0->identifier;
3421     }
3422   else
3423     {
3424       inner_ip0 = (ip4_header_t *)(echo0+1);
3425       l4_header = ip4_next_header (inner_ip0);
3426       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3427       in_addr = inner_ip0->dst_address;
3428       switch (protocol)
3429         {
3430         case SNAT_PROTOCOL_ICMP:
3431           inner_icmp0 = (icmp46_header_t*)l4_header;
3432           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3433           in_port = inner_echo0->identifier;
3434           break;
3435         case SNAT_PROTOCOL_UDP:
3436         case SNAT_PROTOCOL_TCP:
3437           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3438           break;
3439         default:
3440           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3441           next0 = SNAT_IN2OUT_NEXT_DROP;
3442           goto out;
3443         }
3444     }
3445
3446   dm0 = snat_det_map_by_user(sm, &in_addr);
3447   if (PREDICT_FALSE(!dm0))
3448     {
3449       nat_log_info ("no match for internal host %U",
3450                     format_ip4_address, &in_addr);
3451       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3452           IP_PROTOCOL_ICMP, rx_fib_index0)))
3453         {
3454           dont_translate = 1;
3455           goto out;
3456         }
3457       next0 = SNAT_IN2OUT_NEXT_DROP;
3458       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3459       goto out;
3460     }
3461
3462   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3463
3464   key0.ext_host_addr = ip0->dst_address;
3465   key0.ext_host_port = 0;
3466
3467   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3468   if (PREDICT_FALSE(!ses0))
3469     {
3470       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3471           IP_PROTOCOL_ICMP, rx_fib_index0)))
3472         {
3473           dont_translate = 1;
3474           goto out;
3475         }
3476       if (icmp0->type != ICMP4_echo_request)
3477         {
3478           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3479           next0 = SNAT_IN2OUT_NEXT_DROP;
3480           goto out;
3481         }
3482       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3483         {
3484           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3485             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3486
3487           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3488             continue;
3489
3490           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3491           break;
3492         }
3493       if (PREDICT_FALSE(!ses0))
3494         {
3495           next0 = SNAT_IN2OUT_NEXT_DROP;
3496           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3497           goto out;
3498         }
3499     }
3500
3501   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3502                     !icmp_is_error_message (icmp0)))
3503     {
3504       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3505       next0 = SNAT_IN2OUT_NEXT_DROP;
3506       goto out;
3507     }
3508
3509   u32 now = (u32) vlib_time_now (sm->vlib_main);
3510
3511   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3512   ses0->expire = now + sm->icmp_timeout;
3513
3514 out:
3515   *p_proto = protocol;
3516   if (ses0)
3517     {
3518       p_value->addr = new_addr0;
3519       p_value->fib_index = sm->outside_fib_index;
3520       p_value->port = ses0->out.out_port;
3521     }
3522   *p_dont_translate = dont_translate;
3523   if (d)
3524     *(snat_det_session_t**)d = ses0;
3525   if (e)
3526     *(snat_det_map_t**)e = dm0;
3527   return next0;
3528 }
3529
3530 /**********************/
3531 /*** worker handoff ***/
3532 /**********************/
3533 static inline uword
3534 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3535                                       vlib_node_runtime_t * node,
3536                                       vlib_frame_t * frame,
3537                                       u8 is_output)
3538 {
3539   snat_main_t *sm = &snat_main;
3540   vlib_thread_main_t *tm = vlib_get_thread_main ();
3541   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3542   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3543   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3544     = 0;
3545   vlib_frame_queue_elt_t *hf = 0;
3546   vlib_frame_queue_t *fq;
3547   vlib_frame_t *f = 0;
3548   int i;
3549   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3550   u32 next_worker_index = 0;
3551   u32 current_worker_index = ~0;
3552   u32 thread_index = vlib_get_thread_index ();
3553   u32 fq_index;
3554   u32 to_node_index;
3555   vlib_frame_t *d = 0;
3556
3557   ASSERT (vec_len (sm->workers));
3558
3559   if (is_output)
3560     {
3561       fq_index = sm->fq_in2out_output_index;
3562       to_node_index = sm->in2out_output_node_index;
3563     }
3564   else
3565     {
3566       fq_index = sm->fq_in2out_index;
3567       to_node_index = sm->in2out_node_index;
3568     }
3569
3570   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3571     {
3572       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3573
3574       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3575                                tm->n_vlib_mains - 1,
3576                                (vlib_frame_queue_t *) (~0));
3577     }
3578
3579   from = vlib_frame_vector_args (frame);
3580   n_left_from = frame->n_vectors;
3581
3582   while (n_left_from > 0)
3583     {
3584       u32 bi0;
3585       vlib_buffer_t *b0;
3586       u32 sw_if_index0;
3587       u32 rx_fib_index0;
3588       ip4_header_t * ip0;
3589       u8 do_handoff;
3590
3591       bi0 = from[0];
3592       from += 1;
3593       n_left_from -= 1;
3594
3595       b0 = vlib_get_buffer (vm, bi0);
3596
3597       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3598       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3599
3600       ip0 = vlib_buffer_get_current (b0);
3601
3602       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3603
3604       if (PREDICT_FALSE (next_worker_index != thread_index))
3605         {
3606           do_handoff = 1;
3607
3608           if (next_worker_index != current_worker_index)
3609             {
3610               fq = is_vlib_frame_queue_congested (
3611                 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
3612                 congested_handoff_queue_by_worker_index);
3613
3614               if (fq)
3615                 {
3616                   /* if this is 1st frame */
3617                   if (!d)
3618                     {
3619                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
3620                       to_next_drop = vlib_frame_vector_args (d);
3621                     }
3622
3623                   to_next_drop[0] = bi0;
3624                   to_next_drop += 1;
3625                   d->n_vectors++;
3626                   b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
3627                   goto trace0;
3628                 }
3629
3630               if (hf)
3631                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3632
3633               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3634                                                       next_worker_index,
3635                                                       handoff_queue_elt_by_worker_index);
3636
3637               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3638               to_next_worker = &hf->buffer_index[hf->n_vectors];
3639               current_worker_index = next_worker_index;
3640             }
3641
3642           /* enqueue to correct worker thread */
3643           to_next_worker[0] = bi0;
3644           to_next_worker++;
3645           n_left_to_next_worker--;
3646
3647           if (n_left_to_next_worker == 0)
3648             {
3649               hf->n_vectors = VLIB_FRAME_SIZE;
3650               vlib_put_frame_queue_elt (hf);
3651               current_worker_index = ~0;
3652               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3653               hf = 0;
3654             }
3655         }
3656       else
3657         {
3658           do_handoff = 0;
3659           /* if this is 1st frame */
3660           if (!f)
3661             {
3662               f = vlib_get_frame_to_node (vm, to_node_index);
3663               to_next = vlib_frame_vector_args (f);
3664             }
3665
3666           to_next[0] = bi0;
3667           to_next += 1;
3668           f->n_vectors++;
3669         }
3670
3671 trace0:
3672       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3673                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3674         {
3675           snat_in2out_worker_handoff_trace_t *t =
3676             vlib_add_trace (vm, node, b0, sizeof (*t));
3677           t->next_worker_index = next_worker_index;
3678           t->do_handoff = do_handoff;
3679         }
3680     }
3681
3682   if (f)
3683     vlib_put_frame_to_node (vm, to_node_index, f);
3684
3685   if (d)
3686     vlib_put_frame_to_node (vm, sm->error_node_index, d);
3687
3688   if (hf)
3689     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3690
3691   /* Ship frames to the worker nodes */
3692   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3693     {
3694       if (handoff_queue_elt_by_worker_index[i])
3695         {
3696           hf = handoff_queue_elt_by_worker_index[i];
3697           /*
3698            * It works better to let the handoff node
3699            * rate-adapt, always ship the handoff queue element.
3700            */
3701           if (1 || hf->n_vectors == hf->last_n_vectors)
3702             {
3703               vlib_put_frame_queue_elt (hf);
3704               handoff_queue_elt_by_worker_index[i] = 0;
3705             }
3706           else
3707             hf->last_n_vectors = hf->n_vectors;
3708         }
3709       congested_handoff_queue_by_worker_index[i] =
3710         (vlib_frame_queue_t *) (~0);
3711     }
3712   hf = 0;
3713   current_worker_index = ~0;
3714   return frame->n_vectors;
3715 }
3716
3717 static uword
3718 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3719                                vlib_node_runtime_t * node,
3720                                vlib_frame_t * frame)
3721 {
3722   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3723 }
3724
3725 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3726   .function = snat_in2out_worker_handoff_fn,
3727   .name = "nat44-in2out-worker-handoff",
3728   .vector_size = sizeof (u32),
3729   .format_trace = format_snat_in2out_worker_handoff_trace,
3730   .type = VLIB_NODE_TYPE_INTERNAL,
3731
3732   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3733   .error_strings = snat_in2out_error_strings,
3734
3735   .n_next_nodes = 1,
3736
3737   .next_nodes = {
3738     [0] = "error-drop",
3739   },
3740 };
3741
3742 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3743                               snat_in2out_worker_handoff_fn);
3744
3745 static uword
3746 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3747                                       vlib_node_runtime_t * node,
3748                                       vlib_frame_t * frame)
3749 {
3750   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3751 }
3752
3753 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3754   .function = snat_in2out_output_worker_handoff_fn,
3755   .name = "nat44-in2out-output-worker-handoff",
3756   .vector_size = sizeof (u32),
3757   .format_trace = format_snat_in2out_worker_handoff_trace,
3758   .type = VLIB_NODE_TYPE_INTERNAL,
3759
3760   .n_next_nodes = 1,
3761
3762   .next_nodes = {
3763     [0] = "error-drop",
3764   },
3765 };
3766
3767 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3768                               snat_in2out_output_worker_handoff_fn);
3769
3770 static_always_inline int
3771 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3772 {
3773   snat_address_t * ap;
3774   clib_bihash_kv_8_8_t kv, value;
3775   snat_session_key_t m_key;
3776
3777   vec_foreach (ap, sm->addresses)
3778     {
3779       if (ap->addr.as_u32 == dst_addr->as_u32)
3780         return 1;
3781     }
3782
3783   m_key.addr.as_u32 = dst_addr->as_u32;
3784   m_key.fib_index = sm->outside_fib_index;
3785   m_key.port = 0;
3786   m_key.protocol = 0;
3787   kv.key = m_key.as_u64;
3788   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3789     return 1;
3790
3791   return 0;
3792 }
3793
3794 static uword
3795 snat_hairpin_dst_fn (vlib_main_t * vm,
3796                      vlib_node_runtime_t * node,
3797                      vlib_frame_t * frame)
3798 {
3799   u32 n_left_from, * from, * to_next;
3800   snat_in2out_next_t next_index;
3801   u32 pkts_processed = 0;
3802   snat_main_t * sm = &snat_main;
3803
3804   from = vlib_frame_vector_args (frame);
3805   n_left_from = frame->n_vectors;
3806   next_index = node->cached_next_index;
3807
3808   while (n_left_from > 0)
3809     {
3810       u32 n_left_to_next;
3811
3812       vlib_get_next_frame (vm, node, next_index,
3813                            to_next, n_left_to_next);
3814
3815       while (n_left_from > 0 && n_left_to_next > 0)
3816         {
3817           u32 bi0;
3818           vlib_buffer_t * b0;
3819           u32 next0;
3820           ip4_header_t * ip0;
3821           u32 proto0;
3822
3823           /* speculatively enqueue b0 to the current next frame */
3824           bi0 = from[0];
3825           to_next[0] = bi0;
3826           from += 1;
3827           to_next += 1;
3828           n_left_from -= 1;
3829           n_left_to_next -= 1;
3830
3831           b0 = vlib_get_buffer (vm, bi0);
3832           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3833           ip0 = vlib_buffer_get_current (b0);
3834
3835           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3836
3837           vnet_buffer (b0)->snat.flags = 0;
3838           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3839             {
3840               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3841                 {
3842                   udp_header_t * udp0 = ip4_next_header (ip0);
3843                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3844
3845                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3846                 }
3847               else if (proto0 == SNAT_PROTOCOL_ICMP)
3848                 {
3849                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3850
3851                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3852                 }
3853               else
3854                 {
3855                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3856                 }
3857
3858               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3859             }
3860
3861           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3862
3863           /* verify speculative enqueue, maybe switch current next frame */
3864           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3865                                            to_next, n_left_to_next,
3866                                            bi0, next0);
3867          }
3868
3869       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3870     }
3871
3872   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3873                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3874                                pkts_processed);
3875   return frame->n_vectors;
3876 }
3877
3878 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3879   .function = snat_hairpin_dst_fn,
3880   .name = "nat44-hairpin-dst",
3881   .vector_size = sizeof (u32),
3882   .type = VLIB_NODE_TYPE_INTERNAL,
3883   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3884   .error_strings = snat_in2out_error_strings,
3885   .n_next_nodes = 2,
3886   .next_nodes = {
3887     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3888     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3889   },
3890 };
3891
3892 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3893                               snat_hairpin_dst_fn);
3894
3895 static uword
3896 snat_hairpin_src_fn (vlib_main_t * vm,
3897                      vlib_node_runtime_t * node,
3898                      vlib_frame_t * frame)
3899 {
3900   u32 n_left_from, * from, * to_next;
3901   snat_in2out_next_t next_index;
3902   u32 pkts_processed = 0;
3903   snat_main_t *sm = &snat_main;
3904
3905   from = vlib_frame_vector_args (frame);
3906   n_left_from = frame->n_vectors;
3907   next_index = node->cached_next_index;
3908
3909   while (n_left_from > 0)
3910     {
3911       u32 n_left_to_next;
3912
3913       vlib_get_next_frame (vm, node, next_index,
3914                            to_next, n_left_to_next);
3915
3916       while (n_left_from > 0 && n_left_to_next > 0)
3917         {
3918           u32 bi0;
3919           vlib_buffer_t * b0;
3920           u32 next0;
3921           snat_interface_t *i;
3922           u32 sw_if_index0;
3923
3924           /* speculatively enqueue b0 to the current next frame */
3925           bi0 = from[0];
3926           to_next[0] = bi0;
3927           from += 1;
3928           to_next += 1;
3929           n_left_from -= 1;
3930           n_left_to_next -= 1;
3931
3932           b0 = vlib_get_buffer (vm, bi0);
3933           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3934           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3935
3936           pool_foreach (i, sm->output_feature_interfaces,
3937           ({
3938             /* Only packets from NAT inside interface */
3939             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3940               {
3941                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3942                                     SNAT_FLAG_HAIRPINNING))
3943                   {
3944                     if (PREDICT_TRUE (sm->num_workers > 1))
3945                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3946                     else
3947                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3948                   }
3949                 break;
3950               }
3951           }));
3952
3953           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3954
3955           /* verify speculative enqueue, maybe switch current next frame */
3956           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3957                                            to_next, n_left_to_next,
3958                                            bi0, next0);
3959          }
3960
3961       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3962     }
3963
3964   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3965                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3966                                pkts_processed);
3967   return frame->n_vectors;
3968 }
3969
3970 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3971   .function = snat_hairpin_src_fn,
3972   .name = "nat44-hairpin-src",
3973   .vector_size = sizeof (u32),
3974   .type = VLIB_NODE_TYPE_INTERNAL,
3975   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3976   .error_strings = snat_in2out_error_strings,
3977   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3978   .next_nodes = {
3979      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3980      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3981      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3982      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3983   },
3984 };
3985
3986 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3987                               snat_hairpin_src_fn);
3988
3989 static uword
3990 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3991                                 vlib_node_runtime_t * node,
3992                                 vlib_frame_t * frame)
3993 {
3994   u32 n_left_from, * from, * to_next;
3995   snat_in2out_next_t next_index;
3996   u32 pkts_processed = 0;
3997   snat_main_t * sm = &snat_main;
3998   u32 stats_node_index;
3999
4000   stats_node_index = snat_in2out_fast_node.index;
4001
4002   from = vlib_frame_vector_args (frame);
4003   n_left_from = frame->n_vectors;
4004   next_index = node->cached_next_index;
4005
4006   while (n_left_from > 0)
4007     {
4008       u32 n_left_to_next;
4009
4010       vlib_get_next_frame (vm, node, next_index,
4011                            to_next, n_left_to_next);
4012
4013       while (n_left_from > 0 && n_left_to_next > 0)
4014         {
4015           u32 bi0;
4016           vlib_buffer_t * b0;
4017           u32 next0;
4018           u32 sw_if_index0;
4019           ip4_header_t * ip0;
4020           ip_csum_t sum0;
4021           u32 new_addr0, old_addr0;
4022           u16 old_port0, new_port0;
4023           udp_header_t * udp0;
4024           tcp_header_t * tcp0;
4025           icmp46_header_t * icmp0;
4026           snat_session_key_t key0, sm0;
4027           u32 proto0;
4028           u32 rx_fib_index0;
4029
4030           /* speculatively enqueue b0 to the current next frame */
4031           bi0 = from[0];
4032           to_next[0] = bi0;
4033           from += 1;
4034           to_next += 1;
4035           n_left_from -= 1;
4036           n_left_to_next -= 1;
4037
4038           b0 = vlib_get_buffer (vm, bi0);
4039           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4040
4041           ip0 = vlib_buffer_get_current (b0);
4042           udp0 = ip4_next_header (ip0);
4043           tcp0 = (tcp_header_t *) udp0;
4044           icmp0 = (icmp46_header_t *) udp0;
4045
4046           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4047           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4048
4049           if (PREDICT_FALSE(ip0->ttl == 1))
4050             {
4051               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4052               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4053                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4054                                            0);
4055               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4056               goto trace0;
4057             }
4058
4059           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4060
4061           if (PREDICT_FALSE (proto0 == ~0))
4062               goto trace0;
4063
4064           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4065             {
4066               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4067                                   rx_fib_index0, node, next0, ~0, 0, 0);
4068               goto trace0;
4069             }
4070
4071           key0.addr = ip0->src_address;
4072           key0.protocol = proto0;
4073           key0.port = udp0->src_port;
4074           key0.fib_index = rx_fib_index0;
4075
4076           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4077             {
4078               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4079               next0= SNAT_IN2OUT_NEXT_DROP;
4080               goto trace0;
4081             }
4082
4083           new_addr0 = sm0.addr.as_u32;
4084           new_port0 = sm0.port;
4085           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4086           old_addr0 = ip0->src_address.as_u32;
4087           ip0->src_address.as_u32 = new_addr0;
4088
4089           sum0 = ip0->checksum;
4090           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4091                                  ip4_header_t,
4092                                  src_address /* changed member */);
4093           ip0->checksum = ip_csum_fold (sum0);
4094
4095           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4096             {
4097               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4098                 {
4099                   old_port0 = tcp0->src_port;
4100                   tcp0->src_port = new_port0;
4101
4102                   sum0 = tcp0->checksum;
4103                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4104                                          ip4_header_t,
4105                                          dst_address /* changed member */);
4106                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4107                                          ip4_header_t /* cheat */,
4108                                          length /* changed member */);
4109                   tcp0->checksum = ip_csum_fold(sum0);
4110                 }
4111               else
4112                 {
4113                   old_port0 = udp0->src_port;
4114                   udp0->src_port = new_port0;
4115                   udp0->checksum = 0;
4116                 }
4117             }
4118           else
4119             {
4120               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4121                 {
4122                   sum0 = tcp0->checksum;
4123                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4124                                          ip4_header_t,
4125                                          dst_address /* changed member */);
4126                   tcp0->checksum = ip_csum_fold(sum0);
4127                 }
4128             }
4129
4130           /* Hairpinning */
4131           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4132
4133         trace0:
4134           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4135                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4136             {
4137               snat_in2out_trace_t *t =
4138                  vlib_add_trace (vm, node, b0, sizeof (*t));
4139               t->sw_if_index = sw_if_index0;
4140               t->next_index = next0;
4141             }
4142
4143           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4144
4145           /* verify speculative enqueue, maybe switch current next frame */
4146           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4147                                            to_next, n_left_to_next,
4148                                            bi0, next0);
4149         }
4150
4151       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4152     }
4153
4154   vlib_node_increment_counter (vm, stats_node_index,
4155                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4156                                pkts_processed);
4157   return frame->n_vectors;
4158 }
4159
4160
4161 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4162   .function = snat_in2out_fast_static_map_fn,
4163   .name = "nat44-in2out-fast",
4164   .vector_size = sizeof (u32),
4165   .format_trace = format_snat_in2out_fast_trace,
4166   .type = VLIB_NODE_TYPE_INTERNAL,
4167
4168   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4169   .error_strings = snat_in2out_error_strings,
4170
4171   .runtime_data_bytes = sizeof (snat_runtime_t),
4172
4173   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4174
4175   /* edit / add dispositions here */
4176   .next_nodes = {
4177     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4178     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4179     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4180     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4181     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4182   },
4183 };
4184
4185 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);