src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external addresss with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 256                       ip4_header_t * ip0,
 257                       u32 rx_fib_index0,
 258                       snat_session_key_t * key0,
 259                       snat_session_t ** sessionp,
 260                       vlib_node_runtime_t * node,
 261                       u32 next0,
 262                       u32 thread_index)
 263 {
 264   snat_user_t *u;
 265   snat_session_t *s;
 266   clib_bihash_kv_8_8_t kv0;
 267   snat_session_key_t key1;
 268   u32 address_index = ~0;
 269   u32 outside_fib_index;
 270   uword * p;
 271   udp_header_t * udp0 = ip4_next_header (ip0);
 272
 273   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 274     {
 275       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 276       return SNAT_IN2OUT_NEXT_DROP;
 277     }
 278
 279   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 280   if (! p)
 281     {
 282       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 283       return SNAT_IN2OUT_NEXT_DROP;
 284     }
 285   outside_fib_index = p[0];
 286
 287   key1.protocol = key0->protocol;
 288
 289   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 290                               thread_index);
 291   if (!u)
 292     {
 293       clib_warning ("create NAT user failed");
 294       return SNAT_IN2OUT_NEXT_DROP;
 295     }
 296
 297   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 298   if (!s)
 299     {
 300       clib_warning ("create NAT session failed");
 301       return SNAT_IN2OUT_NEXT_DROP;
 302     }
 303
 304   /* First try to match static mapping by local address and port */
 305   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
 306     {
 307       /* Try to create dynamic translation */
 308       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 309                                                thread_index, &key1,
 310                                                &address_index,
 311                                                sm->port_per_thread,
 312                                                sm->per_thread_data[thread_index].snat_thread_index))
 313         {
 314           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 315           return SNAT_IN2OUT_NEXT_DROP;
 316         }
 317       u->nsessions++;
 318     }
 319   else
 320     {
 321       u->nstaticsessions++;
 322       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 323     }
 324
 325   s->outside_address_index = address_index;
 326   s->in2out = *key0;
 327   s->out2in = key1;
 328   s->out2in.protocol = key0->protocol;
 329   s->out2in.fib_index = outside_fib_index;
 330   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 331   s->ext_host_port = udp0->dst_port;
 332   *sessionp = s;
 333
 334   /* Add to translation hashes */
 335   kv0.key = s->in2out.as_u64;
 336   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 337   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 338                                1 /* is_add */))
 339       clib_warning ("in2out key add failed");
 340
 341   kv0.key = s->out2in.as_u64;
 342   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 343
 344   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 345                                1 /* is_add */))
 346       clib_warning ("out2in key add failed");
 347
 348   /* log NAT event */
 349   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 350                                       s->out2in.addr.as_u32,
 351                                       s->in2out.protocol,
 352                                       s->in2out.port,
 353                                       s->out2in.port,
 354                                       s->in2out.fib_index);
 355   return next0;
 356 }
 357
 358 static_always_inline
 359 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 360                                  snat_session_key_t *p_key0)
 361 {
 362   icmp46_header_t *icmp0;
 363   snat_session_key_t key0;
 364   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 365   ip4_header_t *inner_ip0 = 0;
 366   void *l4_header = 0;
 367   icmp46_header_t *inner_icmp0;
 368
 369   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 370   echo0 = (icmp_echo_header_t *)(icmp0+1);
 371
 372   if (!icmp_is_error_message (icmp0))
 373     {
 374       key0.protocol = SNAT_PROTOCOL_ICMP;
 375       key0.addr = ip0->src_address;
 376       key0.port = echo0->identifier;
 377     }
 378   else
 379     {
 380       inner_ip0 = (ip4_header_t *)(echo0+1);
 381       l4_header = ip4_next_header (inner_ip0);
 382       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 383       key0.addr = inner_ip0->dst_address;
 384       switch (key0.protocol)
 385         {
 386         case SNAT_PROTOCOL_ICMP:
 387           inner_icmp0 = (icmp46_header_t*)l4_header;
 388           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 389           key0.port = inner_echo0->identifier;
 390           break;
 391         case SNAT_PROTOCOL_UDP:
 392         case SNAT_PROTOCOL_TCP:
 393           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 394           break;
 395         default:
 396           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 397         }
 398     }
 399   *p_key0 = key0;
 400   return -1; /* success */
 401 }
 402
 403 /**
 404  * Get address and port values to be used for ICMP packet translation
 405  * and create session if needed
 406  *
 407  * @param[in,out] sm             NAT main
 408  * @param[in,out] node           NAT node runtime
 409  * @param[in] thread_index       thread index
 410  * @param[in,out] b0             buffer containing packet to be translated
 411  * @param[out] p_proto           protocol used for matching
 412  * @param[out] p_value           address and port after NAT translation
 413  * @param[out] p_dont_translate  if packet should not be translated
 414  * @param d                      optional parameter
 415  * @param e                      optional parameter
 416  */
 417 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 418                            u32 thread_index, vlib_buffer_t *b0,
 419                            ip4_header_t *ip0, u8 *p_proto,
 420                            snat_session_key_t *p_value,
 421                            u8 *p_dont_translate, void *d, void *e)
 422 {
 423   icmp46_header_t *icmp0;
 424   u32 sw_if_index0;
 425   u32 rx_fib_index0;
 426   snat_session_key_t key0;
 427   snat_session_t *s0 = 0;
 428   u8 dont_translate = 0;
 429   clib_bihash_kv_8_8_t kv0, value0;
 430   u32 next0 = ~0;
 431   int err;
 432
 433   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 434   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 435   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 436
 437   err = icmp_get_key (ip0, &key0);
 438   if (err != -1)
 439     {
 440       b0->error = node->errors[err];
 441       next0 = SNAT_IN2OUT_NEXT_DROP;
 442       goto out;
 443     }
 444   key0.fib_index = rx_fib_index0;
 445
 446   kv0.key = key0.as_u64;
 447
 448   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 449                               &value0))
 450     {
 451       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 452           IP_PROTOCOL_ICMP, rx_fib_index0, thread_index) &&
 453           vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
 454         {
 455           dont_translate = 1;
 456           goto out;
 457         }
 458
 459       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 460         {
 461           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 462           next0 = SNAT_IN2OUT_NEXT_DROP;
 463           goto out;
 464         }
 465
 466       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 467                          &s0, node, next0, thread_index);
 468
 469       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 470         goto out;
 471     }
 472   else
 473     {
 474       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 475                         icmp0->type != ICMP4_echo_reply &&
 476                         !icmp_is_error_message (icmp0)))
 477         {
 478           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 479           next0 = SNAT_IN2OUT_NEXT_DROP;
 480           goto out;
 481         }
 482
 483       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 484                               value0.value);
 485     }
 486
 487 out:
 488   *p_proto = key0.protocol;
 489   if (s0)
 490     *p_value = s0->out2in;
 491   *p_dont_translate = dont_translate;
 492   if (d)
 493     *(snat_session_t**)d = s0;
 494   return next0;
 495 }
 496
 497 /**
 498  * Get address and port values to be used for ICMP packet translation
 499  *
 500  * @param[in] sm                 NAT main
 501  * @param[in,out] node           NAT node runtime
 502  * @param[in] thread_index       thread index
 503  * @param[in,out] b0             buffer containing packet to be translated
 504  * @param[out] p_proto           protocol used for matching
 505  * @param[out] p_value           address and port after NAT translation
 506  * @param[out] p_dont_translate  if packet should not be translated
 507  * @param d                      optional parameter
 508  * @param e                      optional parameter
 509  */
 510 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 511                            u32 thread_index, vlib_buffer_t *b0,
 512                            ip4_header_t *ip0, u8 *p_proto,
 513                            snat_session_key_t *p_value,
 514                            u8 *p_dont_translate, void *d, void *e)
 515 {
 516   icmp46_header_t *icmp0;
 517   u32 sw_if_index0;
 518   u32 rx_fib_index0;
 519   snat_session_key_t key0;
 520   snat_session_key_t sm0;
 521   u8 dont_translate = 0;
 522   u8 is_addr_only;
 523   u32 next0 = ~0;
 524   int err;
 525
 526   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 527   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 528   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 529
 530   err = icmp_get_key (ip0, &key0);
 531   if (err != -1)
 532     {
 533       b0->error = node->errors[err];
 534       next0 = SNAT_IN2OUT_NEXT_DROP;
 535       goto out2;
 536     }
 537   key0.fib_index = rx_fib_index0;
 538
 539   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
 540     {
 541       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 542           IP_PROTOCOL_ICMP, rx_fib_index0)))
 543         {
 544           dont_translate = 1;
 545           goto out;
 546         }
 547
 548       if (icmp_is_error_message (icmp0))
 549         {
 550           next0 = SNAT_IN2OUT_NEXT_DROP;
 551           goto out;
 552         }
 553
 554       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 555       next0 = SNAT_IN2OUT_NEXT_DROP;
 556       goto out;
 557     }
 558
 559   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 560                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 561                     !icmp_is_error_message (icmp0)))
 562     {
 563       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 564       next0 = SNAT_IN2OUT_NEXT_DROP;
 565       goto out;
 566     }
 567
 568 out:
 569   *p_value = sm0;
 570 out2:
 571   *p_proto = key0.protocol;
 572   *p_dont_translate = dont_translate;
 573   return next0;
 574 }
 575
 576 static inline u32 icmp_in2out (snat_main_t *sm,
 577                                vlib_buffer_t * b0,
 578                                ip4_header_t * ip0,
 579                                icmp46_header_t * icmp0,
 580                                u32 sw_if_index0,
 581                                u32 rx_fib_index0,
 582                                vlib_node_runtime_t * node,
 583                                u32 next0,
 584                                u32 thread_index,
 585                                void *d,
 586                                void *e)
 587 {
 588   snat_session_key_t sm0;
 589   u8 protocol;
 590   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 591   ip4_header_t *inner_ip0;
 592   void *l4_header = 0;
 593   icmp46_header_t *inner_icmp0;
 594   u8 dont_translate;
 595   u32 new_addr0, old_addr0;
 596   u16 old_id0, new_id0;
 597   ip_csum_t sum0;
 598   u16 checksum0;
 599   u32 next0_tmp;
 600
 601   echo0 = (icmp_echo_header_t *)(icmp0+1);
 602
 603   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 604                                        &protocol, &sm0, &dont_translate, d, e);
 605   if (next0_tmp != ~0)
 606     next0 = next0_tmp;
 607   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 608     goto out;
 609
 610   sum0 = ip_incremental_checksum (0, icmp0,
 611                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 612   checksum0 = ~ip_csum_fold (sum0);
 613   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 614     {
 615       next0 = SNAT_IN2OUT_NEXT_DROP;
 616       goto out;
 617     }
 618
 619   old_addr0 = ip0->src_address.as_u32;
 620   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 621   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 622     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 623
 624   sum0 = ip0->checksum;
 625   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 626                          src_address /* changed member */);
 627   ip0->checksum = ip_csum_fold (sum0);
 628
 629   if (!icmp_is_error_message (icmp0))
 630     {
 631       new_id0 = sm0.port;
 632       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 633         {
 634           old_id0 = echo0->identifier;
 635           new_id0 = sm0.port;
 636           echo0->identifier = new_id0;
 637
 638           sum0 = icmp0->checksum;
 639           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 640                                  identifier);
 641           icmp0->checksum = ip_csum_fold (sum0);
 642         }
 643     }
 644   else
 645     {
 646       inner_ip0 = (ip4_header_t *)(echo0+1);
 647       l4_header = ip4_next_header (inner_ip0);
 648
 649       if (!ip4_header_checksum_is_valid (inner_ip0))
 650         {
 651           next0 = SNAT_IN2OUT_NEXT_DROP;
 652           goto out;
 653         }
 654
 655       old_addr0 = inner_ip0->dst_address.as_u32;
 656       inner_ip0->dst_address = sm0.addr;
 657       new_addr0 = inner_ip0->dst_address.as_u32;
 658
 659       sum0 = icmp0->checksum;
 660       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 661                              dst_address /* changed member */);
 662       icmp0->checksum = ip_csum_fold (sum0);
 663
 664       switch (protocol)
 665         {
 666           case SNAT_PROTOCOL_ICMP:
 667             inner_icmp0 = (icmp46_header_t*)l4_header;
 668             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 669
 670             old_id0 = inner_echo0->identifier;
 671             new_id0 = sm0.port;
 672             inner_echo0->identifier = new_id0;
 673
 674             sum0 = icmp0->checksum;
 675             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 676                                    identifier);
 677             icmp0->checksum = ip_csum_fold (sum0);
 678             break;
 679           case SNAT_PROTOCOL_UDP:
 680           case SNAT_PROTOCOL_TCP:
 681             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 682             new_id0 = sm0.port;
 683             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 684
 685             sum0 = icmp0->checksum;
 686             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 687                                    dst_port);
 688             icmp0->checksum = ip_csum_fold (sum0);
 689             break;
 690           default:
 691             ASSERT(0);
 692         }
 693     }
 694
 695 out:
 696   return next0;
 697 }
 698
 699 /**
 700  * @brief Hairpinning
 701  *
 702  * Hairpinning allows two endpoints on the internal side of the NAT to
 703  * communicate even if they only use each other's external IP addresses
 704  * and ports.
 705  *
 706  * @param sm     NAT main.
 707  * @param b0     Vlib buffer.
 708  * @param ip0    IP header.
 709  * @param udp0   UDP header.
 710  * @param tcp0   TCP header.
 711  * @param proto0 NAT protocol.
 712  */
 713 static inline int
 714 snat_hairpinning (snat_main_t *sm,
 715                   vlib_buffer_t * b0,
 716                   ip4_header_t * ip0,
 717                   udp_header_t * udp0,
 718                   tcp_header_t * tcp0,
 719                   u32 proto0)
 720 {
 721   snat_session_key_t key0, sm0;
 722   snat_session_t * s0;
 723   clib_bihash_kv_8_8_t kv0, value0;
 724   ip_csum_t sum0;
 725   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 726   u16 new_dst_port0, old_dst_port0;
 727
 728   key0.addr = ip0->dst_address;
 729   key0.port = udp0->dst_port;
 730   key0.protocol = proto0;
 731   key0.fib_index = sm->outside_fib_index;
 732   kv0.key = key0.as_u64;
 733
 734   /* Check if destination is static mappings */
 735   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 736     {
 737       new_dst_addr0 = sm0.addr.as_u32;
 738       new_dst_port0 = sm0.port;
 739       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 740     }
 741   /* or active session */
 742   else
 743     {
 744       if (sm->num_workers > 1)
 745         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 746       else
 747         ti = sm->num_workers;
 748
 749       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 750         {
 751           si = value0.value;
 752
 753           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 754           new_dst_addr0 = s0->in2out.addr.as_u32;
 755           new_dst_port0 = s0->in2out.port;
 756           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 757         }
 758     }
 759
 760   /* Destination is behind the same NAT, use internal address and port */
 761   if (new_dst_addr0)
 762     {
 763       old_dst_addr0 = ip0->dst_address.as_u32;
 764       ip0->dst_address.as_u32 = new_dst_addr0;
 765       sum0 = ip0->checksum;
 766       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 767                              ip4_header_t, dst_address);
 768       ip0->checksum = ip_csum_fold (sum0);
 769
 770       old_dst_port0 = tcp0->dst;
 771       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 772         {
 773           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 774             {
 775               tcp0->dst = new_dst_port0;
 776               sum0 = tcp0->checksum;
 777               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 778                                      ip4_header_t, dst_address);
 779               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 780                                      ip4_header_t /* cheat */, length);
 781               tcp0->checksum = ip_csum_fold(sum0);
 782             }
 783           else
 784             {
 785               udp0->dst_port = new_dst_port0;
 786               udp0->checksum = 0;
 787             }
 788         }
 789       else
 790         {
 791           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 792             {
 793               sum0 = tcp0->checksum;
 794               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 795                                      ip4_header_t, dst_address);
 796               tcp0->checksum = ip_csum_fold(sum0);
 797             }
 798         }
 799       return 1;
 800     }
 801   return 0;
 802 }
 803
 804 static inline void
 805 snat_icmp_hairpinning (snat_main_t *sm,
 806                        vlib_buffer_t * b0,
 807                        ip4_header_t * ip0,
 808                        icmp46_header_t * icmp0)
 809 {
 810   snat_session_key_t key0, sm0;
 811   clib_bihash_kv_8_8_t kv0, value0;
 812   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 813   ip_csum_t sum0;
 814   snat_session_t *s0;
 815
 816   if (!icmp_is_error_message (icmp0))
 817     {
 818       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 819       u16 icmp_id0 = echo0->identifier;
 820       key0.addr = ip0->dst_address;
 821       key0.port = icmp_id0;
 822       key0.protocol = SNAT_PROTOCOL_ICMP;
 823       key0.fib_index = sm->outside_fib_index;
 824       kv0.key = key0.as_u64;
 825
 826       if (sm->num_workers > 1)
 827         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 828       else
 829         ti = sm->num_workers;
 830
 831       /* Check if destination is in active sessions */
 832       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 833                                   &value0))
 834         {
 835           /* or static mappings */
 836           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 837             {
 838               new_dst_addr0 = sm0.addr.as_u32;
 839               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 840             }
 841         }
 842       else
 843         {
 844           si = value0.value;
 845
 846           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 847           new_dst_addr0 = s0->in2out.addr.as_u32;
 848           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 849           echo0->identifier = s0->in2out.port;
 850           sum0 = icmp0->checksum;
 851           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 852                                  icmp_echo_header_t, identifier);
 853           icmp0->checksum = ip_csum_fold (sum0);
 854         }
 855
 856       /* Destination is behind the same NAT, use internal address and port */
 857       if (new_dst_addr0)
 858         {
 859           old_dst_addr0 = ip0->dst_address.as_u32;
 860           ip0->dst_address.as_u32 = new_dst_addr0;
 861           sum0 = ip0->checksum;
 862           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 863                                  ip4_header_t, dst_address);
 864           ip0->checksum = ip_csum_fold (sum0);
 865         }
 866     }
 867
 868 }
 869
 870 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 871                                          vlib_buffer_t * b0,
 872                                          ip4_header_t * ip0,
 873                                          icmp46_header_t * icmp0,
 874                                          u32 sw_if_index0,
 875                                          u32 rx_fib_index0,
 876                                          vlib_node_runtime_t * node,
 877                                          u32 next0,
 878                                          f64 now,
 879                                          u32 thread_index,
 880                                          snat_session_t ** p_s0)
 881 {
 882   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 883                       next0, thread_index, p_s0, 0);
 884   snat_session_t * s0 = *p_s0;
 885   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 886     {
 887       /* Hairpinning */
 888       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
 889         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
 890       /* Accounting */
 891       s0->last_heard = now;
 892       s0->total_pkts++;
 893       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 894       /* Per-user LRU list maintenance */
 895       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 896                          s0->per_user_index);
 897       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 898                           s0->per_user_list_head_index,
 899                           s0->per_user_index);
 900     }
 901   return next0;
 902 }
 903 static inline void
 904 snat_hairpinning_unknown_proto (snat_main_t *sm,
 905                                 vlib_buffer_t * b,
 906                                 ip4_header_t * ip)
 907 {
 908   u32 old_addr, new_addr = 0, ti = 0;
 909   clib_bihash_kv_8_8_t kv, value;
 910   clib_bihash_kv_16_8_t s_kv, s_value;
 911   nat_ed_ses_key_t key;
 912   snat_session_key_t m_key;
 913   snat_static_mapping_t *m;
 914   ip_csum_t sum;
 915   snat_session_t *s;
 916
 917   old_addr = ip->dst_address.as_u32;
 918   key.l_addr.as_u32 = ip->dst_address.as_u32;
 919   key.r_addr.as_u32 = ip->src_address.as_u32;
 920   key.fib_index = sm->outside_fib_index;
 921   key.proto = ip->protocol;
 922   key.r_port = 0;
 923   key.l_port = 0;
 924   s_kv.key[0] = key.as_u64[0];
 925   s_kv.key[1] = key.as_u64[1];
 926   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 927     {
 928       m_key.addr = ip->dst_address;
 929       m_key.fib_index = sm->outside_fib_index;
 930       m_key.port = 0;
 931       m_key.protocol = 0;
 932       kv.key = m_key.as_u64;
 933       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 934         return;
 935
 936       m = pool_elt_at_index (sm->static_mappings, value.value);
 937       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
 938         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 939       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 940     }
 941   else
 942     {
 943       if (sm->num_workers > 1)
 944         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
 945       else
 946         ti = sm->num_workers;
 947
 948       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
 949       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
 950         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 951       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 952     }
 953   sum = ip->checksum;
 954   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 955   ip->checksum = ip_csum_fold (sum);
 956 }
 957
 958 static snat_session_t *
 959 snat_in2out_unknown_proto (snat_main_t *sm,
 960                            vlib_buffer_t * b,
 961                            ip4_header_t * ip,
 962                            u32 rx_fib_index,
 963                            u32 thread_index,
 964                            f64 now,
 965                            vlib_main_t * vm,
 966                            vlib_node_runtime_t * node)
 967 {
 968   clib_bihash_kv_8_8_t kv, value;
 969   clib_bihash_kv_16_8_t s_kv, s_value;
 970   snat_static_mapping_t *m;
 971   snat_session_key_t m_key;
 972   u32 old_addr, new_addr = 0;
 973   ip_csum_t sum;
 974   snat_user_t *u;
 975   dlist_elt_t *head, *elt;
 976   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 977   u32 elt_index, head_index, ses_index;
 978   snat_session_t * s;
 979   nat_ed_ses_key_t key;
 980   u32 address_index = ~0;
 981   int i;
 982   u8 is_sm = 0;
 983
 984   old_addr = ip->src_address.as_u32;
 985
 986   key.l_addr = ip->src_address;
 987   key.r_addr = ip->dst_address;
 988   key.fib_index = rx_fib_index;
 989   key.proto = ip->protocol;
 990   key.l_port = 0;
 991   key.l_port = 0;
 992   s_kv.key[0] = key.as_u64[0];
 993   s_kv.key[1] = key.as_u64[1];
 994
 995   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 996     {
 997       s = pool_elt_at_index (tsm->sessions, s_value.value);
 998       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 999     }
1000   else
1001     {
1002       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1003         {
1004           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1005           return 0;
1006         }
1007
1008       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1009                                   thread_index);
1010       if (!u)
1011         {
1012           clib_warning ("create NAT user failed");
1013           return 0;
1014         }
1015
1016       m_key.addr = ip->src_address;
1017       m_key.port = 0;
1018       m_key.protocol = 0;
1019       m_key.fib_index = rx_fib_index;
1020       kv.key = m_key.as_u64;
1021
1022       /* Try to find static mapping first */
1023       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1024         {
1025           m = pool_elt_at_index (sm->static_mappings, value.value);
1026           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1027           is_sm = 1;
1028           goto create_ses;
1029         }
1030       /* Fallback to 3-tuple key */
1031       else
1032         {
1033           /* Choose same out address as for TCP/UDP session to same destination */
1034           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1035             {
1036               head_index = u->sessions_per_user_list_head_index;
1037               head = pool_elt_at_index (tsm->list_pool, head_index);
1038               elt_index = head->next;
1039               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1040               ses_index = elt->value;
1041               while (ses_index != ~0)
1042                 {
1043                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1044                   elt_index = elt->next;
1045                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1046                   ses_index = elt->value;
1047
1048                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1049                     {
1050                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1051                       address_index = s->outside_address_index;
1052
1053                       key.fib_index = sm->outside_fib_index;
1054                       key.l_addr.as_u32 = new_addr;
1055                       s_kv.key[0] = key.as_u64[0];
1056                       s_kv.key[1] = key.as_u64[1];
1057                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1058                         break;
1059
1060                       goto create_ses;
1061                     }
1062                 }
1063             }
1064           key.fib_index = sm->outside_fib_index;
1065           for (i = 0; i < vec_len (sm->addresses); i++)
1066             {
1067               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1068               s_kv.key[0] = key.as_u64[0];
1069               s_kv.key[1] = key.as_u64[1];
1070               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1071                 {
1072                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1073                   address_index = i;
1074                   goto create_ses;
1075                 }
1076             }
1077           return 0;
1078         }
1079
1080 create_ses:
1081       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1082       if (!s)
1083         {
1084           clib_warning ("create NAT session failed");
1085           return 0;
1086         }
1087
1088       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1089       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1090       s->outside_address_index = address_index;
1091       s->out2in.addr.as_u32 = new_addr;
1092       s->out2in.fib_index = sm->outside_fib_index;
1093       s->in2out.addr.as_u32 = old_addr;
1094       s->in2out.fib_index = rx_fib_index;
1095       s->in2out.port = s->out2in.port = ip->protocol;
1096       if (is_sm)
1097         {
1098           u->nstaticsessions++;
1099           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1100         }
1101       else
1102         {
1103           u->nsessions++;
1104         }
1105
1106       /* Add to lookup tables */
1107       key.l_addr.as_u32 = old_addr;
1108       key.r_addr = ip->dst_address;
1109       key.proto = ip->protocol;
1110       key.fib_index = rx_fib_index;
1111       s_kv.key[0] = key.as_u64[0];
1112       s_kv.key[1] = key.as_u64[1];
1113       s_kv.value = s - tsm->sessions;
1114       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1115         clib_warning ("in2out key add failed");
1116
1117       key.l_addr.as_u32 = new_addr;
1118       key.fib_index = sm->outside_fib_index;
1119       s_kv.key[0] = key.as_u64[0];
1120       s_kv.key[1] = key.as_u64[1];
1121       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1122         clib_warning ("out2in key add failed");
1123   }
1124
1125   /* Update IP checksum */
1126   sum = ip->checksum;
1127   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1128   ip->checksum = ip_csum_fold (sum);
1129
1130   /* Accounting */
1131   s->last_heard = now;
1132   s->total_pkts++;
1133   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1134   /* Per-user LRU list maintenance */
1135   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1136   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1137                       s->per_user_index);
1138
1139   /* Hairpinning */
1140   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1141     snat_hairpinning_unknown_proto(sm, b, ip);
1142
1143   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1144     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1145
1146   return s;
1147 }
1148
1149 static snat_session_t *
1150 snat_in2out_lb (snat_main_t *sm,
1151                 vlib_buffer_t * b,
1152                 ip4_header_t * ip,
1153                 u32 rx_fib_index,
1154                 u32 thread_index,
1155                 f64 now,
1156                 vlib_main_t * vm,
1157                 vlib_node_runtime_t * node)
1158 {
1159   nat_ed_ses_key_t key;
1160   clib_bihash_kv_16_8_t s_kv, s_value;
1161   udp_header_t *udp = ip4_next_header (ip);
1162   tcp_header_t *tcp = (tcp_header_t *) udp;
1163   snat_session_t *s = 0;
1164   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1165   u32 old_addr, new_addr;
1166   u16 new_port, old_port;
1167   ip_csum_t sum;
1168   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1169   snat_session_key_t e_key, l_key;
1170   snat_user_t *u;
1171
1172   old_addr = ip->src_address.as_u32;
1173
1174   key.l_addr = ip->src_address;
1175   key.r_addr = ip->dst_address;
1176   key.fib_index = rx_fib_index;
1177   key.proto = ip->protocol;
1178   key.r_port = udp->dst_port;
1179   key.l_port = udp->src_port;
1180   s_kv.key[0] = key.as_u64[0];
1181   s_kv.key[1] = key.as_u64[1];
1182
1183   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1184     {
1185       s = pool_elt_at_index (tsm->sessions, s_value.value);
1186     }
1187   else
1188     {
1189       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1190         {
1191           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1192           return 0;
1193         }
1194
1195       l_key.addr = ip->src_address;
1196       l_key.port = udp->src_port;
1197       l_key.protocol = proto;
1198       l_key.fib_index = rx_fib_index;
1199       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1200         return 0;
1201
1202       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1203                                   thread_index);
1204       if (!u)
1205         {
1206           clib_warning ("create NAT user failed");
1207           return 0;
1208         }
1209
1210       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1211       if (!s)
1212         {
1213           clib_warning ("create NAT session failed");
1214           return 0;
1215         }
1216
1217       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1218       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1219       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1220       s->outside_address_index = ~0;
1221       s->in2out = l_key;
1222       s->out2in = e_key;
1223       u->nstaticsessions++;
1224
1225       /* Add to lookup tables */
1226       s_kv.value = s - tsm->sessions;
1227       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1228         clib_warning ("in2out-ed key add failed");
1229
1230       key.l_addr = e_key.addr;
1231       key.fib_index = e_key.fib_index;
1232       key.l_port = e_key.port;
1233       s_kv.key[0] = key.as_u64[0];
1234       s_kv.key[1] = key.as_u64[1];
1235       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1236         clib_warning ("out2in-ed key add failed");
1237     }
1238
1239   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1240
1241   /* Update IP checksum */
1242   sum = ip->checksum;
1243   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1244   if (is_twice_nat_session (s))
1245     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1246                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1247   ip->checksum = ip_csum_fold (sum);
1248
1249   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1250     {
1251       old_port = tcp->src_port;
1252       tcp->src_port = s->out2in.port;
1253       new_port = tcp->src_port;
1254
1255       sum = tcp->checksum;
1256       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1257       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1258       if (is_twice_nat_session (s))
1259         {
1260           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1261                                 s->ext_host_addr.as_u32, ip4_header_t,
1262                                 dst_address);
1263           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1264                                 ip4_header_t, length);
1265           tcp->dst_port = s->ext_host_port;
1266           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1267         }
1268       tcp->checksum = ip_csum_fold(sum);
1269     }
1270   else
1271     {
1272       udp->src_port = s->out2in.port;
1273       if (is_twice_nat_session (s))
1274         {
1275           udp->dst_port = s->ext_host_port;
1276           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1277         }
1278       udp->checksum = 0;
1279     }
1280
1281   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1282     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1283
1284   /* Accounting */
1285   s->last_heard = now;
1286   s->total_pkts++;
1287   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1288   /* Per-user LRU list maintenance */
1289   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1290   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1291                       s->per_user_index);
1292   return s;
1293 }
1294
1295 static inline uword
1296 snat_in2out_node_fn_inline (vlib_main_t * vm,
1297                             vlib_node_runtime_t * node,
1298                             vlib_frame_t * frame, int is_slow_path,
1299                             int is_output_feature)
1300 {
1301   u32 n_left_from, * from, * to_next;
1302   snat_in2out_next_t next_index;
1303   u32 pkts_processed = 0;
1304   snat_main_t * sm = &snat_main;
1305   f64 now = vlib_time_now (vm);
1306   u32 stats_node_index;
1307   u32 thread_index = vlib_get_thread_index ();
1308
1309   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1310     snat_in2out_node.index;
1311
1312   from = vlib_frame_vector_args (frame);
1313   n_left_from = frame->n_vectors;
1314   next_index = node->cached_next_index;
1315
1316   while (n_left_from > 0)
1317     {
1318       u32 n_left_to_next;
1319
1320       vlib_get_next_frame (vm, node, next_index,
1321                            to_next, n_left_to_next);
1322
1323       while (n_left_from >= 4 && n_left_to_next >= 2)
1324         {
1325           u32 bi0, bi1;
1326           vlib_buffer_t * b0, * b1;
1327           u32 next0, next1;
1328           u32 sw_if_index0, sw_if_index1;
1329           ip4_header_t * ip0, * ip1;
1330           ip_csum_t sum0, sum1;
1331           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1332           u16 old_port0, new_port0, old_port1, new_port1;
1333           udp_header_t * udp0, * udp1;
1334           tcp_header_t * tcp0, * tcp1;
1335           icmp46_header_t * icmp0, * icmp1;
1336           snat_session_key_t key0, key1;
1337           u32 rx_fib_index0, rx_fib_index1;
1338           u32 proto0, proto1;
1339           snat_session_t * s0 = 0, * s1 = 0;
1340           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1341           u32 iph_offset0 = 0, iph_offset1 = 0;
1342
1343           /* Prefetch next iteration. */
1344           {
1345             vlib_buffer_t * p2, * p3;
1346
1347             p2 = vlib_get_buffer (vm, from[2]);
1348             p3 = vlib_get_buffer (vm, from[3]);
1349
1350             vlib_prefetch_buffer_header (p2, LOAD);
1351             vlib_prefetch_buffer_header (p3, LOAD);
1352
1353             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1354             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1355           }
1356
1357           /* speculatively enqueue b0 and b1 to the current next frame */
1358           to_next[0] = bi0 = from[0];
1359           to_next[1] = bi1 = from[1];
1360           from += 2;
1361           to_next += 2;
1362           n_left_from -= 2;
1363           n_left_to_next -= 2;
1364
1365           b0 = vlib_get_buffer (vm, bi0);
1366           b1 = vlib_get_buffer (vm, bi1);
1367
1368           if (is_output_feature)
1369             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1370
1371           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1372                  iph_offset0);
1373
1374           udp0 = ip4_next_header (ip0);
1375           tcp0 = (tcp_header_t *) udp0;
1376           icmp0 = (icmp46_header_t *) udp0;
1377
1378           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1379           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1380                                    sw_if_index0);
1381
1382           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1383
1384           if (PREDICT_FALSE(ip0->ttl == 1))
1385             {
1386               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1387               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1388                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1389                                            0);
1390               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1391               goto trace00;
1392             }
1393
1394           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1395
1396           /* Next configured feature, probably ip4-lookup */
1397           if (is_slow_path)
1398             {
1399               if (PREDICT_FALSE (proto0 == ~0))
1400                 {
1401                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1402                                                   thread_index, now, vm, node);
1403                   if (!s0)
1404                     next0 = SNAT_IN2OUT_NEXT_DROP;
1405                   goto trace00;
1406                 }
1407
1408               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1409                 {
1410                   next0 = icmp_in2out_slow_path
1411                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1412                      node, next0, now, thread_index, &s0);
1413                   goto trace00;
1414                 }
1415             }
1416           else
1417             {
1418               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1419                 {
1420                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1421                   goto trace00;
1422                 }
1423
1424               if (ip4_is_fragment (ip0))
1425                 {
1426                   next0 = SNAT_IN2OUT_NEXT_REASS;
1427                   goto trace00;
1428                 }
1429             }
1430
1431           key0.addr = ip0->src_address;
1432           key0.port = udp0->src_port;
1433           key0.protocol = proto0;
1434           key0.fib_index = rx_fib_index0;
1435
1436           kv0.key = key0.as_u64;
1437
1438           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1439               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1440             {
1441               if (is_slow_path)
1442                 {
1443                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1444                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1445                     goto trace00;
1446
1447                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1448                                      &s0, node, next0, thread_index);
1449                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1450                     goto trace00;
1451                 }
1452               else
1453                 {
1454                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1455                   goto trace00;
1456                 }
1457             }
1458           else
1459             {
1460               if (PREDICT_FALSE (value0.value == ~0ULL))
1461                 {
1462                   if (is_slow_path)
1463                     {
1464                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1465                                           thread_index, now, vm, node);
1466                       if (!s0)
1467                         next0 = SNAT_IN2OUT_NEXT_DROP;
1468                       goto trace00;
1469                     }
1470                   else
1471                     {
1472                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1473                       goto trace00;
1474                     }
1475                 }
1476               else
1477                 {
1478                   s0 = pool_elt_at_index (
1479                     sm->per_thread_data[thread_index].sessions,
1480                     value0.value);
1481                 }
1482             }
1483
1484           b0->flags |= VNET_BUFFER_F_IS_NATED;
1485
1486           old_addr0 = ip0->src_address.as_u32;
1487           ip0->src_address = s0->out2in.addr;
1488           new_addr0 = ip0->src_address.as_u32;
1489           if (!is_output_feature)
1490             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1491
1492           sum0 = ip0->checksum;
1493           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1494                                  ip4_header_t,
1495                                  src_address /* changed member */);
1496           ip0->checksum = ip_csum_fold (sum0);
1497
1498           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1499             {
1500               old_port0 = tcp0->src_port;
1501               tcp0->src_port = s0->out2in.port;
1502               new_port0 = tcp0->src_port;
1503
1504               sum0 = tcp0->checksum;
1505               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1506                                      ip4_header_t,
1507                                      dst_address /* changed member */);
1508               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1509                                      ip4_header_t /* cheat */,
1510                                      length /* changed member */);
1511               tcp0->checksum = ip_csum_fold(sum0);
1512             }
1513           else
1514             {
1515               old_port0 = udp0->src_port;
1516               udp0->src_port = s0->out2in.port;
1517               udp0->checksum = 0;
1518             }
1519
1520           /* Accounting */
1521           s0->last_heard = now;
1522           s0->total_pkts++;
1523           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1524           /* Per-user LRU list maintenance */
1525           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1526                              s0->per_user_index);
1527           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1528                               s0->per_user_list_head_index,
1529                               s0->per_user_index);
1530         trace00:
1531
1532           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1533                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1534             {
1535               snat_in2out_trace_t *t =
1536                  vlib_add_trace (vm, node, b0, sizeof (*t));
1537               t->is_slow_path = is_slow_path;
1538               t->sw_if_index = sw_if_index0;
1539               t->next_index = next0;
1540                   t->session_index = ~0;
1541               if (s0)
1542                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1543             }
1544
1545           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1546
1547           if (is_output_feature)
1548             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1549
1550           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1551                  iph_offset1);
1552
1553           udp1 = ip4_next_header (ip1);
1554           tcp1 = (tcp_header_t *) udp1;
1555           icmp1 = (icmp46_header_t *) udp1;
1556
1557           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1558           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1559                                    sw_if_index1);
1560
1561           if (PREDICT_FALSE(ip1->ttl == 1))
1562             {
1563               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1564               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1565                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1566                                            0);
1567               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1568               goto trace01;
1569             }
1570
1571           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1572
1573           /* Next configured feature, probably ip4-lookup */
1574           if (is_slow_path)
1575             {
1576               if (PREDICT_FALSE (proto1 == ~0))
1577                 {
1578                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1579                                                   thread_index, now, vm, node);
1580                   if (!s1)
1581                     next1 = SNAT_IN2OUT_NEXT_DROP;
1582                   goto trace01;
1583                 }
1584
1585               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1586                 {
1587                   next1 = icmp_in2out_slow_path
1588                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1589                      next1, now, thread_index, &s1);
1590                   goto trace01;
1591                 }
1592             }
1593           else
1594             {
1595               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1596                 {
1597                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1598                   goto trace01;
1599                 }
1600
1601               if (ip4_is_fragment (ip1))
1602                 {
1603                   next1 = SNAT_IN2OUT_NEXT_REASS;
1604                   goto trace01;
1605                 }
1606             }
1607
1608           b1->flags |= VNET_BUFFER_F_IS_NATED;
1609
1610           key1.addr = ip1->src_address;
1611           key1.port = udp1->src_port;
1612           key1.protocol = proto1;
1613           key1.fib_index = rx_fib_index1;
1614
1615           kv1.key = key1.as_u64;
1616
1617             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1618                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1619             {
1620               if (is_slow_path)
1621                 {
1622                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1623                       ip1, proto1, rx_fib_index1, thread_index)) && !is_output_feature)
1624                     goto trace01;
1625
1626                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1627                                      &s1, node, next1, thread_index);
1628                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1629                     goto trace01;
1630                 }
1631               else
1632                 {
1633                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1634                   goto trace01;
1635                 }
1636             }
1637           else
1638             {
1639               if (PREDICT_FALSE (value1.value == ~0ULL))
1640                 {
1641                   if (is_slow_path)
1642                     {
1643                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1644                                           thread_index, now, vm, node);
1645                       if (!s1)
1646                         next1 = SNAT_IN2OUT_NEXT_DROP;
1647                       goto trace01;
1648                     }
1649                   else
1650                     {
1651                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1652                       goto trace01;
1653                     }
1654                 }
1655               else
1656                 {
1657                   s1 = pool_elt_at_index (
1658                     sm->per_thread_data[thread_index].sessions,
1659                     value1.value);
1660                 }
1661             }
1662
1663           old_addr1 = ip1->src_address.as_u32;
1664           ip1->src_address = s1->out2in.addr;
1665           new_addr1 = ip1->src_address.as_u32;
1666           if (!is_output_feature)
1667             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1668
1669           sum1 = ip1->checksum;
1670           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1671                                  ip4_header_t,
1672                                  src_address /* changed member */);
1673           ip1->checksum = ip_csum_fold (sum1);
1674
1675           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1676             {
1677               old_port1 = tcp1->src_port;
1678               tcp1->src_port = s1->out2in.port;
1679               new_port1 = tcp1->src_port;
1680
1681               sum1 = tcp1->checksum;
1682               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1683                                      ip4_header_t,
1684                                      dst_address /* changed member */);
1685               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1686                                      ip4_header_t /* cheat */,
1687                                      length /* changed member */);
1688               tcp1->checksum = ip_csum_fold(sum1);
1689             }
1690           else
1691             {
1692               old_port1 = udp1->src_port;
1693               udp1->src_port = s1->out2in.port;
1694               udp1->checksum = 0;
1695             }
1696
1697           /* Accounting */
1698           s1->last_heard = now;
1699           s1->total_pkts++;
1700           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1701           /* Per-user LRU list maintenance */
1702           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1703                              s1->per_user_index);
1704           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1705                               s1->per_user_list_head_index,
1706                               s1->per_user_index);
1707         trace01:
1708
1709           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1710                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1711             {
1712               snat_in2out_trace_t *t =
1713                  vlib_add_trace (vm, node, b1, sizeof (*t));
1714               t->sw_if_index = sw_if_index1;
1715               t->next_index = next1;
1716               t->session_index = ~0;
1717               if (s1)
1718                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1719             }
1720
1721           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1722
1723           /* verify speculative enqueues, maybe switch current next frame */
1724           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1725                                            to_next, n_left_to_next,
1726                                            bi0, bi1, next0, next1);
1727         }
1728
1729       while (n_left_from > 0 && n_left_to_next > 0)
1730         {
1731           u32 bi0;
1732           vlib_buffer_t * b0;
1733           u32 next0;
1734           u32 sw_if_index0;
1735           ip4_header_t * ip0;
1736           ip_csum_t sum0;
1737           u32 new_addr0, old_addr0;
1738           u16 old_port0, new_port0;
1739           udp_header_t * udp0;
1740           tcp_header_t * tcp0;
1741           icmp46_header_t * icmp0;
1742           snat_session_key_t key0;
1743           u32 rx_fib_index0;
1744           u32 proto0;
1745           snat_session_t * s0 = 0;
1746           clib_bihash_kv_8_8_t kv0, value0;
1747           u32 iph_offset0 = 0;
1748
1749           /* speculatively enqueue b0 to the current next frame */
1750           bi0 = from[0];
1751           to_next[0] = bi0;
1752           from += 1;
1753           to_next += 1;
1754           n_left_from -= 1;
1755           n_left_to_next -= 1;
1756
1757           b0 = vlib_get_buffer (vm, bi0);
1758           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1759
1760           if (is_output_feature)
1761             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1762
1763           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1764                  iph_offset0);
1765
1766           udp0 = ip4_next_header (ip0);
1767           tcp0 = (tcp_header_t *) udp0;
1768           icmp0 = (icmp46_header_t *) udp0;
1769
1770           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1771           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1772                                    sw_if_index0);
1773
1774           if (PREDICT_FALSE(ip0->ttl == 1))
1775             {
1776               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1777               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1778                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1779                                            0);
1780               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1781               goto trace0;
1782             }
1783
1784           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1785
1786           /* Next configured feature, probably ip4-lookup */
1787           if (is_slow_path)
1788             {
1789               if (PREDICT_FALSE (proto0 == ~0))
1790                 {
1791                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1792                                                   thread_index, now, vm, node);
1793                   if (!s0)
1794                     next0 = SNAT_IN2OUT_NEXT_DROP;
1795                   goto trace0;
1796                 }
1797
1798               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1799                 {
1800                   next0 = icmp_in2out_slow_path
1801                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1802                      next0, now, thread_index, &s0);
1803                   goto trace0;
1804                 }
1805             }
1806           else
1807             {
1808               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1809                 {
1810                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1811                   goto trace0;
1812                 }
1813
1814               if (ip4_is_fragment (ip0))
1815                 {
1816                   next0 = SNAT_IN2OUT_NEXT_REASS;
1817                   goto trace0;
1818                 }
1819             }
1820
1821           key0.addr = ip0->src_address;
1822           key0.port = udp0->src_port;
1823           key0.protocol = proto0;
1824           key0.fib_index = rx_fib_index0;
1825
1826           kv0.key = key0.as_u64;
1827
1828           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1829                                       &kv0, &value0))
1830             {
1831               if (is_slow_path)
1832                 {
1833                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1834                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1835                     goto trace0;
1836
1837                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1838                                      &s0, node, next0, thread_index);
1839
1840                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1841                     goto trace0;
1842                 }
1843               else
1844                 {
1845                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1846                   goto trace0;
1847                 }
1848             }
1849           else
1850             {
1851               if (PREDICT_FALSE (value0.value == ~0ULL))
1852                 {
1853                   if (is_slow_path)
1854                     {
1855                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1856                                           thread_index, now, vm, node);
1857                       if (!s0)
1858                         next0 = SNAT_IN2OUT_NEXT_DROP;
1859                       goto trace0;
1860                     }
1861                   else
1862                     {
1863                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1864                       goto trace0;
1865                     }
1866                 }
1867               else
1868                 {
1869                   s0 = pool_elt_at_index (
1870                     sm->per_thread_data[thread_index].sessions,
1871                     value0.value);
1872                 }
1873             }
1874
1875           b0->flags |= VNET_BUFFER_F_IS_NATED;
1876
1877           old_addr0 = ip0->src_address.as_u32;
1878           ip0->src_address = s0->out2in.addr;
1879           new_addr0 = ip0->src_address.as_u32;
1880           if (!is_output_feature)
1881             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1882
1883           sum0 = ip0->checksum;
1884           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1885                                  ip4_header_t,
1886                                  src_address /* changed member */);
1887           ip0->checksum = ip_csum_fold (sum0);
1888
1889           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1890             {
1891               old_port0 = tcp0->src_port;
1892               tcp0->src_port = s0->out2in.port;
1893               new_port0 = tcp0->src_port;
1894
1895               sum0 = tcp0->checksum;
1896               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1897                                      ip4_header_t,
1898                                      dst_address /* changed member */);
1899               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1900                                      ip4_header_t /* cheat */,
1901                                      length /* changed member */);
1902               tcp0->checksum = ip_csum_fold(sum0);
1903             }
1904           else
1905             {
1906               old_port0 = udp0->src_port;
1907               udp0->src_port = s0->out2in.port;
1908               udp0->checksum = 0;
1909             }
1910
1911           /* Accounting */
1912           s0->last_heard = now;
1913           s0->total_pkts++;
1914           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1915           /* Per-user LRU list maintenance */
1916           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1917                              s0->per_user_index);
1918           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1919                               s0->per_user_list_head_index,
1920                               s0->per_user_index);
1921
1922         trace0:
1923           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1924                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1925             {
1926               snat_in2out_trace_t *t =
1927                  vlib_add_trace (vm, node, b0, sizeof (*t));
1928               t->is_slow_path = is_slow_path;
1929               t->sw_if_index = sw_if_index0;
1930               t->next_index = next0;
1931                   t->session_index = ~0;
1932               if (s0)
1933                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1934             }
1935
1936           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1937
1938           /* verify speculative enqueue, maybe switch current next frame */
1939           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1940                                            to_next, n_left_to_next,
1941                                            bi0, next0);
1942         }
1943
1944       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1945     }
1946
1947   vlib_node_increment_counter (vm, stats_node_index,
1948                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1949                                pkts_processed);
1950   return frame->n_vectors;
1951 }
1952
1953 static uword
1954 snat_in2out_fast_path_fn (vlib_main_t * vm,
1955                           vlib_node_runtime_t * node,
1956                           vlib_frame_t * frame)
1957 {
1958   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1959 }
1960
1961 VLIB_REGISTER_NODE (snat_in2out_node) = {
1962   .function = snat_in2out_fast_path_fn,
1963   .name = "nat44-in2out",
1964   .vector_size = sizeof (u32),
1965   .format_trace = format_snat_in2out_trace,
1966   .type = VLIB_NODE_TYPE_INTERNAL,
1967
1968   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1969   .error_strings = snat_in2out_error_strings,
1970
1971   .runtime_data_bytes = sizeof (snat_runtime_t),
1972
1973   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1974
1975   /* edit / add dispositions here */
1976   .next_nodes = {
1977     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1978     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1979     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1980     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1981     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1982   },
1983 };
1984
1985 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1986
1987 static uword
1988 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1989                                  vlib_node_runtime_t * node,
1990                                  vlib_frame_t * frame)
1991 {
1992   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1993 }
1994
1995 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1996   .function = snat_in2out_output_fast_path_fn,
1997   .name = "nat44-in2out-output",
1998   .vector_size = sizeof (u32),
1999   .format_trace = format_snat_in2out_trace,
2000   .type = VLIB_NODE_TYPE_INTERNAL,
2001
2002   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2003   .error_strings = snat_in2out_error_strings,
2004
2005   .runtime_data_bytes = sizeof (snat_runtime_t),
2006
2007   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2008
2009   /* edit / add dispositions here */
2010   .next_nodes = {
2011     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2012     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2013     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2014     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2015     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2016   },
2017 };
2018
2019 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2020                               snat_in2out_output_fast_path_fn);
2021
2022 static uword
2023 snat_in2out_slow_path_fn (vlib_main_t * vm,
2024                           vlib_node_runtime_t * node,
2025                           vlib_frame_t * frame)
2026 {
2027   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2028 }
2029
2030 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2031   .function = snat_in2out_slow_path_fn,
2032   .name = "nat44-in2out-slowpath",
2033   .vector_size = sizeof (u32),
2034   .format_trace = format_snat_in2out_trace,
2035   .type = VLIB_NODE_TYPE_INTERNAL,
2036
2037   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2038   .error_strings = snat_in2out_error_strings,
2039
2040   .runtime_data_bytes = sizeof (snat_runtime_t),
2041
2042   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2043
2044   /* edit / add dispositions here */
2045   .next_nodes = {
2046     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2047     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2048     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2049     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2050     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2051   },
2052 };
2053
2054 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2055                               snat_in2out_slow_path_fn);
2056
2057 static uword
2058 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2059                                  vlib_node_runtime_t * node,
2060                                  vlib_frame_t * frame)
2061 {
2062   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2063 }
2064
2065 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2066   .function = snat_in2out_output_slow_path_fn,
2067   .name = "nat44-in2out-output-slowpath",
2068   .vector_size = sizeof (u32),
2069   .format_trace = format_snat_in2out_trace,
2070   .type = VLIB_NODE_TYPE_INTERNAL,
2071
2072   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2073   .error_strings = snat_in2out_error_strings,
2074
2075   .runtime_data_bytes = sizeof (snat_runtime_t),
2076
2077   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2078
2079   /* edit / add dispositions here */
2080   .next_nodes = {
2081     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2082     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2083     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2084     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2085     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2086   },
2087 };
2088
2089 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2090                               snat_in2out_output_slow_path_fn);
2091
2092 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2093
2094 static uword
2095 nat44_hairpinning_fn (vlib_main_t * vm,
2096                       vlib_node_runtime_t * node,
2097                       vlib_frame_t * frame)
2098 {
2099   u32 n_left_from, * from, * to_next;
2100   snat_in2out_next_t next_index;
2101   u32 pkts_processed = 0;
2102   snat_main_t * sm = &snat_main;
2103   vnet_feature_main_t *fm = &feature_main;
2104   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2105   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2106
2107   from = vlib_frame_vector_args (frame);
2108   n_left_from = frame->n_vectors;
2109   next_index = node->cached_next_index;
2110
2111   while (n_left_from > 0)
2112     {
2113       u32 n_left_to_next;
2114
2115       vlib_get_next_frame (vm, node, next_index,
2116                            to_next, n_left_to_next);
2117
2118       while (n_left_from > 0 && n_left_to_next > 0)
2119         {
2120           u32 bi0;
2121           vlib_buffer_t * b0;
2122           u32 next0;
2123           ip4_header_t * ip0;
2124           u32 proto0;
2125           udp_header_t * udp0;
2126           tcp_header_t * tcp0;
2127
2128           /* speculatively enqueue b0 to the current next frame */
2129           bi0 = from[0];
2130           to_next[0] = bi0;
2131           from += 1;
2132           to_next += 1;
2133           n_left_from -= 1;
2134           n_left_to_next -= 1;
2135
2136           b0 = vlib_get_buffer (vm, bi0);
2137           ip0 = vlib_buffer_get_current (b0);
2138           udp0 = ip4_next_header (ip0);
2139           tcp0 = (tcp_header_t *) udp0;
2140
2141           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2142
2143           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2144                                 &next0, 0);
2145
2146           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2147             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2148
2149           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2150
2151           /* verify speculative enqueue, maybe switch current next frame */
2152           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2153                                            to_next, n_left_to_next,
2154                                            bi0, next0);
2155          }
2156
2157       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2158     }
2159
2160   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2161                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2162                                pkts_processed);
2163   return frame->n_vectors;
2164 }
2165
2166 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2167   .function = nat44_hairpinning_fn,
2168   .name = "nat44-hairpinning",
2169   .vector_size = sizeof (u32),
2170   .type = VLIB_NODE_TYPE_INTERNAL,
2171   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2172   .error_strings = snat_in2out_error_strings,
2173   .n_next_nodes = 2,
2174   .next_nodes = {
2175     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2176     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2177   },
2178 };
2179
2180 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2181                               nat44_hairpinning_fn);
2182
2183 static inline void
2184 nat44_reass_hairpinning (snat_main_t *sm,
2185                          vlib_buffer_t * b0,
2186                          ip4_header_t * ip0,
2187                          u16 sport,
2188                          u16 dport,
2189                          u32 proto0)
2190 {
2191   snat_session_key_t key0, sm0;
2192   snat_session_t * s0;
2193   clib_bihash_kv_8_8_t kv0, value0;
2194   ip_csum_t sum0;
2195   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2196   u16 new_dst_port0, old_dst_port0;
2197   udp_header_t * udp0;
2198   tcp_header_t * tcp0;
2199
2200   key0.addr = ip0->dst_address;
2201   key0.port = dport;
2202   key0.protocol = proto0;
2203   key0.fib_index = sm->outside_fib_index;
2204   kv0.key = key0.as_u64;
2205
2206   udp0 = ip4_next_header (ip0);
2207
2208   /* Check if destination is static mappings */
2209   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2210     {
2211       new_dst_addr0 = sm0.addr.as_u32;
2212       new_dst_port0 = sm0.port;
2213       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2214     }
2215   /* or active sessions */
2216   else
2217     {
2218       if (sm->num_workers > 1)
2219         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2220       else
2221         ti = sm->num_workers;
2222
2223       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2224         {
2225           si = value0.value;
2226           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2227           new_dst_addr0 = s0->in2out.addr.as_u32;
2228           new_dst_port0 = s0->in2out.port;
2229           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2230         }
2231     }
2232
2233   /* Destination is behind the same NAT, use internal address and port */
2234   if (new_dst_addr0)
2235     {
2236       old_dst_addr0 = ip0->dst_address.as_u32;
2237       ip0->dst_address.as_u32 = new_dst_addr0;
2238       sum0 = ip0->checksum;
2239       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2240                              ip4_header_t, dst_address);
2241       ip0->checksum = ip_csum_fold (sum0);
2242
2243       old_dst_port0 = dport;
2244       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2245                        ip4_is_first_fragment (ip0)))
2246         {
2247           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2248             {
2249               tcp0 = ip4_next_header (ip0);
2250               tcp0->dst = new_dst_port0;
2251               sum0 = tcp0->checksum;
2252               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2253                                      ip4_header_t, dst_address);
2254               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2255                                      ip4_header_t /* cheat */, length);
2256               tcp0->checksum = ip_csum_fold(sum0);
2257             }
2258           else
2259             {
2260               udp0->dst_port = new_dst_port0;
2261               udp0->checksum = 0;
2262             }
2263         }
2264       else
2265         {
2266           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2267             {
2268               tcp0 = ip4_next_header (ip0);
2269               sum0 = tcp0->checksum;
2270               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2271                                      ip4_header_t, dst_address);
2272               tcp0->checksum = ip_csum_fold(sum0);
2273             }
2274         }
2275     }
2276 }
2277
2278 static uword
2279 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2280                             vlib_node_runtime_t * node,
2281                             vlib_frame_t * frame)
2282 {
2283   u32 n_left_from, *from, *to_next;
2284   snat_in2out_next_t next_index;
2285   u32 pkts_processed = 0;
2286   snat_main_t *sm = &snat_main;
2287   f64 now = vlib_time_now (vm);
2288   u32 thread_index = vlib_get_thread_index ();
2289   snat_main_per_thread_data_t *per_thread_data =
2290     &sm->per_thread_data[thread_index];
2291   u32 *fragments_to_drop = 0;
2292   u32 *fragments_to_loopback = 0;
2293
2294   from = vlib_frame_vector_args (frame);
2295   n_left_from = frame->n_vectors;
2296   next_index = node->cached_next_index;
2297
2298   while (n_left_from > 0)
2299     {
2300       u32 n_left_to_next;
2301
2302       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2303
2304       while (n_left_from > 0 && n_left_to_next > 0)
2305        {
2306           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2307           vlib_buffer_t *b0;
2308           u32 next0;
2309           u8 cached0 = 0;
2310           ip4_header_t *ip0;
2311           nat_reass_ip4_t *reass0;
2312           udp_header_t * udp0;
2313           tcp_header_t * tcp0;
2314           snat_session_key_t key0;
2315           clib_bihash_kv_8_8_t kv0, value0;
2316           snat_session_t * s0 = 0;
2317           u16 old_port0, new_port0;
2318           ip_csum_t sum0;
2319
2320           /* speculatively enqueue b0 to the current next frame */
2321           bi0 = from[0];
2322           to_next[0] = bi0;
2323           from += 1;
2324           to_next += 1;
2325           n_left_from -= 1;
2326           n_left_to_next -= 1;
2327
2328           b0 = vlib_get_buffer (vm, bi0);
2329           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2330
2331           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2332           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2333                                                                sw_if_index0);
2334
2335           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2336             {
2337               next0 = SNAT_IN2OUT_NEXT_DROP;
2338               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2339               goto trace0;
2340             }
2341
2342           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2343           udp0 = ip4_next_header (ip0);
2344           tcp0 = (tcp_header_t *) udp0;
2345           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2346
2347           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2348                                                  ip0->dst_address,
2349                                                  ip0->fragment_id,
2350                                                  ip0->protocol,
2351                                                  1,
2352                                                  &fragments_to_drop);
2353
2354           if (PREDICT_FALSE (!reass0))
2355             {
2356               next0 = SNAT_IN2OUT_NEXT_DROP;
2357               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2358               goto trace0;
2359             }
2360
2361           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2362             {
2363               key0.addr = ip0->src_address;
2364               key0.port = udp0->src_port;
2365               key0.protocol = proto0;
2366               key0.fib_index = rx_fib_index0;
2367               kv0.key = key0.as_u64;
2368
2369               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2370                 {
2371                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2372                       ip0, proto0, rx_fib_index0, thread_index)))
2373                     goto trace0;
2374
2375                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2376                                      &s0, node, next0, thread_index);
2377
2378                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2379                     goto trace0;
2380
2381                   reass0->sess_index = s0 - per_thread_data->sessions;
2382                 }
2383               else
2384                 {
2385                   s0 = pool_elt_at_index (per_thread_data->sessions,
2386                                           value0.value);
2387                   reass0->sess_index = value0.value;
2388                 }
2389               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2390             }
2391           else
2392             {
2393               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2394                 {
2395                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2396                     {
2397                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2398                       next0 = SNAT_IN2OUT_NEXT_DROP;
2399                       goto trace0;
2400                     }
2401                   cached0 = 1;
2402                   goto trace0;
2403                 }
2404               s0 = pool_elt_at_index (per_thread_data->sessions,
2405                                       reass0->sess_index);
2406             }
2407
2408           old_addr0 = ip0->src_address.as_u32;
2409           ip0->src_address = s0->out2in.addr;
2410           new_addr0 = ip0->src_address.as_u32;
2411           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2412
2413           sum0 = ip0->checksum;
2414           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2415                                  ip4_header_t,
2416                                  src_address /* changed member */);
2417           ip0->checksum = ip_csum_fold (sum0);
2418
2419           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2420             {
2421               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2422                 {
2423                   old_port0 = tcp0->src_port;
2424                   tcp0->src_port = s0->out2in.port;
2425                   new_port0 = tcp0->src_port;
2426
2427                   sum0 = tcp0->checksum;
2428                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2429                                          ip4_header_t,
2430                                          dst_address /* changed member */);
2431                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2432                                          ip4_header_t /* cheat */,
2433                                          length /* changed member */);
2434                   tcp0->checksum = ip_csum_fold(sum0);
2435                 }
2436               else
2437                 {
2438                   old_port0 = udp0->src_port;
2439                   udp0->src_port = s0->out2in.port;
2440                   udp0->checksum = 0;
2441                 }
2442             }
2443
2444           /* Hairpinning */
2445           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2446                                    s0->ext_host_port, proto0);
2447
2448           /* Accounting */
2449           s0->last_heard = now;
2450           s0->total_pkts++;
2451           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2452           /* Per-user LRU list maintenance */
2453           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2454                              s0->per_user_index);
2455           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2456                               s0->per_user_list_head_index,
2457                               s0->per_user_index);
2458
2459         trace0:
2460           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2461                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2462             {
2463               nat44_in2out_reass_trace_t *t =
2464                  vlib_add_trace (vm, node, b0, sizeof (*t));
2465               t->cached = cached0;
2466               t->sw_if_index = sw_if_index0;
2467               t->next_index = next0;
2468             }
2469
2470           if (cached0)
2471             {
2472               n_left_to_next++;
2473               to_next--;
2474             }
2475           else
2476             {
2477               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2478
2479               /* verify speculative enqueue, maybe switch current next frame */
2480               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2481                                                to_next, n_left_to_next,
2482                                                bi0, next0);
2483             }
2484
2485           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2486             {
2487               from = vlib_frame_vector_args (frame);
2488               u32 len = vec_len (fragments_to_loopback);
2489               if (len <= VLIB_FRAME_SIZE)
2490                 {
2491                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2492                   n_left_from = len;
2493                   vec_reset_length (fragments_to_loopback);
2494                 }
2495               else
2496                 {
2497                   clib_memcpy (from,
2498                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2499                                sizeof (u32) * VLIB_FRAME_SIZE);
2500                   n_left_from = VLIB_FRAME_SIZE;
2501                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2502                 }
2503             }
2504        }
2505
2506       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2507     }
2508
2509   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2510                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2511                                pkts_processed);
2512
2513   nat_send_all_to_node (vm, fragments_to_drop, node,
2514                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2515                         SNAT_IN2OUT_NEXT_DROP);
2516
2517   vec_free (fragments_to_drop);
2518   vec_free (fragments_to_loopback);
2519   return frame->n_vectors;
2520 }
2521
2522 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2523   .function = nat44_in2out_reass_node_fn,
2524   .name = "nat44-in2out-reass",
2525   .vector_size = sizeof (u32),
2526   .format_trace = format_nat44_in2out_reass_trace,
2527   .type = VLIB_NODE_TYPE_INTERNAL,
2528
2529   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2530   .error_strings = snat_in2out_error_strings,
2531
2532   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2533   .next_nodes = {
2534     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2535     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2536     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2537     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2538     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2539   },
2540 };
2541
2542 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2543                               nat44_in2out_reass_node_fn);
2544
2545 /**************************/
2546 /*** deterministic mode ***/
2547 /**************************/
2548 static uword
2549 snat_det_in2out_node_fn (vlib_main_t * vm,
2550                          vlib_node_runtime_t * node,
2551                          vlib_frame_t * frame)
2552 {
2553   u32 n_left_from, * from, * to_next;
2554   snat_in2out_next_t next_index;
2555   u32 pkts_processed = 0;
2556   snat_main_t * sm = &snat_main;
2557   u32 now = (u32) vlib_time_now (vm);
2558   u32 thread_index = vlib_get_thread_index ();
2559
2560   from = vlib_frame_vector_args (frame);
2561   n_left_from = frame->n_vectors;
2562   next_index = node->cached_next_index;
2563
2564   while (n_left_from > 0)
2565     {
2566       u32 n_left_to_next;
2567
2568       vlib_get_next_frame (vm, node, next_index,
2569                            to_next, n_left_to_next);
2570
2571       while (n_left_from >= 4 && n_left_to_next >= 2)
2572         {
2573           u32 bi0, bi1;
2574           vlib_buffer_t * b0, * b1;
2575           u32 next0, next1;
2576           u32 sw_if_index0, sw_if_index1;
2577           ip4_header_t * ip0, * ip1;
2578           ip_csum_t sum0, sum1;
2579           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2580           u16 old_port0, new_port0, lo_port0, i0;
2581           u16 old_port1, new_port1, lo_port1, i1;
2582           udp_header_t * udp0, * udp1;
2583           tcp_header_t * tcp0, * tcp1;
2584           u32 proto0, proto1;
2585           snat_det_out_key_t key0, key1;
2586           snat_det_map_t * dm0, * dm1;
2587           snat_det_session_t * ses0 = 0, * ses1 = 0;
2588           u32 rx_fib_index0, rx_fib_index1;
2589           icmp46_header_t * icmp0, * icmp1;
2590
2591           /* Prefetch next iteration. */
2592           {
2593             vlib_buffer_t * p2, * p3;
2594
2595             p2 = vlib_get_buffer (vm, from[2]);
2596             p3 = vlib_get_buffer (vm, from[3]);
2597
2598             vlib_prefetch_buffer_header (p2, LOAD);
2599             vlib_prefetch_buffer_header (p3, LOAD);
2600
2601             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2602             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2603           }
2604
2605           /* speculatively enqueue b0 and b1 to the current next frame */
2606           to_next[0] = bi0 = from[0];
2607           to_next[1] = bi1 = from[1];
2608           from += 2;
2609           to_next += 2;
2610           n_left_from -= 2;
2611           n_left_to_next -= 2;
2612
2613           b0 = vlib_get_buffer (vm, bi0);
2614           b1 = vlib_get_buffer (vm, bi1);
2615
2616           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2617           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2618
2619           ip0 = vlib_buffer_get_current (b0);
2620           udp0 = ip4_next_header (ip0);
2621           tcp0 = (tcp_header_t *) udp0;
2622
2623           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2624
2625           if (PREDICT_FALSE(ip0->ttl == 1))
2626             {
2627               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2628               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2629                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2630                                            0);
2631               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2632               goto trace0;
2633             }
2634
2635           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2636
2637           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2638             {
2639               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2640               icmp0 = (icmp46_header_t *) udp0;
2641
2642               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2643                                   rx_fib_index0, node, next0, thread_index,
2644                                   &ses0, &dm0);
2645               goto trace0;
2646             }
2647
2648           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2649           if (PREDICT_FALSE(!dm0))
2650             {
2651               clib_warning("no match for internal host %U",
2652                            format_ip4_address, &ip0->src_address);
2653               next0 = SNAT_IN2OUT_NEXT_DROP;
2654               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2655               goto trace0;
2656             }
2657
2658           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2659
2660           key0.ext_host_addr = ip0->dst_address;
2661           key0.ext_host_port = tcp0->dst;
2662
2663           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2664           if (PREDICT_FALSE(!ses0))
2665             {
2666               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2667                 {
2668                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2669                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2670
2671                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2672                     continue;
2673
2674                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2675                   break;
2676                 }
2677               if (PREDICT_FALSE(!ses0))
2678                 {
2679                   /* too many sessions for user, send ICMP error packet */
2680
2681                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2682                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2683                                                ICMP4_destination_unreachable_destination_unreachable_host,
2684                                                0);
2685                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2686                   goto trace0;
2687                 }
2688             }
2689
2690           new_port0 = ses0->out.out_port;
2691
2692           old_addr0.as_u32 = ip0->src_address.as_u32;
2693           ip0->src_address.as_u32 = new_addr0.as_u32;
2694           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2695
2696           sum0 = ip0->checksum;
2697           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2698                                  ip4_header_t,
2699                                  src_address /* changed member */);
2700           ip0->checksum = ip_csum_fold (sum0);
2701
2702           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2703             {
2704               if (tcp0->flags & TCP_FLAG_SYN)
2705                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2706               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2707                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2708               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2709                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2710               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2711                 snat_det_ses_close(dm0, ses0);
2712               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2713                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2714               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2715                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2716
2717               old_port0 = tcp0->src;
2718               tcp0->src = new_port0;
2719
2720               sum0 = tcp0->checksum;
2721               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2722                                      ip4_header_t,
2723                                      dst_address /* changed member */);
2724               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2725                                      ip4_header_t /* cheat */,
2726                                      length /* changed member */);
2727               tcp0->checksum = ip_csum_fold(sum0);
2728             }
2729           else
2730             {
2731               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2732               old_port0 = udp0->src_port;
2733               udp0->src_port = new_port0;
2734               udp0->checksum = 0;
2735             }
2736
2737           switch(ses0->state)
2738             {
2739             case SNAT_SESSION_UDP_ACTIVE:
2740                 ses0->expire = now + sm->udp_timeout;
2741                 break;
2742             case SNAT_SESSION_TCP_SYN_SENT:
2743             case SNAT_SESSION_TCP_FIN_WAIT:
2744             case SNAT_SESSION_TCP_CLOSE_WAIT:
2745             case SNAT_SESSION_TCP_LAST_ACK:
2746                 ses0->expire = now + sm->tcp_transitory_timeout;
2747                 break;
2748             case SNAT_SESSION_TCP_ESTABLISHED:
2749                 ses0->expire = now + sm->tcp_established_timeout;
2750                 break;
2751             }
2752
2753         trace0:
2754           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2755                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2756             {
2757               snat_in2out_trace_t *t =
2758                  vlib_add_trace (vm, node, b0, sizeof (*t));
2759               t->is_slow_path = 0;
2760               t->sw_if_index = sw_if_index0;
2761               t->next_index = next0;
2762               t->session_index = ~0;
2763               if (ses0)
2764                 t->session_index = ses0 - dm0->sessions;
2765             }
2766
2767           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2768
2769           ip1 = vlib_buffer_get_current (b1);
2770           udp1 = ip4_next_header (ip1);
2771           tcp1 = (tcp_header_t *) udp1;
2772
2773           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2774
2775           if (PREDICT_FALSE(ip1->ttl == 1))
2776             {
2777               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2778               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2779                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2780                                            0);
2781               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2782               goto trace1;
2783             }
2784
2785           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2786
2787           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2788             {
2789               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2790               icmp1 = (icmp46_header_t *) udp1;
2791
2792               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2793                                   rx_fib_index1, node, next1, thread_index,
2794                                   &ses1, &dm1);
2795               goto trace1;
2796             }
2797
2798           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2799           if (PREDICT_FALSE(!dm1))
2800             {
2801               clib_warning("no match for internal host %U",
2802                            format_ip4_address, &ip0->src_address);
2803               next1 = SNAT_IN2OUT_NEXT_DROP;
2804               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2805               goto trace1;
2806             }
2807
2808           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2809
2810           key1.ext_host_addr = ip1->dst_address;
2811           key1.ext_host_port = tcp1->dst;
2812
2813           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2814           if (PREDICT_FALSE(!ses1))
2815             {
2816               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2817                 {
2818                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2819                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2820
2821                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2822                     continue;
2823
2824                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2825                   break;
2826                 }
2827               if (PREDICT_FALSE(!ses1))
2828                 {
2829                   /* too many sessions for user, send ICMP error packet */
2830
2831                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2832                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2833                                                ICMP4_destination_unreachable_destination_unreachable_host,
2834                                                0);
2835                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2836                   goto trace1;
2837                 }
2838             }
2839
2840           new_port1 = ses1->out.out_port;
2841
2842           old_addr1.as_u32 = ip1->src_address.as_u32;
2843           ip1->src_address.as_u32 = new_addr1.as_u32;
2844           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2845
2846           sum1 = ip1->checksum;
2847           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2848                                  ip4_header_t,
2849                                  src_address /* changed member */);
2850           ip1->checksum = ip_csum_fold (sum1);
2851
2852           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2853             {
2854               if (tcp1->flags & TCP_FLAG_SYN)
2855                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2856               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2857                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2858               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2859                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2860               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2861                 snat_det_ses_close(dm1, ses1);
2862               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2863                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2864               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2865                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2866
2867               old_port1 = tcp1->src;
2868               tcp1->src = new_port1;
2869
2870               sum1 = tcp1->checksum;
2871               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2872                                      ip4_header_t,
2873                                      dst_address /* changed member */);
2874               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2875                                      ip4_header_t /* cheat */,
2876                                      length /* changed member */);
2877               tcp1->checksum = ip_csum_fold(sum1);
2878             }
2879           else
2880             {
2881               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2882               old_port1 = udp1->src_port;
2883               udp1->src_port = new_port1;
2884               udp1->checksum = 0;
2885             }
2886
2887           switch(ses1->state)
2888             {
2889             case SNAT_SESSION_UDP_ACTIVE:
2890                 ses1->expire = now + sm->udp_timeout;
2891                 break;
2892             case SNAT_SESSION_TCP_SYN_SENT:
2893             case SNAT_SESSION_TCP_FIN_WAIT:
2894             case SNAT_SESSION_TCP_CLOSE_WAIT:
2895             case SNAT_SESSION_TCP_LAST_ACK:
2896                 ses1->expire = now + sm->tcp_transitory_timeout;
2897                 break;
2898             case SNAT_SESSION_TCP_ESTABLISHED:
2899                 ses1->expire = now + sm->tcp_established_timeout;
2900                 break;
2901             }
2902
2903         trace1:
2904           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2905                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2906             {
2907               snat_in2out_trace_t *t =
2908                  vlib_add_trace (vm, node, b1, sizeof (*t));
2909               t->is_slow_path = 0;
2910               t->sw_if_index = sw_if_index1;
2911               t->next_index = next1;
2912               t->session_index = ~0;
2913               if (ses1)
2914                 t->session_index = ses1 - dm1->sessions;
2915             }
2916
2917           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2918
2919           /* verify speculative enqueues, maybe switch current next frame */
2920           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2921                                            to_next, n_left_to_next,
2922                                            bi0, bi1, next0, next1);
2923          }
2924
2925       while (n_left_from > 0 && n_left_to_next > 0)
2926         {
2927           u32 bi0;
2928           vlib_buffer_t * b0;
2929           u32 next0;
2930           u32 sw_if_index0;
2931           ip4_header_t * ip0;
2932           ip_csum_t sum0;
2933           ip4_address_t new_addr0, old_addr0;
2934           u16 old_port0, new_port0, lo_port0, i0;
2935           udp_header_t * udp0;
2936           tcp_header_t * tcp0;
2937           u32 proto0;
2938           snat_det_out_key_t key0;
2939           snat_det_map_t * dm0;
2940           snat_det_session_t * ses0 = 0;
2941           u32 rx_fib_index0;
2942           icmp46_header_t * icmp0;
2943
2944           /* speculatively enqueue b0 to the current next frame */
2945           bi0 = from[0];
2946           to_next[0] = bi0;
2947           from += 1;
2948           to_next += 1;
2949           n_left_from -= 1;
2950           n_left_to_next -= 1;
2951
2952           b0 = vlib_get_buffer (vm, bi0);
2953           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2954
2955           ip0 = vlib_buffer_get_current (b0);
2956           udp0 = ip4_next_header (ip0);
2957           tcp0 = (tcp_header_t *) udp0;
2958
2959           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2960
2961           if (PREDICT_FALSE(ip0->ttl == 1))
2962             {
2963               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2964               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2965                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2966                                            0);
2967               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2968               goto trace00;
2969             }
2970
2971           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2972
2973           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2974             {
2975               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2976               icmp0 = (icmp46_header_t *) udp0;
2977
2978               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2979                                   rx_fib_index0, node, next0, thread_index,
2980                                   &ses0, &dm0);
2981               goto trace00;
2982             }
2983
2984           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2985           if (PREDICT_FALSE(!dm0))
2986             {
2987               clib_warning("no match for internal host %U",
2988                            format_ip4_address, &ip0->src_address);
2989               next0 = SNAT_IN2OUT_NEXT_DROP;
2990               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2991               goto trace00;
2992             }
2993
2994           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2995
2996           key0.ext_host_addr = ip0->dst_address;
2997           key0.ext_host_port = tcp0->dst;
2998
2999           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3000           if (PREDICT_FALSE(!ses0))
3001             {
3002               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3003                 {
3004                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3005                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3006
3007                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3008                     continue;
3009
3010                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3011                   break;
3012                 }
3013               if (PREDICT_FALSE(!ses0))
3014                 {
3015                   /* too many sessions for user, send ICMP error packet */
3016
3017                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3018                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3019                                                ICMP4_destination_unreachable_destination_unreachable_host,
3020                                                0);
3021                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3022                   goto trace00;
3023                 }
3024             }
3025
3026           new_port0 = ses0->out.out_port;
3027
3028           old_addr0.as_u32 = ip0->src_address.as_u32;
3029           ip0->src_address.as_u32 = new_addr0.as_u32;
3030           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3031
3032           sum0 = ip0->checksum;
3033           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3034                                  ip4_header_t,
3035                                  src_address /* changed member */);
3036           ip0->checksum = ip_csum_fold (sum0);
3037
3038           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3039             {
3040               if (tcp0->flags & TCP_FLAG_SYN)
3041                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3042               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3043                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3044               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3045                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3046               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3047                 snat_det_ses_close(dm0, ses0);
3048               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3049                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3050               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3051                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3052
3053               old_port0 = tcp0->src;
3054               tcp0->src = new_port0;
3055
3056               sum0 = tcp0->checksum;
3057               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3058                                      ip4_header_t,
3059                                      dst_address /* changed member */);
3060               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3061                                      ip4_header_t /* cheat */,
3062                                      length /* changed member */);
3063               tcp0->checksum = ip_csum_fold(sum0);
3064             }
3065           else
3066             {
3067               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3068               old_port0 = udp0->src_port;
3069               udp0->src_port = new_port0;
3070               udp0->checksum = 0;
3071             }
3072
3073           switch(ses0->state)
3074             {
3075             case SNAT_SESSION_UDP_ACTIVE:
3076                 ses0->expire = now + sm->udp_timeout;
3077                 break;
3078             case SNAT_SESSION_TCP_SYN_SENT:
3079             case SNAT_SESSION_TCP_FIN_WAIT:
3080             case SNAT_SESSION_TCP_CLOSE_WAIT:
3081             case SNAT_SESSION_TCP_LAST_ACK:
3082                 ses0->expire = now + sm->tcp_transitory_timeout;
3083                 break;
3084             case SNAT_SESSION_TCP_ESTABLISHED:
3085                 ses0->expire = now + sm->tcp_established_timeout;
3086                 break;
3087             }
3088
3089         trace00:
3090           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3091                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3092             {
3093               snat_in2out_trace_t *t =
3094                  vlib_add_trace (vm, node, b0, sizeof (*t));
3095               t->is_slow_path = 0;
3096               t->sw_if_index = sw_if_index0;
3097               t->next_index = next0;
3098               t->session_index = ~0;
3099               if (ses0)
3100                 t->session_index = ses0 - dm0->sessions;
3101             }
3102
3103           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3104
3105           /* verify speculative enqueue, maybe switch current next frame */
3106           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3107                                            to_next, n_left_to_next,
3108                                            bi0, next0);
3109         }
3110
3111       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3112     }
3113
3114   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3115                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3116                                pkts_processed);
3117   return frame->n_vectors;
3118 }
3119
3120 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3121   .function = snat_det_in2out_node_fn,
3122   .name = "nat44-det-in2out",
3123   .vector_size = sizeof (u32),
3124   .format_trace = format_snat_in2out_trace,
3125   .type = VLIB_NODE_TYPE_INTERNAL,
3126
3127   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3128   .error_strings = snat_in2out_error_strings,
3129
3130   .runtime_data_bytes = sizeof (snat_runtime_t),
3131
3132   .n_next_nodes = 3,
3133
3134   /* edit / add dispositions here */
3135   .next_nodes = {
3136     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3137     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3138     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3139   },
3140 };
3141
3142 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3143
3144 /**
3145  * Get address and port values to be used for ICMP packet translation
3146  * and create session if needed
3147  *
3148  * @param[in,out] sm             NAT main
3149  * @param[in,out] node           NAT node runtime
3150  * @param[in] thread_index       thread index
3151  * @param[in,out] b0             buffer containing packet to be translated
3152  * @param[out] p_proto           protocol used for matching
3153  * @param[out] p_value           address and port after NAT translation
3154  * @param[out] p_dont_translate  if packet should not be translated
3155  * @param d                      optional parameter
3156  * @param e                      optional parameter
3157  */
3158 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3159                           u32 thread_index, vlib_buffer_t *b0,
3160                           ip4_header_t *ip0, u8 *p_proto,
3161                           snat_session_key_t *p_value,
3162                           u8 *p_dont_translate, void *d, void *e)
3163 {
3164   icmp46_header_t *icmp0;
3165   u32 sw_if_index0;
3166   u32 rx_fib_index0;
3167   u8 protocol;
3168   snat_det_out_key_t key0;
3169   u8 dont_translate = 0;
3170   u32 next0 = ~0;
3171   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3172   ip4_header_t *inner_ip0;
3173   void *l4_header = 0;
3174   icmp46_header_t *inner_icmp0;
3175   snat_det_map_t * dm0 = 0;
3176   ip4_address_t new_addr0;
3177   u16 lo_port0, i0;
3178   snat_det_session_t * ses0 = 0;
3179   ip4_address_t in_addr;
3180   u16 in_port;
3181
3182   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3183   echo0 = (icmp_echo_header_t *)(icmp0+1);
3184   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3185   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3186
3187   if (!icmp_is_error_message (icmp0))
3188     {
3189       protocol = SNAT_PROTOCOL_ICMP;
3190       in_addr = ip0->src_address;
3191       in_port = echo0->identifier;
3192     }
3193   else
3194     {
3195       inner_ip0 = (ip4_header_t *)(echo0+1);
3196       l4_header = ip4_next_header (inner_ip0);
3197       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3198       in_addr = inner_ip0->dst_address;
3199       switch (protocol)
3200         {
3201         case SNAT_PROTOCOL_ICMP:
3202           inner_icmp0 = (icmp46_header_t*)l4_header;
3203           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3204           in_port = inner_echo0->identifier;
3205           break;
3206         case SNAT_PROTOCOL_UDP:
3207         case SNAT_PROTOCOL_TCP:
3208           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3209           break;
3210         default:
3211           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3212           next0 = SNAT_IN2OUT_NEXT_DROP;
3213           goto out;
3214         }
3215     }
3216
3217   dm0 = snat_det_map_by_user(sm, &in_addr);
3218   if (PREDICT_FALSE(!dm0))
3219     {
3220       clib_warning("no match for internal host %U",
3221                    format_ip4_address, &in_addr);
3222       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3223           IP_PROTOCOL_ICMP, rx_fib_index0)))
3224         {
3225           dont_translate = 1;
3226           goto out;
3227         }
3228       next0 = SNAT_IN2OUT_NEXT_DROP;
3229       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3230       goto out;
3231     }
3232
3233   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3234
3235   key0.ext_host_addr = ip0->dst_address;
3236   key0.ext_host_port = 0;
3237
3238   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3239   if (PREDICT_FALSE(!ses0))
3240     {
3241       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3242           IP_PROTOCOL_ICMP, rx_fib_index0)))
3243         {
3244           dont_translate = 1;
3245           goto out;
3246         }
3247       if (icmp0->type != ICMP4_echo_request)
3248         {
3249           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3250           next0 = SNAT_IN2OUT_NEXT_DROP;
3251           goto out;
3252         }
3253       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3254         {
3255           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3256             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3257
3258           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3259             continue;
3260
3261           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3262           break;
3263         }
3264       if (PREDICT_FALSE(!ses0))
3265         {
3266           next0 = SNAT_IN2OUT_NEXT_DROP;
3267           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3268           goto out;
3269         }
3270     }
3271
3272   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3273                     !icmp_is_error_message (icmp0)))
3274     {
3275       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3276       next0 = SNAT_IN2OUT_NEXT_DROP;
3277       goto out;
3278     }
3279
3280   u32 now = (u32) vlib_time_now (sm->vlib_main);
3281
3282   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3283   ses0->expire = now + sm->icmp_timeout;
3284
3285 out:
3286   *p_proto = protocol;
3287   if (ses0)
3288     {
3289       p_value->addr = new_addr0;
3290       p_value->fib_index = sm->outside_fib_index;
3291       p_value->port = ses0->out.out_port;
3292     }
3293   *p_dont_translate = dont_translate;
3294   if (d)
3295     *(snat_det_session_t**)d = ses0;
3296   if (e)
3297     *(snat_det_map_t**)e = dm0;
3298   return next0;
3299 }
3300
3301 /**********************/
3302 /*** worker handoff ***/
3303 /**********************/
3304 static inline uword
3305 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3306                                       vlib_node_runtime_t * node,
3307                                       vlib_frame_t * frame,
3308                                       u8 is_output)
3309 {
3310   snat_main_t *sm = &snat_main;
3311   vlib_thread_main_t *tm = vlib_get_thread_main ();
3312   u32 n_left_from, *from, *to_next = 0;
3313   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3314   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3315     = 0;
3316   vlib_frame_queue_elt_t *hf = 0;
3317   vlib_frame_t *f = 0;
3318   int i;
3319   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3320   u32 next_worker_index = 0;
3321   u32 current_worker_index = ~0;
3322   u32 thread_index = vlib_get_thread_index ();
3323   u32 fq_index;
3324   u32 to_node_index;
3325
3326   ASSERT (vec_len (sm->workers));
3327
3328   if (is_output)
3329     {
3330       fq_index = sm->fq_in2out_output_index;
3331       to_node_index = sm->in2out_output_node_index;
3332     }
3333   else
3334     {
3335       fq_index = sm->fq_in2out_index;
3336       to_node_index = sm->in2out_node_index;
3337     }
3338
3339   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3340     {
3341       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3342
3343       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3344                                sm->first_worker_index + sm->num_workers - 1,
3345                                (vlib_frame_queue_t *) (~0));
3346     }
3347
3348   from = vlib_frame_vector_args (frame);
3349   n_left_from = frame->n_vectors;
3350
3351   while (n_left_from > 0)
3352     {
3353       u32 bi0;
3354       vlib_buffer_t *b0;
3355       u32 sw_if_index0;
3356       u32 rx_fib_index0;
3357       ip4_header_t * ip0;
3358       u8 do_handoff;
3359
3360       bi0 = from[0];
3361       from += 1;
3362       n_left_from -= 1;
3363
3364       b0 = vlib_get_buffer (vm, bi0);
3365
3366       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3367       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3368
3369       ip0 = vlib_buffer_get_current (b0);
3370
3371       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3372
3373       if (PREDICT_FALSE (next_worker_index != thread_index))
3374         {
3375           do_handoff = 1;
3376
3377           if (next_worker_index != current_worker_index)
3378             {
3379               if (hf)
3380                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3381
3382               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3383                                                       next_worker_index,
3384                                                       handoff_queue_elt_by_worker_index);
3385
3386               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3387               to_next_worker = &hf->buffer_index[hf->n_vectors];
3388               current_worker_index = next_worker_index;
3389             }
3390
3391           /* enqueue to correct worker thread */
3392           to_next_worker[0] = bi0;
3393           to_next_worker++;
3394           n_left_to_next_worker--;
3395
3396           if (n_left_to_next_worker == 0)
3397             {
3398               hf->n_vectors = VLIB_FRAME_SIZE;
3399               vlib_put_frame_queue_elt (hf);
3400               current_worker_index = ~0;
3401               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3402               hf = 0;
3403             }
3404         }
3405       else
3406         {
3407           do_handoff = 0;
3408           /* if this is 1st frame */
3409           if (!f)
3410             {
3411               f = vlib_get_frame_to_node (vm, to_node_index);
3412               to_next = vlib_frame_vector_args (f);
3413             }
3414
3415           to_next[0] = bi0;
3416           to_next += 1;
3417           f->n_vectors++;
3418         }
3419
3420       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3421                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3422         {
3423           snat_in2out_worker_handoff_trace_t *t =
3424             vlib_add_trace (vm, node, b0, sizeof (*t));
3425           t->next_worker_index = next_worker_index;
3426           t->do_handoff = do_handoff;
3427         }
3428     }
3429
3430   if (f)
3431     vlib_put_frame_to_node (vm, to_node_index, f);
3432
3433   if (hf)
3434     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3435
3436   /* Ship frames to the worker nodes */
3437   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3438     {
3439       if (handoff_queue_elt_by_worker_index[i])
3440         {
3441           hf = handoff_queue_elt_by_worker_index[i];
3442           /*
3443            * It works better to let the handoff node
3444            * rate-adapt, always ship the handoff queue element.
3445            */
3446           if (1 || hf->n_vectors == hf->last_n_vectors)
3447             {
3448               vlib_put_frame_queue_elt (hf);
3449               handoff_queue_elt_by_worker_index[i] = 0;
3450             }
3451           else
3452             hf->last_n_vectors = hf->n_vectors;
3453         }
3454       congested_handoff_queue_by_worker_index[i] =
3455         (vlib_frame_queue_t *) (~0);
3456     }
3457   hf = 0;
3458   current_worker_index = ~0;
3459   return frame->n_vectors;
3460 }
3461
3462 static uword
3463 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3464                                vlib_node_runtime_t * node,
3465                                vlib_frame_t * frame)
3466 {
3467   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3468 }
3469
3470 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3471   .function = snat_in2out_worker_handoff_fn,
3472   .name = "nat44-in2out-worker-handoff",
3473   .vector_size = sizeof (u32),
3474   .format_trace = format_snat_in2out_worker_handoff_trace,
3475   .type = VLIB_NODE_TYPE_INTERNAL,
3476
3477   .n_next_nodes = 1,
3478
3479   .next_nodes = {
3480     [0] = "error-drop",
3481   },
3482 };
3483
3484 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3485                               snat_in2out_worker_handoff_fn);
3486
3487 static uword
3488 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3489                                       vlib_node_runtime_t * node,
3490                                       vlib_frame_t * frame)
3491 {
3492   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3493 }
3494
3495 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3496   .function = snat_in2out_output_worker_handoff_fn,
3497   .name = "nat44-in2out-output-worker-handoff",
3498   .vector_size = sizeof (u32),
3499   .format_trace = format_snat_in2out_worker_handoff_trace,
3500   .type = VLIB_NODE_TYPE_INTERNAL,
3501
3502   .n_next_nodes = 1,
3503
3504   .next_nodes = {
3505     [0] = "error-drop",
3506   },
3507 };
3508
3509 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3510                               snat_in2out_output_worker_handoff_fn);
3511
3512 static_always_inline int
3513 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3514 {
3515   snat_address_t * ap;
3516   clib_bihash_kv_8_8_t kv, value;
3517   snat_session_key_t m_key;
3518
3519   vec_foreach (ap, sm->addresses)
3520     {
3521       if (ap->addr.as_u32 == dst_addr->as_u32)
3522         return 1;
3523     }
3524
3525   m_key.addr.as_u32 = dst_addr->as_u32;
3526   m_key.fib_index = sm->outside_fib_index;
3527   m_key.port = 0;
3528   m_key.protocol = 0;
3529   kv.key = m_key.as_u64;
3530   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3531     return 1;
3532
3533   return 0;
3534 }
3535
3536 static uword
3537 snat_hairpin_dst_fn (vlib_main_t * vm,
3538                      vlib_node_runtime_t * node,
3539                      vlib_frame_t * frame)
3540 {
3541   u32 n_left_from, * from, * to_next;
3542   snat_in2out_next_t next_index;
3543   u32 pkts_processed = 0;
3544   snat_main_t * sm = &snat_main;
3545
3546   from = vlib_frame_vector_args (frame);
3547   n_left_from = frame->n_vectors;
3548   next_index = node->cached_next_index;
3549
3550   while (n_left_from > 0)
3551     {
3552       u32 n_left_to_next;
3553
3554       vlib_get_next_frame (vm, node, next_index,
3555                            to_next, n_left_to_next);
3556
3557       while (n_left_from > 0 && n_left_to_next > 0)
3558         {
3559           u32 bi0;
3560           vlib_buffer_t * b0;
3561           u32 next0;
3562           ip4_header_t * ip0;
3563           u32 proto0;
3564
3565           /* speculatively enqueue b0 to the current next frame */
3566           bi0 = from[0];
3567           to_next[0] = bi0;
3568           from += 1;
3569           to_next += 1;
3570           n_left_from -= 1;
3571           n_left_to_next -= 1;
3572
3573           b0 = vlib_get_buffer (vm, bi0);
3574           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3575           ip0 = vlib_buffer_get_current (b0);
3576
3577           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3578
3579           vnet_buffer (b0)->snat.flags = 0;
3580           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3581             {
3582               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3583                 {
3584                   udp_header_t * udp0 = ip4_next_header (ip0);
3585                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3586
3587                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3588                 }
3589               else if (proto0 == SNAT_PROTOCOL_ICMP)
3590                 {
3591                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3592
3593                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3594                 }
3595               else
3596                 {
3597                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3598                 }
3599
3600               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3601             }
3602
3603           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3604
3605           /* verify speculative enqueue, maybe switch current next frame */
3606           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3607                                            to_next, n_left_to_next,
3608                                            bi0, next0);
3609          }
3610
3611       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3612     }
3613
3614   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3615                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3616                                pkts_processed);
3617   return frame->n_vectors;
3618 }
3619
3620 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3621   .function = snat_hairpin_dst_fn,
3622   .name = "nat44-hairpin-dst",
3623   .vector_size = sizeof (u32),
3624   .type = VLIB_NODE_TYPE_INTERNAL,
3625   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3626   .error_strings = snat_in2out_error_strings,
3627   .n_next_nodes = 2,
3628   .next_nodes = {
3629     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3630     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3631   },
3632 };
3633
3634 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3635                               snat_hairpin_dst_fn);
3636
3637 static uword
3638 snat_hairpin_src_fn (vlib_main_t * vm,
3639                      vlib_node_runtime_t * node,
3640                      vlib_frame_t * frame)
3641 {
3642   u32 n_left_from, * from, * to_next;
3643   snat_in2out_next_t next_index;
3644   u32 pkts_processed = 0;
3645   snat_main_t *sm = &snat_main;
3646
3647   from = vlib_frame_vector_args (frame);
3648   n_left_from = frame->n_vectors;
3649   next_index = node->cached_next_index;
3650
3651   while (n_left_from > 0)
3652     {
3653       u32 n_left_to_next;
3654
3655       vlib_get_next_frame (vm, node, next_index,
3656                            to_next, n_left_to_next);
3657
3658       while (n_left_from > 0 && n_left_to_next > 0)
3659         {
3660           u32 bi0;
3661           vlib_buffer_t * b0;
3662           u32 next0;
3663           snat_interface_t *i;
3664           u32 sw_if_index0;
3665
3666           /* speculatively enqueue b0 to the current next frame */
3667           bi0 = from[0];
3668           to_next[0] = bi0;
3669           from += 1;
3670           to_next += 1;
3671           n_left_from -= 1;
3672           n_left_to_next -= 1;
3673
3674           b0 = vlib_get_buffer (vm, bi0);
3675           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3676           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3677
3678           pool_foreach (i, sm->output_feature_interfaces,
3679           ({
3680             /* Only packets from NAT inside interface */
3681             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3682               {
3683                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3684                                     SNAT_FLAG_HAIRPINNING))
3685                   {
3686                     if (PREDICT_TRUE (sm->num_workers > 1))
3687                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3688                     else
3689                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3690                   }
3691                 break;
3692               }
3693           }));
3694
3695           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3696
3697           /* verify speculative enqueue, maybe switch current next frame */
3698           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3699                                            to_next, n_left_to_next,
3700                                            bi0, next0);
3701          }
3702
3703       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3704     }
3705
3706   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3707                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3708                                pkts_processed);
3709   return frame->n_vectors;
3710 }
3711
3712 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3713   .function = snat_hairpin_src_fn,
3714   .name = "nat44-hairpin-src",
3715   .vector_size = sizeof (u32),
3716   .type = VLIB_NODE_TYPE_INTERNAL,
3717   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3718   .error_strings = snat_in2out_error_strings,
3719   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3720   .next_nodes = {
3721      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3722      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3723      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3724      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3725   },
3726 };
3727
3728 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3729                               snat_hairpin_src_fn);
3730
3731 static uword
3732 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3733                                 vlib_node_runtime_t * node,
3734                                 vlib_frame_t * frame)
3735 {
3736   u32 n_left_from, * from, * to_next;
3737   snat_in2out_next_t next_index;
3738   u32 pkts_processed = 0;
3739   snat_main_t * sm = &snat_main;
3740   u32 stats_node_index;
3741
3742   stats_node_index = snat_in2out_fast_node.index;
3743
3744   from = vlib_frame_vector_args (frame);
3745   n_left_from = frame->n_vectors;
3746   next_index = node->cached_next_index;
3747
3748   while (n_left_from > 0)
3749     {
3750       u32 n_left_to_next;
3751
3752       vlib_get_next_frame (vm, node, next_index,
3753                            to_next, n_left_to_next);
3754
3755       while (n_left_from > 0 && n_left_to_next > 0)
3756         {
3757           u32 bi0;
3758           vlib_buffer_t * b0;
3759           u32 next0;
3760           u32 sw_if_index0;
3761           ip4_header_t * ip0;
3762           ip_csum_t sum0;
3763           u32 new_addr0, old_addr0;
3764           u16 old_port0, new_port0;
3765           udp_header_t * udp0;
3766           tcp_header_t * tcp0;
3767           icmp46_header_t * icmp0;
3768           snat_session_key_t key0, sm0;
3769           u32 proto0;
3770           u32 rx_fib_index0;
3771
3772           /* speculatively enqueue b0 to the current next frame */
3773           bi0 = from[0];
3774           to_next[0] = bi0;
3775           from += 1;
3776           to_next += 1;
3777           n_left_from -= 1;
3778           n_left_to_next -= 1;
3779
3780           b0 = vlib_get_buffer (vm, bi0);
3781           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3782
3783           ip0 = vlib_buffer_get_current (b0);
3784           udp0 = ip4_next_header (ip0);
3785           tcp0 = (tcp_header_t *) udp0;
3786           icmp0 = (icmp46_header_t *) udp0;
3787
3788           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3789           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3790
3791           if (PREDICT_FALSE(ip0->ttl == 1))
3792             {
3793               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3794               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3795                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3796                                            0);
3797               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3798               goto trace0;
3799             }
3800
3801           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3802
3803           if (PREDICT_FALSE (proto0 == ~0))
3804               goto trace0;
3805
3806           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3807             {
3808               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3809                                   rx_fib_index0, node, next0, ~0, 0, 0);
3810               goto trace0;
3811             }
3812
3813           key0.addr = ip0->src_address;
3814           key0.protocol = proto0;
3815           key0.port = udp0->src_port;
3816           key0.fib_index = rx_fib_index0;
3817
3818           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
3819             {
3820               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3821               next0= SNAT_IN2OUT_NEXT_DROP;
3822               goto trace0;
3823             }
3824
3825           new_addr0 = sm0.addr.as_u32;
3826           new_port0 = sm0.port;
3827           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3828           old_addr0 = ip0->src_address.as_u32;
3829           ip0->src_address.as_u32 = new_addr0;
3830
3831           sum0 = ip0->checksum;
3832           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3833                                  ip4_header_t,
3834                                  src_address /* changed member */);
3835           ip0->checksum = ip_csum_fold (sum0);
3836
3837           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3838             {
3839               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3840                 {
3841                   old_port0 = tcp0->src_port;
3842                   tcp0->src_port = new_port0;
3843
3844                   sum0 = tcp0->checksum;
3845                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3846                                          ip4_header_t,
3847                                          dst_address /* changed member */);
3848                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3849                                          ip4_header_t /* cheat */,
3850                                          length /* changed member */);
3851                   tcp0->checksum = ip_csum_fold(sum0);
3852                 }
3853               else
3854                 {
3855                   old_port0 = udp0->src_port;
3856                   udp0->src_port = new_port0;
3857                   udp0->checksum = 0;
3858                 }
3859             }
3860           else
3861             {
3862               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3863                 {
3864                   sum0 = tcp0->checksum;
3865                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3866                                          ip4_header_t,
3867                                          dst_address /* changed member */);
3868                   tcp0->checksum = ip_csum_fold(sum0);
3869                 }
3870             }
3871
3872           /* Hairpinning */
3873           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3874
3875         trace0:
3876           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3877                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3878             {
3879               snat_in2out_trace_t *t =
3880                  vlib_add_trace (vm, node, b0, sizeof (*t));
3881               t->sw_if_index = sw_if_index0;
3882               t->next_index = next0;
3883             }
3884
3885           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3886
3887           /* verify speculative enqueue, maybe switch current next frame */
3888           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3889                                            to_next, n_left_to_next,
3890                                            bi0, next0);
3891         }
3892
3893       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3894     }
3895
3896   vlib_node_increment_counter (vm, stats_node_index,
3897                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3898                                pkts_processed);
3899   return frame->n_vectors;
3900 }
3901
3902
3903 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3904   .function = snat_in2out_fast_static_map_fn,
3905   .name = "nat44-in2out-fast",
3906   .vector_size = sizeof (u32),
3907   .format_trace = format_snat_in2out_fast_trace,
3908   .type = VLIB_NODE_TYPE_INTERNAL,
3909
3910   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3911   .error_strings = snat_in2out_error_strings,
3912
3913   .runtime_data_bytes = sizeof (snat_runtime_t),
3914
3915   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3916
3917   /* edit / add dispositions here */
3918   .next_nodes = {
3919     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3920     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3921     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3922     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3923     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3924   },
3925 };
3926
3927 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);