src/plugins/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25 #include <snat/snat_ipfix_logging.h>
  26 #include <snat/snat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91
  92 #define foreach_snat_in2out_error                       \
  93 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  94 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  95 _(OUT_OF_PORTS, "Out of ports")                         \
  96 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  97 _(BAD_ICMP_TYPE, "icmp type not echo-request")          \
  98 _(NO_TRANSLATION, "No translation")
  99
 100 typedef enum {
 101 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 102   foreach_snat_in2out_error
 103 #undef _
 104   SNAT_IN2OUT_N_ERROR,
 105 } snat_in2out_error_t;
 106
 107 static char * snat_in2out_error_strings[] = {
 108 #define _(sym,string) string,
 109   foreach_snat_in2out_error
 110 #undef _
 111 };
 112
 113 typedef enum {
 114   SNAT_IN2OUT_NEXT_LOOKUP,
 115   SNAT_IN2OUT_NEXT_DROP,
 116   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 117   SNAT_IN2OUT_NEXT_SLOW_PATH,
 118   SNAT_IN2OUT_N_NEXT,
 119 } snat_in2out_next_t;
 120
 121 /**
 122  * @brief Check if packet should be translated
 123  *
 124  * Packets aimed at outside interface and external addresss with active session
 125  * should be translated.
 126  *
 127  * @param sm            SNAT main
 128  * @param rt            SNAT runtime data
 129  * @param sw_if_index0  index of the inside interface
 130  * @param ip0           IPv4 header
 131  * @param proto0        SNAT protocol
 132  * @param rx_fib_index0 RX FIB index
 133  *
 134  * @returns 0 if packet should be translated otherwise 1
 135  */
 136 static inline int
 137 snat_not_translate (snat_main_t * sm, snat_runtime_t * rt, u32 sw_if_index0,
 138                    ip4_header_t * ip0, u32 proto0, u32 rx_fib_index0)
 139 {
 140   ip4_address_t * first_int_addr;
 141   udp_header_t * udp0 = ip4_next_header (ip0);
 142   snat_session_key_t key0, sm0;
 143   clib_bihash_kv_8_8_t kv0, value0;
 144   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 145   fib_prefix_t pfx = {
 146     .fp_proto = FIB_PROTOCOL_IP4,
 147     .fp_len = 32,
 148     .fp_addr = {
 149         .ip4.as_u32 = ip0->dst_address.as_u32,
 150     },
 151   };
 152
 153   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 154     {
 155       first_int_addr =
 156         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 157                                      0 /* just want the address */);
 158       rt->cached_sw_if_index = sw_if_index0;
 159       if (first_int_addr)
 160         rt->cached_ip4_address = first_int_addr->as_u32;
 161       else
 162         rt->cached_ip4_address = 0;
 163     }
 164
 165   /* Don't NAT packet aimed at the intfc address */
 166   if (PREDICT_FALSE(ip0->dst_address.as_u32 == rt->cached_ip4_address))
 167     return 1;
 168
 169   key0.addr = ip0->dst_address;
 170   key0.port = udp0->dst_port;
 171   key0.protocol = proto0;
 172   key0.fib_index = sm->outside_fib_index;
 173   kv0.key = key0.as_u64;
 174
 175   /* NAT packet aimed at external address if */
 176   /* has active sessions */
 177   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 178     {
 179       /* or is static mappings */
 180       if (!snat_static_mapping_match(sm, key0, &sm0, 1))
 181         return 0;
 182     }
 183   else
 184     return 0;
 185
 186   fei = fib_table_lookup (rx_fib_index0, &pfx);
 187   if (FIB_NODE_INDEX_INVALID != fei)
 188     {
 189       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 190       if (sw_if_index == ~0)
 191         {
 192           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 193           if (FIB_NODE_INDEX_INVALID != fei)
 194             sw_if_index = fib_entry_get_resolving_interface (fei);
 195         }
 196       snat_interface_t *i;
 197       pool_foreach (i, sm->interfaces,
 198       ({
 199         /* NAT packet aimed at outside interface */
 200         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 201           return 0;
 202       }));
 203     }
 204
 205   return 1;
 206 }
 207
 208 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 209                       ip4_header_t * ip0,
 210                       u32 rx_fib_index0,
 211                       snat_session_key_t * key0,
 212                       snat_session_t ** sessionp,
 213                       vlib_node_runtime_t * node,
 214                       u32 next0,
 215                       u32 thread_index)
 216 {
 217   snat_user_t *u;
 218   snat_user_key_t user_key;
 219   snat_session_t *s;
 220   clib_bihash_kv_8_8_t kv0, value0;
 221   u32 oldest_per_user_translation_list_index;
 222   dlist_elt_t * oldest_per_user_translation_list_elt;
 223   dlist_elt_t * per_user_translation_list_elt;
 224   dlist_elt_t * per_user_list_head_elt;
 225   u32 session_index;
 226   snat_session_key_t key1;
 227   u32 address_index = ~0;
 228   u32 outside_fib_index;
 229   uword * p;
 230   snat_worker_key_t worker_by_out_key;
 231
 232   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 233   if (! p)
 234     {
 235       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 236       return SNAT_IN2OUT_NEXT_DROP;
 237     }
 238   outside_fib_index = p[0];
 239
 240   key1.protocol = key0->protocol;
 241   user_key.addr = ip0->src_address;
 242   user_key.fib_index = rx_fib_index0;
 243   kv0.key = user_key.as_u64;
 244
 245   /* Ever heard of the "user" = src ip4 address before? */
 246   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 247     {
 248       /* no, make a new one */
 249       pool_get (sm->per_thread_data[thread_index].users, u);
 250       memset (u, 0, sizeof (*u));
 251       u->addr = ip0->src_address;
 252       u->fib_index = rx_fib_index0;
 253
 254       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 255
 256       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 257         sm->per_thread_data[thread_index].list_pool;
 258
 259       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 260                        u->sessions_per_user_list_head_index);
 261
 262       kv0.value = u - sm->per_thread_data[thread_index].users;
 263
 264       /* add user */
 265       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 266     }
 267   else
 268     {
 269       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 270                              value0.value);
 271     }
 272
 273   /* Over quota? Recycle the least recently used dynamic translation */
 274   if (u->nsessions >= sm->max_translations_per_user)
 275     {
 276       /* Remove the oldest dynamic translation */
 277       do {
 278           oldest_per_user_translation_list_index =
 279             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 280                                     u->sessions_per_user_list_head_index);
 281
 282           ASSERT (oldest_per_user_translation_list_index != ~0);
 283
 284           /* add it back to the end of the LRU list */
 285           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 286                               u->sessions_per_user_list_head_index,
 287                               oldest_per_user_translation_list_index);
 288           /* Get the list element */
 289           oldest_per_user_translation_list_elt =
 290             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 291                                oldest_per_user_translation_list_index);
 292
 293           /* Get the session index from the list element */
 294           session_index = oldest_per_user_translation_list_elt->value;
 295
 296           /* Get the session */
 297           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 298                                  session_index);
 299       } while (snat_is_session_static (s));
 300
 301       /* Remove in2out, out2in keys */
 302       kv0.key = s->in2out.as_u64;
 303       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 304           clib_warning ("in2out key delete failed");
 305       kv0.key = s->out2in.as_u64;
 306       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 307           clib_warning ("out2in key delete failed");
 308
 309       /* log NAT event */
 310       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 311                                           s->out2in.addr.as_u32,
 312                                           s->in2out.protocol,
 313                                           s->in2out.port,
 314                                           s->out2in.port,
 315                                           s->in2out.fib_index);
 316
 317       snat_free_outside_address_and_port
 318         (sm, &s->out2in, s->outside_address_index);
 319       s->outside_address_index = ~0;
 320
 321       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 322                                                &address_index))
 323         {
 324           ASSERT(0);
 325
 326           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 327           return SNAT_IN2OUT_NEXT_DROP;
 328         }
 329       s->outside_address_index = address_index;
 330     }
 331   else
 332     {
 333       u8 static_mapping = 1;
 334
 335       /* First try to match static mapping by local address and port */
 336       if (snat_static_mapping_match (sm, *key0, &key1, 0))
 337         {
 338           static_mapping = 0;
 339           /* Try to create dynamic translation */
 340           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 341                                                    &address_index))
 342             {
 343               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 344               return SNAT_IN2OUT_NEXT_DROP;
 345             }
 346         }
 347
 348       /* Create a new session */
 349       pool_get (sm->per_thread_data[thread_index].sessions, s);
 350       memset (s, 0, sizeof (*s));
 351
 352       s->outside_address_index = address_index;
 353
 354       if (static_mapping)
 355         {
 356           u->nstaticsessions++;
 357           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 358         }
 359       else
 360         {
 361           u->nsessions++;
 362         }
 363
 364       /* Create list elts */
 365       pool_get (sm->per_thread_data[thread_index].list_pool,
 366                 per_user_translation_list_elt);
 367       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 368                        per_user_translation_list_elt -
 369                        sm->per_thread_data[thread_index].list_pool);
 370
 371       per_user_translation_list_elt->value =
 372         s - sm->per_thread_data[thread_index].sessions;
 373       s->per_user_index = per_user_translation_list_elt -
 374                           sm->per_thread_data[thread_index].list_pool;
 375       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 376
 377       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 378                           s->per_user_list_head_index,
 379                           per_user_translation_list_elt -
 380                           sm->per_thread_data[thread_index].list_pool);
 381    }
 382
 383   s->in2out = *key0;
 384   s->out2in = key1;
 385   s->out2in.protocol = key0->protocol;
 386   s->out2in.fib_index = outside_fib_index;
 387   *sessionp = s;
 388
 389   /* Add to translation hashes */
 390   kv0.key = s->in2out.as_u64;
 391   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 392   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 393       clib_warning ("in2out key add failed");
 394
 395   kv0.key = s->out2in.as_u64;
 396   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 397
 398   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 399       clib_warning ("out2in key add failed");
 400
 401   /* Add to translated packets worker lookup */
 402   worker_by_out_key.addr = s->out2in.addr;
 403   worker_by_out_key.port = s->out2in.port;
 404   worker_by_out_key.fib_index = s->out2in.fib_index;
 405   kv0.key = worker_by_out_key.as_u64;
 406   kv0.value = thread_index;
 407   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 408
 409   /* log NAT event */
 410   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 411                                       s->out2in.addr.as_u32,
 412                                       s->in2out.protocol,
 413                                       s->in2out.port,
 414                                       s->out2in.port,
 415                                       s->in2out.fib_index);
 416   return next0;
 417 }
 418
 419 static_always_inline
 420 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 421                                  snat_session_key_t *p_key0)
 422 {
 423   icmp46_header_t *icmp0;
 424   snat_session_key_t key0;
 425   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 426   ip4_header_t *inner_ip0 = 0;
 427   void *l4_header = 0;
 428   icmp46_header_t *inner_icmp0;
 429
 430   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 431   echo0 = (icmp_echo_header_t *)(icmp0+1);
 432
 433   if (!icmp_is_error_message (icmp0))
 434     {
 435       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request))
 436         return SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE;
 437       key0.protocol = SNAT_PROTOCOL_ICMP;
 438       key0.addr = ip0->src_address;
 439       key0.port = echo0->identifier;
 440     }
 441   else
 442     {
 443       inner_ip0 = (ip4_header_t *)(echo0+1);
 444       l4_header = ip4_next_header (inner_ip0);
 445       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 446       key0.addr = inner_ip0->dst_address;
 447       switch (key0.protocol)
 448         {
 449         case SNAT_PROTOCOL_ICMP:
 450           inner_icmp0 = (icmp46_header_t*)l4_header;
 451           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 452           key0.port = inner_echo0->identifier;
 453           break;
 454         case SNAT_PROTOCOL_UDP:
 455         case SNAT_PROTOCOL_TCP:
 456           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 457           break;
 458         default:
 459           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 460         }
 461     }
 462   *p_key0 = key0;
 463   return -1; /* success */
 464 }
 465
 466 /**
 467  * Get address and port values to be used for packet SNAT translation
 468  * and create session if needed
 469  *
 470  * @param[in,out] sm             SNAT main
 471  * @param[in,out] node           SNAT node runtime
 472  * @param[in] thread_index       thread index
 473  * @param[in,out] b0             buffer containing packet to be translated
 474  * @param[out] p_key             address and port before NAT translation
 475  * @param[out] p_value           address and port after NAT translation
 476  * @param[out] p_dont_translate  if packet should not be translated
 477  * @param d                      optional parameter
 478  */
 479 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 480                            u32 thread_index, vlib_buffer_t *b0,
 481                            snat_session_key_t *p_key,
 482                            snat_session_key_t *p_value,
 483                            u8 *p_dont_translate, void *d)
 484 {
 485   snat_runtime_t *rt;
 486   ip4_header_t *ip0;
 487   icmp46_header_t *icmp0;
 488   u32 sw_if_index0;
 489   u32 rx_fib_index0;
 490   snat_session_key_t key0;
 491   snat_session_t *s0 = 0;
 492   u8 dont_translate = 0;
 493   clib_bihash_kv_8_8_t kv0, value0;
 494   u32 next0 = ~0;
 495   int err;
 496
 497   rt = (snat_runtime_t *) node->runtime_data;
 498   ip0 = vlib_buffer_get_current (b0);
 499   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 500   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 501   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 502
 503   err = icmp_get_key (ip0, &key0);
 504   if (err != -1)
 505     {
 506       b0->error = node->errors[err];
 507       next0 = SNAT_IN2OUT_NEXT_DROP;
 508       goto out;
 509     }
 510   key0.fib_index = rx_fib_index0;
 511
 512   kv0.key = key0.as_u64;
 513
 514   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 515     {
 516       if (PREDICT_FALSE(snat_not_translate(sm, rt, sw_if_index0, ip0,
 517           IP_PROTOCOL_ICMP, rx_fib_index0)))
 518         {
 519           dont_translate = 1;
 520           goto out;
 521         }
 522
 523       if (icmp_is_error_message (icmp0))
 524         {
 525           next0 = SNAT_IN2OUT_NEXT_DROP;
 526           goto out;
 527         }
 528
 529       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 530                          &s0, node, next0, thread_index);
 531
 532       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 533         goto out;
 534     }
 535   else
 536     s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 537                             value0.value);
 538
 539 out:
 540   *p_key = key0;
 541   if (s0)
 542     *p_value = s0->out2in;
 543   *p_dont_translate = dont_translate;
 544   if (d)
 545     *(snat_session_t**)d = s0;
 546   return next0;
 547 }
 548
 549 /**
 550  * Get address and port values to be used for packet SNAT translation
 551  *
 552  * @param[in] sm                 SNAT main
 553  * @param[in,out] node           SNAT node runtime
 554  * @param[in] thread_index       thread index
 555  * @param[in,out] b0             buffer containing packet to be translated
 556  * @param[out] p_key             address and port before NAT translation
 557  * @param[out] p_value           address and port after NAT translation
 558  * @param[out] p_dont_translate  if packet should not be translated
 559  * @param d                      optional parameter
 560  */
 561 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 562                            u32 thread_index, vlib_buffer_t *b0,
 563                            snat_session_key_t *p_key,
 564                            snat_session_key_t *p_value,
 565                            u8 *p_dont_translate, void *d)
 566 {
 567   snat_runtime_t *rt;
 568   ip4_header_t *ip0;
 569   icmp46_header_t *icmp0;
 570   u32 sw_if_index0;
 571   u32 rx_fib_index0;
 572   snat_session_key_t key0;
 573   snat_session_key_t sm0;
 574   u8 dont_translate = 0;
 575   u32 next0 = ~0;
 576   int err;
 577
 578   rt = (snat_runtime_t *) node->runtime_data;
 579   ip0 = vlib_buffer_get_current (b0);
 580   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 581   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 582   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 583
 584   err = icmp_get_key (ip0, &key0);
 585   if (err != -1)
 586     {
 587       b0->error = node->errors[err];
 588       next0 = SNAT_IN2OUT_NEXT_DROP;
 589       goto out2;
 590     }
 591   key0.fib_index = rx_fib_index0;
 592
 593   if (snat_static_mapping_match(sm, key0, &sm0, 0))
 594     {
 595       if (PREDICT_FALSE(snat_not_translate(sm, rt, sw_if_index0, ip0,
 596           IP_PROTOCOL_ICMP, rx_fib_index0)))
 597         {
 598           dont_translate = 1;
 599           goto out;
 600         }
 601
 602       if (icmp_is_error_message (icmp0))
 603         {
 604           next0 = SNAT_IN2OUT_NEXT_DROP;
 605           goto out;
 606         }
 607
 608       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 609       next0 = SNAT_IN2OUT_NEXT_DROP;
 610       goto out;
 611     }
 612
 613 out:
 614   *p_value = sm0;
 615 out2:
 616   *p_key = key0;
 617   *p_dont_translate = dont_translate;
 618   return next0;
 619 }
 620
 621 static inline u32 icmp_in2out (snat_main_t *sm,
 622                                vlib_buffer_t * b0,
 623                                ip4_header_t * ip0,
 624                                icmp46_header_t * icmp0,
 625                                u32 sw_if_index0,
 626                                u32 rx_fib_index0,
 627                                vlib_node_runtime_t * node,
 628                                u32 next0,
 629                                u32 thread_index,
 630                                void *d)
 631 {
 632   snat_session_key_t key0, sm0;
 633   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 634   ip4_header_t *inner_ip0;
 635   void *l4_header = 0;
 636   icmp46_header_t *inner_icmp0;
 637   u8 dont_translate;
 638   u32 new_addr0, old_addr0;
 639   u16 old_id0, new_id0;
 640   ip_csum_t sum0;
 641   u16 checksum0;
 642   u32 next0_tmp;
 643
 644   echo0 = (icmp_echo_header_t *)(icmp0+1);
 645
 646   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 647                                        &key0, &sm0, &dont_translate, d);
 648   if (next0_tmp != ~0)
 649     next0 = next0_tmp;
 650   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 651     goto out;
 652
 653   sum0 = ip_incremental_checksum (0, icmp0,
 654                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 655   checksum0 = ~ip_csum_fold (sum0);
 656   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 657     {
 658       next0 = SNAT_IN2OUT_NEXT_DROP;
 659       goto out;
 660     }
 661
 662   old_addr0 = ip0->src_address.as_u32;
 663   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 664   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 665
 666   sum0 = ip0->checksum;
 667   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 668                          src_address /* changed member */);
 669   ip0->checksum = ip_csum_fold (sum0);
 670
 671   if (!icmp_is_error_message (icmp0))
 672     {
 673       new_id0 = sm0.port;
 674       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 675         {
 676           old_id0 = echo0->identifier;
 677           new_id0 = sm0.port;
 678           echo0->identifier = new_id0;
 679
 680           sum0 = icmp0->checksum;
 681           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 682                                  identifier);
 683           icmp0->checksum = ip_csum_fold (sum0);
 684         }
 685     }
 686   else
 687     {
 688       inner_ip0 = (ip4_header_t *)(echo0+1);
 689       l4_header = ip4_next_header (inner_ip0);
 690
 691       if (!ip4_header_checksum_is_valid (inner_ip0))
 692         {
 693           next0 = SNAT_IN2OUT_NEXT_DROP;
 694           goto out;
 695         }
 696
 697       old_addr0 = inner_ip0->dst_address.as_u32;
 698       inner_ip0->dst_address = sm0.addr;
 699       new_addr0 = inner_ip0->dst_address.as_u32;
 700
 701       sum0 = icmp0->checksum;
 702       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 703                              dst_address /* changed member */);
 704       icmp0->checksum = ip_csum_fold (sum0);
 705
 706       switch (key0.protocol)
 707         {
 708           case SNAT_PROTOCOL_ICMP:
 709             inner_icmp0 = (icmp46_header_t*)l4_header;
 710             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 711
 712             old_id0 = inner_echo0->identifier;
 713             new_id0 = sm0.port;
 714             inner_echo0->identifier = new_id0;
 715
 716             sum0 = icmp0->checksum;
 717             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 718                                    identifier);
 719             icmp0->checksum = ip_csum_fold (sum0);
 720             break;
 721           case SNAT_PROTOCOL_UDP:
 722           case SNAT_PROTOCOL_TCP:
 723             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 724             new_id0 = sm0.port;
 725             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 726
 727             sum0 = icmp0->checksum;
 728             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 729                                    dst_port);
 730             icmp0->checksum = ip_csum_fold (sum0);
 731             break;
 732           default:
 733             ASSERT(0);
 734         }
 735     }
 736
 737 out:
 738   return next0;
 739 }
 740
 741 /**
 742  * @brief Hairpinning
 743  *
 744  * Hairpinning allows two endpoints on the internal side of the NAT to
 745  * communicate even if they only use each other's external IP addresses
 746  * and ports.
 747  *
 748  * @param sm     SNAT main.
 749  * @param b0     Vlib buffer.
 750  * @param ip0    IP header.
 751  * @param udp0   UDP header.
 752  * @param tcp0   TCP header.
 753  * @param proto0 SNAT protocol.
 754  */
 755 static inline void
 756 snat_hairpinning (snat_main_t *sm,
 757                   vlib_buffer_t * b0,
 758                   ip4_header_t * ip0,
 759                   udp_header_t * udp0,
 760                   tcp_header_t * tcp0,
 761                   u32 proto0)
 762 {
 763   snat_session_key_t key0, sm0;
 764   snat_worker_key_t k0;
 765   snat_session_t * s0;
 766   clib_bihash_kv_8_8_t kv0, value0;
 767   ip_csum_t sum0;
 768   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 769   u16 new_dst_port0, old_dst_port0;
 770
 771   key0.addr = ip0->dst_address;
 772   key0.port = udp0->dst_port;
 773   key0.protocol = proto0;
 774   key0.fib_index = sm->outside_fib_index;
 775   kv0.key = key0.as_u64;
 776
 777   /* Check if destination is in active sessions */
 778   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 779     {
 780       /* or static mappings */
 781       if (!snat_static_mapping_match(sm, key0, &sm0, 1))
 782         {
 783           new_dst_addr0 = sm0.addr.as_u32;
 784           new_dst_port0 = sm0.port;
 785           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 786         }
 787     }
 788   else
 789     {
 790       si = value0.value;
 791       if (sm->num_workers > 1)
 792         {
 793           k0.addr = ip0->dst_address;
 794           k0.port = udp0->dst_port;
 795           k0.fib_index = sm->outside_fib_index;
 796           kv0.key = k0.as_u64;
 797           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 798             ASSERT(0);
 799           else
 800             ti = value0.value;
 801         }
 802       else
 803         ti = sm->num_workers;
 804
 805       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 806       new_dst_addr0 = s0->in2out.addr.as_u32;
 807       new_dst_port0 = s0->in2out.port;
 808       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 809     }
 810
 811   /* Destination is behind the same NAT, use internal address and port */
 812   if (new_dst_addr0)
 813     {
 814       old_dst_addr0 = ip0->dst_address.as_u32;
 815       ip0->dst_address.as_u32 = new_dst_addr0;
 816       sum0 = ip0->checksum;
 817       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 818                              ip4_header_t, dst_address);
 819       ip0->checksum = ip_csum_fold (sum0);
 820
 821       old_dst_port0 = tcp0->dst;
 822       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 823         {
 824           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 825             {
 826               tcp0->dst = new_dst_port0;
 827               sum0 = tcp0->checksum;
 828               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 829                                      ip4_header_t, dst_address);
 830               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 831                                      ip4_header_t /* cheat */, length);
 832               tcp0->checksum = ip_csum_fold(sum0);
 833             }
 834           else
 835             {
 836               udp0->dst_port = new_dst_port0;
 837               udp0->checksum = 0;
 838             }
 839         }
 840     }
 841 }
 842
 843 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 844                                          vlib_buffer_t * b0,
 845                                          ip4_header_t * ip0,
 846                                          icmp46_header_t * icmp0,
 847                                          u32 sw_if_index0,
 848                                          u32 rx_fib_index0,
 849                                          vlib_node_runtime_t * node,
 850                                          u32 next0,
 851                                          f64 now,
 852                                          u32 thread_index,
 853                                          snat_session_t ** p_s0)
 854 {
 855   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 856                       next0, thread_index, p_s0);
 857   snat_session_t * s0 = *p_s0;
 858   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 859     {
 860       /* Accounting */
 861       s0->last_heard = now;
 862       s0->total_pkts++;
 863       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 864       /* Per-user LRU list maintenance for dynamic translations */
 865       if (!snat_is_session_static (s0))
 866         {
 867           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 868                              s0->per_user_index);
 869           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 870                               s0->per_user_list_head_index,
 871                               s0->per_user_index);
 872         }
 873     }
 874   return next0;
 875 }
 876
 877 static inline uword
 878 snat_in2out_node_fn_inline (vlib_main_t * vm,
 879                             vlib_node_runtime_t * node,
 880                             vlib_frame_t * frame, int is_slow_path)
 881 {
 882   u32 n_left_from, * from, * to_next;
 883   snat_in2out_next_t next_index;
 884   u32 pkts_processed = 0;
 885   snat_main_t * sm = &snat_main;
 886   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 887   f64 now = vlib_time_now (vm);
 888   u32 stats_node_index;
 889   u32 thread_index = vlib_get_thread_index ();
 890
 891   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 892     snat_in2out_node.index;
 893
 894   from = vlib_frame_vector_args (frame);
 895   n_left_from = frame->n_vectors;
 896   next_index = node->cached_next_index;
 897
 898   while (n_left_from > 0)
 899     {
 900       u32 n_left_to_next;
 901
 902       vlib_get_next_frame (vm, node, next_index,
 903                            to_next, n_left_to_next);
 904
 905       while (n_left_from >= 4 && n_left_to_next >= 2)
 906         {
 907           u32 bi0, bi1;
 908           vlib_buffer_t * b0, * b1;
 909           u32 next0, next1;
 910           u32 sw_if_index0, sw_if_index1;
 911           ip4_header_t * ip0, * ip1;
 912           ip_csum_t sum0, sum1;
 913           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 914           u16 old_port0, new_port0, old_port1, new_port1;
 915           udp_header_t * udp0, * udp1;
 916           tcp_header_t * tcp0, * tcp1;
 917           icmp46_header_t * icmp0, * icmp1;
 918           snat_session_key_t key0, key1;
 919           u32 rx_fib_index0, rx_fib_index1;
 920           u32 proto0, proto1;
 921           snat_session_t * s0 = 0, * s1 = 0;
 922           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 923
 924           /* Prefetch next iteration. */
 925           {
 926             vlib_buffer_t * p2, * p3;
 927
 928             p2 = vlib_get_buffer (vm, from[2]);
 929             p3 = vlib_get_buffer (vm, from[3]);
 930
 931             vlib_prefetch_buffer_header (p2, LOAD);
 932             vlib_prefetch_buffer_header (p3, LOAD);
 933
 934             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 935             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 936           }
 937
 938           /* speculatively enqueue b0 and b1 to the current next frame */
 939           to_next[0] = bi0 = from[0];
 940           to_next[1] = bi1 = from[1];
 941           from += 2;
 942           to_next += 2;
 943           n_left_from -= 2;
 944           n_left_to_next -= 2;
 945
 946           b0 = vlib_get_buffer (vm, bi0);
 947           b1 = vlib_get_buffer (vm, bi1);
 948
 949           ip0 = vlib_buffer_get_current (b0);
 950           udp0 = ip4_next_header (ip0);
 951           tcp0 = (tcp_header_t *) udp0;
 952           icmp0 = (icmp46_header_t *) udp0;
 953
 954           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 955           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 956                                    sw_if_index0);
 957
 958           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 959
 960           if (PREDICT_FALSE(ip0->ttl == 1))
 961             {
 962               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 963               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 964                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 965                                            0);
 966               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 967               goto trace00;
 968             }
 969
 970           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 971
 972           /* Next configured feature, probably ip4-lookup */
 973           if (is_slow_path)
 974             {
 975               if (PREDICT_FALSE (proto0 == ~0))
 976                 goto trace00;
 977
 978               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 979                 {
 980                   next0 = icmp_in2out_slow_path
 981                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 982                      node, next0, now, thread_index, &s0);
 983                   goto trace00;
 984                 }
 985             }
 986           else
 987             {
 988               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 989                 {
 990                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 991                   goto trace00;
 992                 }
 993             }
 994
 995           key0.addr = ip0->src_address;
 996           key0.port = udp0->src_port;
 997           key0.protocol = proto0;
 998           key0.fib_index = rx_fib_index0;
 999
1000           kv0.key = key0.as_u64;
1001
1002           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
1003             {
1004               if (is_slow_path)
1005                 {
1006                   if (PREDICT_FALSE(snat_not_translate(sm, rt, sw_if_index0, ip0,
1007                       proto0, rx_fib_index0)))
1008                     goto trace00;
1009
1010                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1011                                      &s0, node, next0, thread_index);
1012                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1013                     goto trace00;
1014                 }
1015               else
1016                 {
1017                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1018                   goto trace00;
1019                 }
1020             }
1021           else
1022             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1023                                     value0.value);
1024
1025           old_addr0 = ip0->src_address.as_u32;
1026           ip0->src_address = s0->out2in.addr;
1027           new_addr0 = ip0->src_address.as_u32;
1028           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1029
1030           sum0 = ip0->checksum;
1031           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1032                                  ip4_header_t,
1033                                  src_address /* changed member */);
1034           ip0->checksum = ip_csum_fold (sum0);
1035
1036           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1037             {
1038               old_port0 = tcp0->src_port;
1039               tcp0->src_port = s0->out2in.port;
1040               new_port0 = tcp0->src_port;
1041
1042               sum0 = tcp0->checksum;
1043               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1044                                      ip4_header_t,
1045                                      dst_address /* changed member */);
1046               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1047                                      ip4_header_t /* cheat */,
1048                                      length /* changed member */);
1049               tcp0->checksum = ip_csum_fold(sum0);
1050             }
1051           else
1052             {
1053               old_port0 = udp0->src_port;
1054               udp0->src_port = s0->out2in.port;
1055               udp0->checksum = 0;
1056             }
1057
1058           /* Hairpinning */
1059           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1060
1061           /* Accounting */
1062           s0->last_heard = now;
1063           s0->total_pkts++;
1064           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1065           /* Per-user LRU list maintenance for dynamic translation */
1066           if (!snat_is_session_static (s0))
1067             {
1068               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1069                                  s0->per_user_index);
1070               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1071                                   s0->per_user_list_head_index,
1072                                   s0->per_user_index);
1073             }
1074         trace00:
1075
1076           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1077                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1078             {
1079               snat_in2out_trace_t *t =
1080                  vlib_add_trace (vm, node, b0, sizeof (*t));
1081               t->is_slow_path = is_slow_path;
1082               t->sw_if_index = sw_if_index0;
1083               t->next_index = next0;
1084                   t->session_index = ~0;
1085               if (s0)
1086                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1087             }
1088
1089           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1090
1091           ip1 = vlib_buffer_get_current (b1);
1092           udp1 = ip4_next_header (ip1);
1093           tcp1 = (tcp_header_t *) udp1;
1094           icmp1 = (icmp46_header_t *) udp1;
1095
1096           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1097           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1098                                    sw_if_index1);
1099
1100           if (PREDICT_FALSE(ip1->ttl == 1))
1101             {
1102               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1103               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1104                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1105                                            0);
1106               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1107               goto trace01;
1108             }
1109
1110           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1111
1112           /* Next configured feature, probably ip4-lookup */
1113           if (is_slow_path)
1114             {
1115               if (PREDICT_FALSE (proto1 == ~0))
1116                 goto trace01;
1117
1118               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1119                 {
1120                   next1 = icmp_in2out_slow_path
1121                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1122                      next1, now, thread_index, &s1);
1123                   goto trace01;
1124                 }
1125             }
1126           else
1127             {
1128               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1129                 {
1130                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1131                   goto trace01;
1132                 }
1133             }
1134
1135           key1.addr = ip1->src_address;
1136           key1.port = udp1->src_port;
1137           key1.protocol = proto1;
1138           key1.fib_index = rx_fib_index1;
1139
1140           kv1.key = key1.as_u64;
1141
1142             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
1143             {
1144               if (is_slow_path)
1145                 {
1146                   if (PREDICT_FALSE(snat_not_translate(sm, rt, sw_if_index1, ip1,
1147                       proto1, rx_fib_index1)))
1148                     goto trace01;
1149
1150                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1151                                      &s1, node, next1, thread_index);
1152                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1153                     goto trace01;
1154                 }
1155               else
1156                 {
1157                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1158                   goto trace01;
1159                 }
1160             }
1161           else
1162             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1163                                     value1.value);
1164
1165           old_addr1 = ip1->src_address.as_u32;
1166           ip1->src_address = s1->out2in.addr;
1167           new_addr1 = ip1->src_address.as_u32;
1168           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1169
1170           sum1 = ip1->checksum;
1171           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1172                                  ip4_header_t,
1173                                  src_address /* changed member */);
1174           ip1->checksum = ip_csum_fold (sum1);
1175
1176           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1177             {
1178               old_port1 = tcp1->src_port;
1179               tcp1->src_port = s1->out2in.port;
1180               new_port1 = tcp1->src_port;
1181
1182               sum1 = tcp1->checksum;
1183               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1184                                      ip4_header_t,
1185                                      dst_address /* changed member */);
1186               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1187                                      ip4_header_t /* cheat */,
1188                                      length /* changed member */);
1189               tcp1->checksum = ip_csum_fold(sum1);
1190             }
1191           else
1192             {
1193               old_port1 = udp1->src_port;
1194               udp1->src_port = s1->out2in.port;
1195               udp1->checksum = 0;
1196             }
1197
1198           /* Hairpinning */
1199           snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1200
1201           /* Accounting */
1202           s1->last_heard = now;
1203           s1->total_pkts++;
1204           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1205           /* Per-user LRU list maintenance for dynamic translation */
1206           if (!snat_is_session_static (s1))
1207             {
1208               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1209                                  s1->per_user_index);
1210               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1211                                   s1->per_user_list_head_index,
1212                                   s1->per_user_index);
1213             }
1214         trace01:
1215
1216           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1217                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1218             {
1219               snat_in2out_trace_t *t =
1220                  vlib_add_trace (vm, node, b1, sizeof (*t));
1221               t->sw_if_index = sw_if_index1;
1222               t->next_index = next1;
1223               t->session_index = ~0;
1224               if (s1)
1225                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1226             }
1227
1228           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1229
1230           /* verify speculative enqueues, maybe switch current next frame */
1231           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1232                                            to_next, n_left_to_next,
1233                                            bi0, bi1, next0, next1);
1234         }
1235
1236       while (n_left_from > 0 && n_left_to_next > 0)
1237         {
1238           u32 bi0;
1239           vlib_buffer_t * b0;
1240           u32 next0;
1241           u32 sw_if_index0;
1242           ip4_header_t * ip0;
1243           ip_csum_t sum0;
1244           u32 new_addr0, old_addr0;
1245           u16 old_port0, new_port0;
1246           udp_header_t * udp0;
1247           tcp_header_t * tcp0;
1248           icmp46_header_t * icmp0;
1249           snat_session_key_t key0;
1250           u32 rx_fib_index0;
1251           u32 proto0;
1252           snat_session_t * s0 = 0;
1253           clib_bihash_kv_8_8_t kv0, value0;
1254
1255           /* speculatively enqueue b0 to the current next frame */
1256           bi0 = from[0];
1257           to_next[0] = bi0;
1258           from += 1;
1259           to_next += 1;
1260           n_left_from -= 1;
1261           n_left_to_next -= 1;
1262
1263           b0 = vlib_get_buffer (vm, bi0);
1264           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1265
1266           ip0 = vlib_buffer_get_current (b0);
1267           udp0 = ip4_next_header (ip0);
1268           tcp0 = (tcp_header_t *) udp0;
1269           icmp0 = (icmp46_header_t *) udp0;
1270
1271           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1272           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1273                                    sw_if_index0);
1274
1275           if (PREDICT_FALSE(ip0->ttl == 1))
1276             {
1277               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1278               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1279                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1280                                            0);
1281               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1282               goto trace0;
1283             }
1284
1285           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1286
1287           /* Next configured feature, probably ip4-lookup */
1288           if (is_slow_path)
1289             {
1290               if (PREDICT_FALSE (proto0 == ~0))
1291                 goto trace0;
1292
1293               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1294                 {
1295                   next0 = icmp_in2out_slow_path
1296                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1297                      next0, now, thread_index, &s0);
1298                   goto trace0;
1299                 }
1300             }
1301           else
1302             {
1303               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1304                 {
1305                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1306                   goto trace0;
1307                 }
1308             }
1309
1310           key0.addr = ip0->src_address;
1311           key0.port = udp0->src_port;
1312           key0.protocol = proto0;
1313           key0.fib_index = rx_fib_index0;
1314
1315           kv0.key = key0.as_u64;
1316
1317           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
1318             {
1319               if (is_slow_path)
1320                 {
1321                   if (PREDICT_FALSE(snat_not_translate(sm, rt, sw_if_index0, ip0,
1322                       proto0, rx_fib_index0)))
1323                     goto trace0;
1324
1325                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1326                                      &s0, node, next0, thread_index);
1327
1328                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1329                     goto trace0;
1330                 }
1331               else
1332                 {
1333                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1334                   goto trace0;
1335                 }
1336             }
1337           else
1338             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1339                                     value0.value);
1340
1341           old_addr0 = ip0->src_address.as_u32;
1342           ip0->src_address = s0->out2in.addr;
1343           new_addr0 = ip0->src_address.as_u32;
1344           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1345
1346           sum0 = ip0->checksum;
1347           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1348                                  ip4_header_t,
1349                                  src_address /* changed member */);
1350           ip0->checksum = ip_csum_fold (sum0);
1351
1352           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1353             {
1354               old_port0 = tcp0->src_port;
1355               tcp0->src_port = s0->out2in.port;
1356               new_port0 = tcp0->src_port;
1357
1358               sum0 = tcp0->checksum;
1359               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1360                                      ip4_header_t,
1361                                      dst_address /* changed member */);
1362               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1363                                      ip4_header_t /* cheat */,
1364                                      length /* changed member */);
1365               tcp0->checksum = ip_csum_fold(sum0);
1366             }
1367           else
1368             {
1369               old_port0 = udp0->src_port;
1370               udp0->src_port = s0->out2in.port;
1371               udp0->checksum = 0;
1372             }
1373
1374           /* Hairpinning */
1375           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1376
1377           /* Accounting */
1378           s0->last_heard = now;
1379           s0->total_pkts++;
1380           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1381           /* Per-user LRU list maintenance for dynamic translation */
1382           if (!snat_is_session_static (s0))
1383             {
1384               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1385                                  s0->per_user_index);
1386               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1387                                   s0->per_user_list_head_index,
1388                                   s0->per_user_index);
1389             }
1390
1391         trace0:
1392           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1393                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1394             {
1395               snat_in2out_trace_t *t =
1396                  vlib_add_trace (vm, node, b0, sizeof (*t));
1397               t->is_slow_path = is_slow_path;
1398               t->sw_if_index = sw_if_index0;
1399               t->next_index = next0;
1400                   t->session_index = ~0;
1401               if (s0)
1402                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1403             }
1404
1405           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1406
1407           /* verify speculative enqueue, maybe switch current next frame */
1408           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1409                                            to_next, n_left_to_next,
1410                                            bi0, next0);
1411         }
1412
1413       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1414     }
1415
1416   vlib_node_increment_counter (vm, stats_node_index,
1417                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1418                                pkts_processed);
1419   return frame->n_vectors;
1420 }
1421
1422 static uword
1423 snat_in2out_fast_path_fn (vlib_main_t * vm,
1424                           vlib_node_runtime_t * node,
1425                           vlib_frame_t * frame)
1426 {
1427   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
1428 }
1429
1430 VLIB_REGISTER_NODE (snat_in2out_node) = {
1431   .function = snat_in2out_fast_path_fn,
1432   .name = "snat-in2out",
1433   .vector_size = sizeof (u32),
1434   .format_trace = format_snat_in2out_trace,
1435   .type = VLIB_NODE_TYPE_INTERNAL,
1436
1437   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1438   .error_strings = snat_in2out_error_strings,
1439
1440   .runtime_data_bytes = sizeof (snat_runtime_t),
1441
1442   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1443
1444   /* edit / add dispositions here */
1445   .next_nodes = {
1446     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1447     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1448     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1449     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1450   },
1451 };
1452
1453 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1454
1455 static uword
1456 snat_in2out_slow_path_fn (vlib_main_t * vm,
1457                           vlib_node_runtime_t * node,
1458                           vlib_frame_t * frame)
1459 {
1460   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
1461 }
1462
1463 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1464   .function = snat_in2out_slow_path_fn,
1465   .name = "snat-in2out-slowpath",
1466   .vector_size = sizeof (u32),
1467   .format_trace = format_snat_in2out_trace,
1468   .type = VLIB_NODE_TYPE_INTERNAL,
1469
1470   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1471   .error_strings = snat_in2out_error_strings,
1472
1473   .runtime_data_bytes = sizeof (snat_runtime_t),
1474
1475   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1476
1477   /* edit / add dispositions here */
1478   .next_nodes = {
1479     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1480     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1481     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1482     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1483   },
1484 };
1485
1486 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1487
1488 /**************************/
1489 /*** deterministic mode ***/
1490 /**************************/
1491 static uword
1492 snat_det_in2out_node_fn (vlib_main_t * vm,
1493                          vlib_node_runtime_t * node,
1494                          vlib_frame_t * frame)
1495 {
1496   u32 n_left_from, * from, * to_next;
1497   snat_in2out_next_t next_index;
1498   u32 pkts_processed = 0;
1499   snat_main_t * sm = &snat_main;
1500   u32 now = (u32) vlib_time_now (vm);
1501
1502   from = vlib_frame_vector_args (frame);
1503   n_left_from = frame->n_vectors;
1504   next_index = node->cached_next_index;
1505
1506   while (n_left_from > 0)
1507     {
1508       u32 n_left_to_next;
1509
1510       vlib_get_next_frame (vm, node, next_index,
1511                            to_next, n_left_to_next);
1512
1513       while (n_left_from >= 4 && n_left_to_next >= 2)
1514         {
1515           u32 bi0, bi1;
1516           vlib_buffer_t * b0, * b1;
1517           u32 next0, next1;
1518           u32 sw_if_index0, sw_if_index1;
1519           ip4_header_t * ip0, * ip1;
1520           ip_csum_t sum0, sum1;
1521           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1522           u16 old_port0, new_port0, lo_port0, i0;
1523           u16 old_port1, new_port1, lo_port1, i1;
1524           udp_header_t * udp0, * udp1;
1525           tcp_header_t * tcp0, * tcp1;
1526           u32 proto0, proto1;
1527           snat_det_out_key_t key0, key1;
1528           snat_det_map_t * dm0, * dm1;
1529           snat_det_session_t * ses0 = 0, * ses1 = 0;
1530
1531           /* Prefetch next iteration. */
1532           {
1533             vlib_buffer_t * p2, * p3;
1534
1535             p2 = vlib_get_buffer (vm, from[2]);
1536             p3 = vlib_get_buffer (vm, from[3]);
1537
1538             vlib_prefetch_buffer_header (p2, LOAD);
1539             vlib_prefetch_buffer_header (p3, LOAD);
1540
1541             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1542             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1543           }
1544
1545           /* speculatively enqueue b0 and b1 to the current next frame */
1546           to_next[0] = bi0 = from[0];
1547           to_next[1] = bi1 = from[1];
1548           from += 2;
1549           to_next += 2;
1550           n_left_from -= 2;
1551           n_left_to_next -= 2;
1552
1553           b0 = vlib_get_buffer (vm, bi0);
1554           b1 = vlib_get_buffer (vm, bi1);
1555
1556           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1557           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1558
1559           ip0 = vlib_buffer_get_current (b0);
1560           udp0 = ip4_next_header (ip0);
1561           tcp0 = (tcp_header_t *) udp0;
1562
1563           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1564
1565           if (PREDICT_FALSE(ip0->ttl == 1))
1566             {
1567               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1568               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1569                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1570                                            0);
1571               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1572               goto trace0;
1573             }
1574
1575           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
1576           if (PREDICT_FALSE(!dm0))
1577             {
1578               clib_warning("no match for internal host %U",
1579                            format_ip4_address, &ip0->src_address);
1580               next0 = SNAT_IN2OUT_NEXT_DROP;
1581               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1582               goto trace0;
1583             }
1584
1585           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
1586
1587           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src);
1588           if (PREDICT_FALSE(!ses0))
1589             {
1590               key0.ext_host_addr = ip0->dst_address;
1591               key0.ext_host_port = tcp0->dst;
1592               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
1593                 {
1594                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
1595                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
1596
1597                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
1598                     continue;
1599
1600                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
1601                   break;
1602                 }
1603               if (PREDICT_FALSE(!ses0))
1604                 {
1605                   next0 = SNAT_IN2OUT_NEXT_DROP;
1606                   b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
1607                   goto trace0;
1608                 }
1609             }
1610
1611           new_port0 = ses0->out.out_port;
1612           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1613
1614           old_addr0.as_u32 = ip0->src_address.as_u32;
1615           ip0->src_address.as_u32 = new_addr0.as_u32;
1616           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1617
1618           sum0 = ip0->checksum;
1619           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1620                                  ip4_header_t,
1621                                  src_address /* changed member */);
1622           ip0->checksum = ip_csum_fold (sum0);
1623
1624           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1625             {
1626               if (tcp0->flags & TCP_FLAG_SYN)
1627                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
1628               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
1629                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1630               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1631                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
1632               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
1633                 snat_det_ses_close(dm0, ses0);
1634               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
1635                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
1636               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
1637                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1638
1639               old_port0 = tcp0->src;
1640               tcp0->src = new_port0;
1641
1642               sum0 = tcp0->checksum;
1643               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1644                                      ip4_header_t,
1645                                      dst_address /* changed member */);
1646               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1647                                      ip4_header_t /* cheat */,
1648                                      length /* changed member */);
1649               tcp0->checksum = ip_csum_fold(sum0);
1650             }
1651           else
1652             {
1653               ses0->state = SNAT_SESSION_UDP_ACTIVE;
1654               old_port0 = udp0->src_port;
1655               udp0->src_port = new_port0;
1656               udp0->checksum = 0;
1657             }
1658
1659           switch(ses0->state)
1660             {
1661             case SNAT_SESSION_UDP_ACTIVE:
1662                 ses0->expire = now + SNAT_UDP_TIMEOUT;
1663                 break;
1664             case SNAT_SESSION_TCP_SYN_SENT:
1665             case SNAT_SESSION_TCP_FIN_WAIT:
1666             case SNAT_SESSION_TCP_CLOSE_WAIT:
1667             case SNAT_SESSION_TCP_LAST_ACK:
1668                 ses0->expire = now + SNAT_TCP_TRANSITORY_TIMEOUT;
1669                 break;
1670             case SNAT_SESSION_TCP_ESTABLISHED:
1671                 ses0->expire = now + SNAT_TCP_ESTABLISHED_TIMEOUT;
1672                 break;
1673             }
1674
1675         trace0:
1676           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1677                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1678             {
1679               snat_in2out_trace_t *t =
1680                  vlib_add_trace (vm, node, b0, sizeof (*t));
1681               t->is_slow_path = 0;
1682               t->sw_if_index = sw_if_index0;
1683               t->next_index = next0;
1684               t->session_index = ~0;
1685               if (ses0)
1686                 t->session_index = ses0 - dm0->sessions;
1687             }
1688
1689           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1690
1691           ip1 = vlib_buffer_get_current (b1);
1692           udp1 = ip4_next_header (ip1);
1693           tcp1 = (tcp_header_t *) udp1;
1694
1695           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1696
1697           if (PREDICT_FALSE(ip1->ttl == 1))
1698             {
1699               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1700               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1701                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1702                                            0);
1703               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1704               goto trace1;
1705             }
1706
1707           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
1708           if (PREDICT_FALSE(!dm1))
1709             {
1710               clib_warning("no match for internal host %U",
1711                            format_ip4_address, &ip0->src_address);
1712               next1 = SNAT_IN2OUT_NEXT_DROP;
1713               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1714               goto trace1;
1715             }
1716
1717           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
1718
1719           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src);
1720           if (PREDICT_FALSE(!ses1))
1721             {
1722               key1.ext_host_addr = ip1->dst_address;
1723               key1.ext_host_port = tcp1->dst;
1724               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
1725                 {
1726                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
1727                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
1728
1729                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
1730                     continue;
1731
1732                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
1733                   break;
1734                 }
1735               if (PREDICT_FALSE(!ses1))
1736                 {
1737                   next1 = SNAT_IN2OUT_NEXT_DROP;
1738                   b1->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
1739                   goto trace1;
1740                 }
1741             }
1742
1743           new_port1 = ses1->out.out_port;
1744           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1745
1746           old_addr1.as_u32 = ip1->src_address.as_u32;
1747           ip1->src_address.as_u32 = new_addr1.as_u32;
1748           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1749
1750           sum1 = ip1->checksum;
1751           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1752                                  ip4_header_t,
1753                                  src_address /* changed member */);
1754           ip1->checksum = ip_csum_fold (sum1);
1755
1756           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1757             {
1758               if (tcp1->flags & TCP_FLAG_SYN)
1759                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
1760               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
1761                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
1762               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1763                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
1764               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
1765                 snat_det_ses_close(dm1, ses1);
1766               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
1767                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
1768               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
1769                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
1770
1771               old_port1 = tcp1->src;
1772               tcp1->src = new_port1;
1773
1774               sum1 = tcp1->checksum;
1775               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1776                                      ip4_header_t,
1777                                      dst_address /* changed member */);
1778               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1779                                      ip4_header_t /* cheat */,
1780                                      length /* changed member */);
1781               tcp1->checksum = ip_csum_fold(sum1);
1782             }
1783           else
1784             {
1785               ses1->state = SNAT_SESSION_UDP_ACTIVE;
1786               old_port1 = udp1->src_port;
1787               udp1->src_port = new_port1;
1788               udp1->checksum = 0;
1789             }
1790
1791           switch(ses1->state)
1792             {
1793             case SNAT_SESSION_UDP_ACTIVE:
1794                 ses1->expire = now + SNAT_UDP_TIMEOUT;
1795                 break;
1796             case SNAT_SESSION_TCP_SYN_SENT:
1797             case SNAT_SESSION_TCP_FIN_WAIT:
1798             case SNAT_SESSION_TCP_CLOSE_WAIT:
1799             case SNAT_SESSION_TCP_LAST_ACK:
1800                 ses1->expire = now + SNAT_TCP_TRANSITORY_TIMEOUT;
1801                 break;
1802             case SNAT_SESSION_TCP_ESTABLISHED:
1803                 ses1->expire = now + SNAT_TCP_ESTABLISHED_TIMEOUT;
1804                 break;
1805             }
1806
1807         trace1:
1808           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1809                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1810             {
1811               snat_in2out_trace_t *t =
1812                  vlib_add_trace (vm, node, b1, sizeof (*t));
1813               t->is_slow_path = 0;
1814               t->sw_if_index = sw_if_index1;
1815               t->next_index = next1;
1816               t->session_index = ~0;
1817               if (ses1)
1818                 t->session_index = ses1 - dm1->sessions;
1819             }
1820
1821           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1822
1823           /* verify speculative enqueues, maybe switch current next frame */
1824           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1825                                            to_next, n_left_to_next,
1826                                            bi0, bi1, next0, next1);
1827          }
1828
1829       while (n_left_from > 0 && n_left_to_next > 0)
1830         {
1831           u32 bi0;
1832           vlib_buffer_t * b0;
1833           u32 next0;
1834           u32 sw_if_index0;
1835           ip4_header_t * ip0;
1836           ip_csum_t sum0;
1837           ip4_address_t new_addr0, old_addr0;
1838           u16 old_port0, new_port0, lo_port0, i0;
1839           udp_header_t * udp0;
1840           tcp_header_t * tcp0;
1841           u32 proto0;
1842           snat_det_out_key_t key0;
1843           snat_det_map_t * dm0;
1844           snat_det_session_t * ses0 = 0;
1845
1846           /* speculatively enqueue b0 to the current next frame */
1847           bi0 = from[0];
1848           to_next[0] = bi0;
1849           from += 1;
1850           to_next += 1;
1851           n_left_from -= 1;
1852           n_left_to_next -= 1;
1853
1854           b0 = vlib_get_buffer (vm, bi0);
1855           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1856
1857           ip0 = vlib_buffer_get_current (b0);
1858           udp0 = ip4_next_header (ip0);
1859           tcp0 = (tcp_header_t *) udp0;
1860
1861           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1862
1863           if (PREDICT_FALSE(ip0->ttl == 1))
1864             {
1865               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1866               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1867                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1868                                            0);
1869               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1870               goto trace00;
1871             }
1872
1873           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
1874           if (PREDICT_FALSE(!dm0))
1875             {
1876               clib_warning("no match for internal host %U",
1877                            format_ip4_address, &ip0->src_address);
1878               next0 = SNAT_IN2OUT_NEXT_DROP;
1879               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1880               goto trace00;
1881             }
1882
1883           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
1884
1885           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src);
1886           if (PREDICT_FALSE(!ses0))
1887             {
1888               key0.ext_host_addr = ip0->dst_address;
1889               key0.ext_host_port = tcp0->dst;
1890               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
1891                 {
1892                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
1893                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
1894
1895                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
1896                     continue;
1897
1898                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
1899                   break;
1900                 }
1901               if (PREDICT_FALSE(!ses0))
1902                 {
1903                   next0 = SNAT_IN2OUT_NEXT_DROP;
1904                   b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
1905                   goto trace00;
1906                 }
1907             }
1908
1909           new_port0 = ses0->out.out_port;
1910           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1911
1912           old_addr0.as_u32 = ip0->src_address.as_u32;
1913           ip0->src_address.as_u32 = new_addr0.as_u32;
1914           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1915
1916           sum0 = ip0->checksum;
1917           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1918                                  ip4_header_t,
1919                                  src_address /* changed member */);
1920           ip0->checksum = ip_csum_fold (sum0);
1921
1922           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1923             {
1924               if (tcp0->flags & TCP_FLAG_SYN)
1925                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
1926               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
1927                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1928               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1929                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
1930               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
1931                 snat_det_ses_close(dm0, ses0);
1932               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
1933                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
1934               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
1935                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1936
1937               old_port0 = tcp0->src;
1938               tcp0->src = new_port0;
1939
1940               sum0 = tcp0->checksum;
1941               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1942                                      ip4_header_t,
1943                                      dst_address /* changed member */);
1944               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1945                                      ip4_header_t /* cheat */,
1946                                      length /* changed member */);
1947               tcp0->checksum = ip_csum_fold(sum0);
1948             }
1949           else
1950             {
1951               ses0->state = SNAT_SESSION_UDP_ACTIVE;
1952               old_port0 = udp0->src_port;
1953               udp0->src_port = new_port0;
1954               udp0->checksum = 0;
1955             }
1956
1957           switch(ses0->state)
1958             {
1959             case SNAT_SESSION_UDP_ACTIVE:
1960                 ses0->expire = now + SNAT_UDP_TIMEOUT;
1961                 break;
1962             case SNAT_SESSION_TCP_SYN_SENT:
1963             case SNAT_SESSION_TCP_FIN_WAIT:
1964             case SNAT_SESSION_TCP_CLOSE_WAIT:
1965             case SNAT_SESSION_TCP_LAST_ACK:
1966                 ses0->expire = now + SNAT_TCP_TRANSITORY_TIMEOUT;
1967                 break;
1968             case SNAT_SESSION_TCP_ESTABLISHED:
1969                 ses0->expire = now + SNAT_TCP_ESTABLISHED_TIMEOUT;
1970                 break;
1971             }
1972
1973         trace00:
1974           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1975                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1976             {
1977               snat_in2out_trace_t *t =
1978                  vlib_add_trace (vm, node, b0, sizeof (*t));
1979               t->is_slow_path = 0;
1980               t->sw_if_index = sw_if_index0;
1981               t->next_index = next0;
1982               t->session_index = ~0;
1983               if (ses0)
1984                 t->session_index = ses0 - dm0->sessions;
1985             }
1986
1987           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1988
1989           /* verify speculative enqueue, maybe switch current next frame */
1990           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1991                                            to_next, n_left_to_next,
1992                                            bi0, next0);
1993         }
1994
1995       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1996     }
1997
1998   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
1999                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2000                                pkts_processed);
2001   return frame->n_vectors;
2002 }
2003
2004 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2005   .function = snat_det_in2out_node_fn,
2006   .name = "snat-det-in2out",
2007   .vector_size = sizeof (u32),
2008   .format_trace = format_snat_in2out_trace,
2009   .type = VLIB_NODE_TYPE_INTERNAL,
2010
2011   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2012   .error_strings = snat_in2out_error_strings,
2013
2014   .runtime_data_bytes = sizeof (snat_runtime_t),
2015
2016   .n_next_nodes = 3,
2017
2018   /* edit / add dispositions here */
2019   .next_nodes = {
2020     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2021     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2022     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2023   },
2024 };
2025
2026 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2027
2028 /**********************/
2029 /*** worker handoff ***/
2030 /**********************/
2031 static uword
2032 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
2033                                vlib_node_runtime_t * node,
2034                                vlib_frame_t * frame)
2035 {
2036   snat_main_t *sm = &snat_main;
2037   vlib_thread_main_t *tm = vlib_get_thread_main ();
2038   u32 n_left_from, *from, *to_next = 0;
2039   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2040   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2041     = 0;
2042   vlib_frame_queue_elt_t *hf = 0;
2043   vlib_frame_t *f = 0;
2044   int i;
2045   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2046   u32 next_worker_index = 0;
2047   u32 current_worker_index = ~0;
2048   u32 thread_index = vlib_get_thread_index ();
2049
2050   ASSERT (vec_len (sm->workers));
2051
2052   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2053     {
2054       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2055
2056       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2057                                sm->first_worker_index + sm->num_workers - 1,
2058                                (vlib_frame_queue_t *) (~0));
2059     }
2060
2061   from = vlib_frame_vector_args (frame);
2062   n_left_from = frame->n_vectors;
2063
2064   while (n_left_from > 0)
2065     {
2066       u32 bi0;
2067       vlib_buffer_t *b0;
2068       u32 sw_if_index0;
2069       u32 rx_fib_index0;
2070       ip4_header_t * ip0;
2071       u8 do_handoff;
2072
2073       bi0 = from[0];
2074       from += 1;
2075       n_left_from -= 1;
2076
2077       b0 = vlib_get_buffer (vm, bi0);
2078
2079       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2080       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2081
2082       ip0 = vlib_buffer_get_current (b0);
2083
2084       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
2085
2086       if (PREDICT_FALSE (next_worker_index != thread_index))
2087         {
2088           do_handoff = 1;
2089
2090           if (next_worker_index != current_worker_index)
2091             {
2092               if (hf)
2093                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2094
2095               hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index,
2096                                                       next_worker_index,
2097                                                       handoff_queue_elt_by_worker_index);
2098
2099               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2100               to_next_worker = &hf->buffer_index[hf->n_vectors];
2101               current_worker_index = next_worker_index;
2102             }
2103
2104           /* enqueue to correct worker thread */
2105           to_next_worker[0] = bi0;
2106           to_next_worker++;
2107           n_left_to_next_worker--;
2108
2109           if (n_left_to_next_worker == 0)
2110             {
2111               hf->n_vectors = VLIB_FRAME_SIZE;
2112               vlib_put_frame_queue_elt (hf);
2113               current_worker_index = ~0;
2114               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2115               hf = 0;
2116             }
2117         }
2118       else
2119         {
2120           do_handoff = 0;
2121           /* if this is 1st frame */
2122           if (!f)
2123             {
2124               f = vlib_get_frame_to_node (vm, sm->in2out_node_index);
2125               to_next = vlib_frame_vector_args (f);
2126             }
2127
2128           to_next[0] = bi0;
2129           to_next += 1;
2130           f->n_vectors++;
2131         }
2132
2133       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2134                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2135         {
2136           snat_in2out_worker_handoff_trace_t *t =
2137             vlib_add_trace (vm, node, b0, sizeof (*t));
2138           t->next_worker_index = next_worker_index;
2139           t->do_handoff = do_handoff;
2140         }
2141     }
2142
2143   if (f)
2144     vlib_put_frame_to_node (vm, sm->in2out_node_index, f);
2145
2146   if (hf)
2147     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2148
2149   /* Ship frames to the worker nodes */
2150   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2151     {
2152       if (handoff_queue_elt_by_worker_index[i])
2153         {
2154           hf = handoff_queue_elt_by_worker_index[i];
2155           /*
2156            * It works better to let the handoff node
2157            * rate-adapt, always ship the handoff queue element.
2158            */
2159           if (1 || hf->n_vectors == hf->last_n_vectors)
2160             {
2161               vlib_put_frame_queue_elt (hf);
2162               handoff_queue_elt_by_worker_index[i] = 0;
2163             }
2164           else
2165             hf->last_n_vectors = hf->n_vectors;
2166         }
2167       congested_handoff_queue_by_worker_index[i] =
2168         (vlib_frame_queue_t *) (~0);
2169     }
2170   hf = 0;
2171   current_worker_index = ~0;
2172   return frame->n_vectors;
2173 }
2174
2175 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
2176   .function = snat_in2out_worker_handoff_fn,
2177   .name = "snat-in2out-worker-handoff",
2178   .vector_size = sizeof (u32),
2179   .format_trace = format_snat_in2out_worker_handoff_trace,
2180   .type = VLIB_NODE_TYPE_INTERNAL,
2181
2182   .n_next_nodes = 1,
2183
2184   .next_nodes = {
2185     [0] = "error-drop",
2186   },
2187 };
2188
2189 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn);
2190
2191 static uword
2192 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
2193                                 vlib_node_runtime_t * node,
2194                                 vlib_frame_t * frame)
2195 {
2196   u32 n_left_from, * from, * to_next;
2197   snat_in2out_next_t next_index;
2198   u32 pkts_processed = 0;
2199   snat_main_t * sm = &snat_main;
2200   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
2201   u32 stats_node_index;
2202
2203   stats_node_index = snat_in2out_fast_node.index;
2204
2205   from = vlib_frame_vector_args (frame);
2206   n_left_from = frame->n_vectors;
2207   next_index = node->cached_next_index;
2208
2209   while (n_left_from > 0)
2210     {
2211       u32 n_left_to_next;
2212
2213       vlib_get_next_frame (vm, node, next_index,
2214                            to_next, n_left_to_next);
2215
2216       while (n_left_from > 0 && n_left_to_next > 0)
2217         {
2218           u32 bi0;
2219           vlib_buffer_t * b0;
2220           u32 next0;
2221           u32 sw_if_index0;
2222           ip4_header_t * ip0;
2223           ip_csum_t sum0;
2224           u32 new_addr0, old_addr0;
2225           u16 old_port0, new_port0;
2226           udp_header_t * udp0;
2227           tcp_header_t * tcp0;
2228           icmp46_header_t * icmp0;
2229           snat_session_key_t key0, sm0;
2230           u32 proto0;
2231           u32 rx_fib_index0;
2232
2233           /* speculatively enqueue b0 to the current next frame */
2234           bi0 = from[0];
2235           to_next[0] = bi0;
2236           from += 1;
2237           to_next += 1;
2238           n_left_from -= 1;
2239           n_left_to_next -= 1;
2240
2241           b0 = vlib_get_buffer (vm, bi0);
2242           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2243
2244           ip0 = vlib_buffer_get_current (b0);
2245           udp0 = ip4_next_header (ip0);
2246           tcp0 = (tcp_header_t *) udp0;
2247           icmp0 = (icmp46_header_t *) udp0;
2248
2249           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2250           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2251
2252           if (PREDICT_FALSE(ip0->ttl == 1))
2253             {
2254               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2255               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2256                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2257                                            0);
2258               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2259               goto trace0;
2260             }
2261
2262           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2263
2264           if (PREDICT_FALSE (proto0 == ~0))
2265               goto trace0;
2266
2267           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2268             {
2269               if (PREDICT_FALSE(snat_not_translate(sm, rt, sw_if_index0, ip0,
2270                   proto0, rx_fib_index0)))
2271                 goto trace0;
2272
2273               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2274                                   rx_fib_index0, node, next0, ~0, 0);
2275               goto trace0;
2276             }
2277
2278           key0.addr = ip0->src_address;
2279           key0.port = udp0->src_port;
2280           key0.fib_index = rx_fib_index0;
2281
2282           if (snat_static_mapping_match(sm, key0, &sm0, 0))
2283             {
2284               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2285               next0= SNAT_IN2OUT_NEXT_DROP;
2286               goto trace0;
2287             }
2288
2289           new_addr0 = sm0.addr.as_u32;
2290           new_port0 = sm0.port;
2291           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2292           old_addr0 = ip0->src_address.as_u32;
2293           ip0->src_address.as_u32 = new_addr0;
2294
2295           sum0 = ip0->checksum;
2296           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2297                                  ip4_header_t,
2298                                  src_address /* changed member */);
2299           ip0->checksum = ip_csum_fold (sum0);
2300
2301           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2302             {
2303               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2304                 {
2305                   old_port0 = tcp0->src_port;
2306                   tcp0->src_port = new_port0;
2307
2308                   sum0 = tcp0->checksum;
2309                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2310                                          ip4_header_t,
2311                                          dst_address /* changed member */);
2312                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2313                                          ip4_header_t /* cheat */,
2314                                          length /* changed member */);
2315                   tcp0->checksum = ip_csum_fold(sum0);
2316                 }
2317               else
2318                 {
2319                   old_port0 = udp0->src_port;
2320                   udp0->src_port = new_port0;
2321                   udp0->checksum = 0;
2322                 }
2323             }
2324           else
2325             {
2326               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2327                 {
2328                   sum0 = tcp0->checksum;
2329                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2330                                          ip4_header_t,
2331                                          dst_address /* changed member */);
2332                   tcp0->checksum = ip_csum_fold(sum0);
2333                 }
2334             }
2335
2336           /* Hairpinning */
2337           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2338
2339         trace0:
2340           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2341                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2342             {
2343               snat_in2out_trace_t *t =
2344                  vlib_add_trace (vm, node, b0, sizeof (*t));
2345               t->sw_if_index = sw_if_index0;
2346               t->next_index = next0;
2347             }
2348
2349           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2350
2351           /* verify speculative enqueue, maybe switch current next frame */
2352           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2353                                            to_next, n_left_to_next,
2354                                            bi0, next0);
2355         }
2356
2357       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2358     }
2359
2360   vlib_node_increment_counter (vm, stats_node_index,
2361                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2362                                pkts_processed);
2363   return frame->n_vectors;
2364 }
2365
2366
2367 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2368   .function = snat_in2out_fast_static_map_fn,
2369   .name = "snat-in2out-fast",
2370   .vector_size = sizeof (u32),
2371   .format_trace = format_snat_in2out_fast_trace,
2372   .type = VLIB_NODE_TYPE_INTERNAL,
2373
2374   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2375   .error_strings = snat_in2out_error_strings,
2376
2377   .runtime_data_bytes = sizeof (snat_runtime_t),
2378
2379   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2380
2381   /* edit / add dispositions here */
2382   .next_nodes = {
2383     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2384     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2385     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
2386     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2387   },
2388 };
2389
2390 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);