src/plugins/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25 #include <snat/snat_ipfix_logging.h>
  26 #include <snat/snat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91
  92 #define foreach_snat_in2out_error                       \
  93 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  94 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  95 _(OUT_OF_PORTS, "Out of ports")                         \
  96 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  97 _(BAD_ICMP_TYPE, "icmp type not echo-request")          \
  98 _(NO_TRANSLATION, "No translation")
  99
 100 typedef enum {
 101 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 102   foreach_snat_in2out_error
 103 #undef _
 104   SNAT_IN2OUT_N_ERROR,
 105 } snat_in2out_error_t;
 106
 107 static char * snat_in2out_error_strings[] = {
 108 #define _(sym,string) string,
 109   foreach_snat_in2out_error
 110 #undef _
 111 };
 112
 113 typedef enum {
 114   SNAT_IN2OUT_NEXT_LOOKUP,
 115   SNAT_IN2OUT_NEXT_DROP,
 116   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 117   SNAT_IN2OUT_NEXT_SLOW_PATH,
 118   SNAT_IN2OUT_N_NEXT,
 119 } snat_in2out_next_t;
 120
 121 /**
 122  * @brief Check if packet should be translated
 123  *
 124  * Packets aimed at outside interface and external addresss with active session
 125  * should be translated.
 126  *
 127  * @param sm            SNAT main
 128  * @param rt            SNAT runtime data
 129  * @param sw_if_index0  index of the inside interface
 130  * @param ip0           IPv4 header
 131  * @param proto0        SNAT protocol
 132  * @param rx_fib_index0 RX FIB index
 133  *
 134  * @returns 0 if packet should be translated otherwise 1
 135  */
 136 static inline int
 137 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 138                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 139                          u32 rx_fib_index0)
 140 {
 141   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 142   fib_prefix_t pfx = {
 143     .fp_proto = FIB_PROTOCOL_IP4,
 144     .fp_len = 32,
 145     .fp_addr = {
 146         .ip4.as_u32 = ip0->dst_address.as_u32,
 147     },
 148   };
 149
 150   /* Don't NAT packet aimed at the intfc address */
 151   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 152                                       ip0->dst_address.as_u32)))
 153     return 1;
 154
 155   fei = fib_table_lookup (rx_fib_index0, &pfx);
 156   if (FIB_NODE_INDEX_INVALID != fei)
 157     {
 158       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 159       if (sw_if_index == ~0)
 160         {
 161           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 162           if (FIB_NODE_INDEX_INVALID != fei)
 163             sw_if_index = fib_entry_get_resolving_interface (fei);
 164         }
 165       snat_interface_t *i;
 166       pool_foreach (i, sm->interfaces,
 167       ({
 168         /* NAT packet aimed at outside interface */
 169         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 170           return 0;
 171       }));
 172     }
 173
 174   return 1;
 175 }
 176
 177 static inline int
 178 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 179                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                     u32 rx_fib_index0)
 181 {
 182   udp_header_t * udp0 = ip4_next_header (ip0);
 183   snat_session_key_t key0, sm0;
 184   clib_bihash_kv_8_8_t kv0, value0;
 185
 186   key0.addr = ip0->dst_address;
 187   key0.port = udp0->dst_port;
 188   key0.protocol = proto0;
 189   key0.fib_index = sm->outside_fib_index;
 190   kv0.key = key0.as_u64;
 191
 192   /* NAT packet aimed at external address if */
 193   /* has active sessions */
 194   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 195     {
 196       /* or is static mappings */
 197       if (!snat_static_mapping_match(sm, key0, &sm0, 1))
 198         return 0;
 199     }
 200   else
 201     return 0;
 202
 203   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 204                                  rx_fib_index0);
 205 }
 206
 207 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 208                       ip4_header_t * ip0,
 209                       u32 rx_fib_index0,
 210                       snat_session_key_t * key0,
 211                       snat_session_t ** sessionp,
 212                       vlib_node_runtime_t * node,
 213                       u32 next0,
 214                       u32 thread_index)
 215 {
 216   snat_user_t *u;
 217   snat_user_key_t user_key;
 218   snat_session_t *s;
 219   clib_bihash_kv_8_8_t kv0, value0;
 220   u32 oldest_per_user_translation_list_index;
 221   dlist_elt_t * oldest_per_user_translation_list_elt;
 222   dlist_elt_t * per_user_translation_list_elt;
 223   dlist_elt_t * per_user_list_head_elt;
 224   u32 session_index;
 225   snat_session_key_t key1;
 226   u32 address_index = ~0;
 227   u32 outside_fib_index;
 228   uword * p;
 229   snat_worker_key_t worker_by_out_key;
 230
 231   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 232   if (! p)
 233     {
 234       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 235       return SNAT_IN2OUT_NEXT_DROP;
 236     }
 237   outside_fib_index = p[0];
 238
 239   key1.protocol = key0->protocol;
 240   user_key.addr = ip0->src_address;
 241   user_key.fib_index = rx_fib_index0;
 242   kv0.key = user_key.as_u64;
 243
 244   /* Ever heard of the "user" = src ip4 address before? */
 245   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 246     {
 247       /* no, make a new one */
 248       pool_get (sm->per_thread_data[thread_index].users, u);
 249       memset (u, 0, sizeof (*u));
 250       u->addr = ip0->src_address;
 251       u->fib_index = rx_fib_index0;
 252
 253       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 254
 255       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 256         sm->per_thread_data[thread_index].list_pool;
 257
 258       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 259                        u->sessions_per_user_list_head_index);
 260
 261       kv0.value = u - sm->per_thread_data[thread_index].users;
 262
 263       /* add user */
 264       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 265     }
 266   else
 267     {
 268       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 269                              value0.value);
 270     }
 271
 272   /* Over quota? Recycle the least recently used dynamic translation */
 273   if (u->nsessions >= sm->max_translations_per_user)
 274     {
 275       /* Remove the oldest dynamic translation */
 276       do {
 277           oldest_per_user_translation_list_index =
 278             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 279                                     u->sessions_per_user_list_head_index);
 280
 281           ASSERT (oldest_per_user_translation_list_index != ~0);
 282
 283           /* add it back to the end of the LRU list */
 284           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 285                               u->sessions_per_user_list_head_index,
 286                               oldest_per_user_translation_list_index);
 287           /* Get the list element */
 288           oldest_per_user_translation_list_elt =
 289             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 290                                oldest_per_user_translation_list_index);
 291
 292           /* Get the session index from the list element */
 293           session_index = oldest_per_user_translation_list_elt->value;
 294
 295           /* Get the session */
 296           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 297                                  session_index);
 298       } while (snat_is_session_static (s));
 299
 300       /* Remove in2out, out2in keys */
 301       kv0.key = s->in2out.as_u64;
 302       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 303           clib_warning ("in2out key delete failed");
 304       kv0.key = s->out2in.as_u64;
 305       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 306           clib_warning ("out2in key delete failed");
 307
 308       /* log NAT event */
 309       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 310                                           s->out2in.addr.as_u32,
 311                                           s->in2out.protocol,
 312                                           s->in2out.port,
 313                                           s->out2in.port,
 314                                           s->in2out.fib_index);
 315
 316       snat_free_outside_address_and_port
 317         (sm, &s->out2in, s->outside_address_index);
 318       s->outside_address_index = ~0;
 319
 320       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 321                                                &address_index))
 322         {
 323           ASSERT(0);
 324
 325           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 326           return SNAT_IN2OUT_NEXT_DROP;
 327         }
 328       s->outside_address_index = address_index;
 329     }
 330   else
 331     {
 332       u8 static_mapping = 1;
 333
 334       /* First try to match static mapping by local address and port */
 335       if (snat_static_mapping_match (sm, *key0, &key1, 0))
 336         {
 337           static_mapping = 0;
 338           /* Try to create dynamic translation */
 339           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 340                                                    &address_index))
 341             {
 342               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 343               return SNAT_IN2OUT_NEXT_DROP;
 344             }
 345         }
 346
 347       /* Create a new session */
 348       pool_get (sm->per_thread_data[thread_index].sessions, s);
 349       memset (s, 0, sizeof (*s));
 350
 351       s->outside_address_index = address_index;
 352
 353       if (static_mapping)
 354         {
 355           u->nstaticsessions++;
 356           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 357         }
 358       else
 359         {
 360           u->nsessions++;
 361         }
 362
 363       /* Create list elts */
 364       pool_get (sm->per_thread_data[thread_index].list_pool,
 365                 per_user_translation_list_elt);
 366       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 367                        per_user_translation_list_elt -
 368                        sm->per_thread_data[thread_index].list_pool);
 369
 370       per_user_translation_list_elt->value =
 371         s - sm->per_thread_data[thread_index].sessions;
 372       s->per_user_index = per_user_translation_list_elt -
 373                           sm->per_thread_data[thread_index].list_pool;
 374       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 375
 376       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 377                           s->per_user_list_head_index,
 378                           per_user_translation_list_elt -
 379                           sm->per_thread_data[thread_index].list_pool);
 380    }
 381
 382   s->in2out = *key0;
 383   s->out2in = key1;
 384   s->out2in.protocol = key0->protocol;
 385   s->out2in.fib_index = outside_fib_index;
 386   *sessionp = s;
 387
 388   /* Add to translation hashes */
 389   kv0.key = s->in2out.as_u64;
 390   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 391   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 392       clib_warning ("in2out key add failed");
 393
 394   kv0.key = s->out2in.as_u64;
 395   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 396
 397   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 398       clib_warning ("out2in key add failed");
 399
 400   /* Add to translated packets worker lookup */
 401   worker_by_out_key.addr = s->out2in.addr;
 402   worker_by_out_key.port = s->out2in.port;
 403   worker_by_out_key.fib_index = s->out2in.fib_index;
 404   kv0.key = worker_by_out_key.as_u64;
 405   kv0.value = thread_index;
 406   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 407
 408   /* log NAT event */
 409   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 410                                       s->out2in.addr.as_u32,
 411                                       s->in2out.protocol,
 412                                       s->in2out.port,
 413                                       s->out2in.port,
 414                                       s->in2out.fib_index);
 415   return next0;
 416 }
 417
 418 static_always_inline
 419 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 420                                  snat_session_key_t *p_key0)
 421 {
 422   icmp46_header_t *icmp0;
 423   snat_session_key_t key0;
 424   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 425   ip4_header_t *inner_ip0 = 0;
 426   void *l4_header = 0;
 427   icmp46_header_t *inner_icmp0;
 428
 429   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 430   echo0 = (icmp_echo_header_t *)(icmp0+1);
 431
 432   if (!icmp_is_error_message (icmp0))
 433     {
 434       key0.protocol = SNAT_PROTOCOL_ICMP;
 435       key0.addr = ip0->src_address;
 436       key0.port = echo0->identifier;
 437     }
 438   else
 439     {
 440       inner_ip0 = (ip4_header_t *)(echo0+1);
 441       l4_header = ip4_next_header (inner_ip0);
 442       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 443       key0.addr = inner_ip0->dst_address;
 444       switch (key0.protocol)
 445         {
 446         case SNAT_PROTOCOL_ICMP:
 447           inner_icmp0 = (icmp46_header_t*)l4_header;
 448           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 449           key0.port = inner_echo0->identifier;
 450           break;
 451         case SNAT_PROTOCOL_UDP:
 452         case SNAT_PROTOCOL_TCP:
 453           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 454           break;
 455         default:
 456           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 457         }
 458     }
 459   *p_key0 = key0;
 460   return -1; /* success */
 461 }
 462
 463 /**
 464  * Get address and port values to be used for packet SNAT translation
 465  * and create session if needed
 466  *
 467  * @param[in,out] sm             SNAT main
 468  * @param[in,out] node           SNAT node runtime
 469  * @param[in] thread_index       thread index
 470  * @param[in,out] b0             buffer containing packet to be translated
 471  * @param[out] p_proto           protocol used for matching
 472  * @param[out] p_value           address and port after NAT translation
 473  * @param[out] p_dont_translate  if packet should not be translated
 474  * @param d                      optional parameter
 475  */
 476 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 477                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 478                            snat_session_key_t *p_value,
 479                            u8 *p_dont_translate, void *d)
 480 {
 481   ip4_header_t *ip0;
 482   icmp46_header_t *icmp0;
 483   u32 sw_if_index0;
 484   u32 rx_fib_index0;
 485   snat_session_key_t key0;
 486   snat_session_t *s0 = 0;
 487   u8 dont_translate = 0;
 488   clib_bihash_kv_8_8_t kv0, value0;
 489   u32 next0 = ~0;
 490   int err;
 491
 492   ip0 = vlib_buffer_get_current (b0);
 493   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 494   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 495   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 496
 497   err = icmp_get_key (ip0, &key0);
 498   if (err != -1)
 499     {
 500       b0->error = node->errors[err];
 501       next0 = SNAT_IN2OUT_NEXT_DROP;
 502       goto out;
 503     }
 504   key0.fib_index = rx_fib_index0;
 505
 506   kv0.key = key0.as_u64;
 507
 508   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 509     {
 510       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 511           IP_PROTOCOL_ICMP, rx_fib_index0)))
 512         {
 513           dont_translate = 1;
 514           goto out;
 515         }
 516
 517       if (icmp_is_error_message (icmp0))
 518         {
 519           next0 = SNAT_IN2OUT_NEXT_DROP;
 520           goto out;
 521         }
 522
 523       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 524                          &s0, node, next0, thread_index);
 525
 526       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 527         goto out;
 528     }
 529   else
 530     s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 531                             value0.value);
 532
 533   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 534                     !icmp_is_error_message (icmp0)))
 535     {
 536       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 537       next0 = SNAT_IN2OUT_NEXT_DROP;
 538       goto out;
 539     }
 540
 541 out:
 542   *p_proto = key0.protocol;
 543   if (s0)
 544     *p_value = s0->out2in;
 545   *p_dont_translate = dont_translate;
 546   if (d)
 547     *(snat_session_t**)d = s0;
 548   return next0;
 549 }
 550
 551 /**
 552  * Get address and port values to be used for packet SNAT translation
 553  *
 554  * @param[in] sm                 SNAT main
 555  * @param[in,out] node           SNAT node runtime
 556  * @param[in] thread_index       thread index
 557  * @param[in,out] b0             buffer containing packet to be translated
 558  * @param[out] p_proto           protocol used for matching
 559  * @param[out] p_value           address and port after NAT translation
 560  * @param[out] p_dont_translate  if packet should not be translated
 561  * @param d                      optional parameter
 562  */
 563 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 564                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 565                            snat_session_key_t *p_value,
 566                            u8 *p_dont_translate, void *d)
 567 {
 568   ip4_header_t *ip0;
 569   icmp46_header_t *icmp0;
 570   u32 sw_if_index0;
 571   u32 rx_fib_index0;
 572   snat_session_key_t key0;
 573   snat_session_key_t sm0;
 574   u8 dont_translate = 0;
 575   u32 next0 = ~0;
 576   int err;
 577
 578   ip0 = vlib_buffer_get_current (b0);
 579   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 580   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 581   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 582
 583   err = icmp_get_key (ip0, &key0);
 584   if (err != -1)
 585     {
 586       b0->error = node->errors[err];
 587       next0 = SNAT_IN2OUT_NEXT_DROP;
 588       goto out2;
 589     }
 590   key0.fib_index = rx_fib_index0;
 591
 592   if (snat_static_mapping_match(sm, key0, &sm0, 0))
 593     {
 594       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 595           IP_PROTOCOL_ICMP, rx_fib_index0)))
 596         {
 597           dont_translate = 1;
 598           goto out;
 599         }
 600
 601       if (icmp_is_error_message (icmp0))
 602         {
 603           next0 = SNAT_IN2OUT_NEXT_DROP;
 604           goto out;
 605         }
 606
 607       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 608       next0 = SNAT_IN2OUT_NEXT_DROP;
 609       goto out;
 610     }
 611
 612   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 613                     !icmp_is_error_message (icmp0)))
 614     {
 615       if (icmp0->type != ICMP4_echo_reply || key0.port != sm0.port)
 616         {
 617           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 618           next0 = SNAT_IN2OUT_NEXT_DROP;
 619           goto out;
 620         }
 621     }
 622
 623 out:
 624   *p_value = sm0;
 625 out2:
 626   *p_proto = key0.protocol;
 627   *p_dont_translate = dont_translate;
 628   return next0;
 629 }
 630
 631 static inline u32 icmp_in2out (snat_main_t *sm,
 632                                vlib_buffer_t * b0,
 633                                ip4_header_t * ip0,
 634                                icmp46_header_t * icmp0,
 635                                u32 sw_if_index0,
 636                                u32 rx_fib_index0,
 637                                vlib_node_runtime_t * node,
 638                                u32 next0,
 639                                u32 thread_index,
 640                                void *d)
 641 {
 642   snat_session_key_t sm0;
 643   u8 protocol;
 644   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 645   ip4_header_t *inner_ip0;
 646   void *l4_header = 0;
 647   icmp46_header_t *inner_icmp0;
 648   u8 dont_translate;
 649   u32 new_addr0, old_addr0;
 650   u16 old_id0, new_id0;
 651   ip_csum_t sum0;
 652   u16 checksum0;
 653   u32 next0_tmp;
 654
 655   echo0 = (icmp_echo_header_t *)(icmp0+1);
 656
 657   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 658                                        &protocol, &sm0, &dont_translate, d);
 659   if (next0_tmp != ~0)
 660     next0 = next0_tmp;
 661   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 662     goto out;
 663
 664   sum0 = ip_incremental_checksum (0, icmp0,
 665                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 666   checksum0 = ~ip_csum_fold (sum0);
 667   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 668     {
 669       next0 = SNAT_IN2OUT_NEXT_DROP;
 670       goto out;
 671     }
 672
 673   old_addr0 = ip0->src_address.as_u32;
 674   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 675   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 676
 677   sum0 = ip0->checksum;
 678   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 679                          src_address /* changed member */);
 680   ip0->checksum = ip_csum_fold (sum0);
 681
 682   if (!icmp_is_error_message (icmp0))
 683     {
 684       new_id0 = sm0.port;
 685       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 686         {
 687           old_id0 = echo0->identifier;
 688           new_id0 = sm0.port;
 689           echo0->identifier = new_id0;
 690
 691           sum0 = icmp0->checksum;
 692           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 693                                  identifier);
 694           icmp0->checksum = ip_csum_fold (sum0);
 695         }
 696     }
 697   else
 698     {
 699       inner_ip0 = (ip4_header_t *)(echo0+1);
 700       l4_header = ip4_next_header (inner_ip0);
 701
 702       if (!ip4_header_checksum_is_valid (inner_ip0))
 703         {
 704           next0 = SNAT_IN2OUT_NEXT_DROP;
 705           goto out;
 706         }
 707
 708       old_addr0 = inner_ip0->dst_address.as_u32;
 709       inner_ip0->dst_address = sm0.addr;
 710       new_addr0 = inner_ip0->dst_address.as_u32;
 711
 712       sum0 = icmp0->checksum;
 713       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 714                              dst_address /* changed member */);
 715       icmp0->checksum = ip_csum_fold (sum0);
 716
 717       switch (protocol)
 718         {
 719           case SNAT_PROTOCOL_ICMP:
 720             inner_icmp0 = (icmp46_header_t*)l4_header;
 721             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 722
 723             old_id0 = inner_echo0->identifier;
 724             new_id0 = sm0.port;
 725             inner_echo0->identifier = new_id0;
 726
 727             sum0 = icmp0->checksum;
 728             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 729                                    identifier);
 730             icmp0->checksum = ip_csum_fold (sum0);
 731             break;
 732           case SNAT_PROTOCOL_UDP:
 733           case SNAT_PROTOCOL_TCP:
 734             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 735             new_id0 = sm0.port;
 736             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 737
 738             sum0 = icmp0->checksum;
 739             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 740                                    dst_port);
 741             icmp0->checksum = ip_csum_fold (sum0);
 742             break;
 743           default:
 744             ASSERT(0);
 745         }
 746     }
 747
 748 out:
 749   return next0;
 750 }
 751
 752 /**
 753  * @brief Hairpinning
 754  *
 755  * Hairpinning allows two endpoints on the internal side of the NAT to
 756  * communicate even if they only use each other's external IP addresses
 757  * and ports.
 758  *
 759  * @param sm     SNAT main.
 760  * @param b0     Vlib buffer.
 761  * @param ip0    IP header.
 762  * @param udp0   UDP header.
 763  * @param tcp0   TCP header.
 764  * @param proto0 SNAT protocol.
 765  */
 766 static inline void
 767 snat_hairpinning (snat_main_t *sm,
 768                   vlib_buffer_t * b0,
 769                   ip4_header_t * ip0,
 770                   udp_header_t * udp0,
 771                   tcp_header_t * tcp0,
 772                   u32 proto0)
 773 {
 774   snat_session_key_t key0, sm0;
 775   snat_worker_key_t k0;
 776   snat_session_t * s0;
 777   clib_bihash_kv_8_8_t kv0, value0;
 778   ip_csum_t sum0;
 779   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 780   u16 new_dst_port0, old_dst_port0;
 781
 782   key0.addr = ip0->dst_address;
 783   key0.port = udp0->dst_port;
 784   key0.protocol = proto0;
 785   key0.fib_index = sm->outside_fib_index;
 786   kv0.key = key0.as_u64;
 787
 788   /* Check if destination is in active sessions */
 789   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 790     {
 791       /* or static mappings */
 792       if (!snat_static_mapping_match(sm, key0, &sm0, 1))
 793         {
 794           new_dst_addr0 = sm0.addr.as_u32;
 795           new_dst_port0 = sm0.port;
 796           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 797         }
 798     }
 799   else
 800     {
 801       si = value0.value;
 802       if (sm->num_workers > 1)
 803         {
 804           k0.addr = ip0->dst_address;
 805           k0.port = udp0->dst_port;
 806           k0.fib_index = sm->outside_fib_index;
 807           kv0.key = k0.as_u64;
 808           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 809             ASSERT(0);
 810           else
 811             ti = value0.value;
 812         }
 813       else
 814         ti = sm->num_workers;
 815
 816       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 817       new_dst_addr0 = s0->in2out.addr.as_u32;
 818       new_dst_port0 = s0->in2out.port;
 819       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 820     }
 821
 822   /* Destination is behind the same NAT, use internal address and port */
 823   if (new_dst_addr0)
 824     {
 825       old_dst_addr0 = ip0->dst_address.as_u32;
 826       ip0->dst_address.as_u32 = new_dst_addr0;
 827       sum0 = ip0->checksum;
 828       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 829                              ip4_header_t, dst_address);
 830       ip0->checksum = ip_csum_fold (sum0);
 831
 832       old_dst_port0 = tcp0->dst;
 833       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 834         {
 835           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 836             {
 837               tcp0->dst = new_dst_port0;
 838               sum0 = tcp0->checksum;
 839               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 840                                      ip4_header_t, dst_address);
 841               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 842                                      ip4_header_t /* cheat */, length);
 843               tcp0->checksum = ip_csum_fold(sum0);
 844             }
 845           else
 846             {
 847               udp0->dst_port = new_dst_port0;
 848               udp0->checksum = 0;
 849             }
 850         }
 851     }
 852 }
 853
 854 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 855                                          vlib_buffer_t * b0,
 856                                          ip4_header_t * ip0,
 857                                          icmp46_header_t * icmp0,
 858                                          u32 sw_if_index0,
 859                                          u32 rx_fib_index0,
 860                                          vlib_node_runtime_t * node,
 861                                          u32 next0,
 862                                          f64 now,
 863                                          u32 thread_index,
 864                                          snat_session_t ** p_s0)
 865 {
 866   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 867                       next0, thread_index, p_s0);
 868   snat_session_t * s0 = *p_s0;
 869   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 870     {
 871       /* Accounting */
 872       s0->last_heard = now;
 873       s0->total_pkts++;
 874       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 875       /* Per-user LRU list maintenance for dynamic translations */
 876       if (!snat_is_session_static (s0))
 877         {
 878           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 879                              s0->per_user_index);
 880           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 881                               s0->per_user_list_head_index,
 882                               s0->per_user_index);
 883         }
 884     }
 885   return next0;
 886 }
 887
 888 static inline uword
 889 snat_in2out_node_fn_inline (vlib_main_t * vm,
 890                             vlib_node_runtime_t * node,
 891                             vlib_frame_t * frame, int is_slow_path)
 892 {
 893   u32 n_left_from, * from, * to_next;
 894   snat_in2out_next_t next_index;
 895   u32 pkts_processed = 0;
 896   snat_main_t * sm = &snat_main;
 897   f64 now = vlib_time_now (vm);
 898   u32 stats_node_index;
 899   u32 thread_index = vlib_get_thread_index ();
 900
 901   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 902     snat_in2out_node.index;
 903
 904   from = vlib_frame_vector_args (frame);
 905   n_left_from = frame->n_vectors;
 906   next_index = node->cached_next_index;
 907
 908   while (n_left_from > 0)
 909     {
 910       u32 n_left_to_next;
 911
 912       vlib_get_next_frame (vm, node, next_index,
 913                            to_next, n_left_to_next);
 914
 915       while (n_left_from >= 4 && n_left_to_next >= 2)
 916         {
 917           u32 bi0, bi1;
 918           vlib_buffer_t * b0, * b1;
 919           u32 next0, next1;
 920           u32 sw_if_index0, sw_if_index1;
 921           ip4_header_t * ip0, * ip1;
 922           ip_csum_t sum0, sum1;
 923           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 924           u16 old_port0, new_port0, old_port1, new_port1;
 925           udp_header_t * udp0, * udp1;
 926           tcp_header_t * tcp0, * tcp1;
 927           icmp46_header_t * icmp0, * icmp1;
 928           snat_session_key_t key0, key1;
 929           u32 rx_fib_index0, rx_fib_index1;
 930           u32 proto0, proto1;
 931           snat_session_t * s0 = 0, * s1 = 0;
 932           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 933
 934           /* Prefetch next iteration. */
 935           {
 936             vlib_buffer_t * p2, * p3;
 937
 938             p2 = vlib_get_buffer (vm, from[2]);
 939             p3 = vlib_get_buffer (vm, from[3]);
 940
 941             vlib_prefetch_buffer_header (p2, LOAD);
 942             vlib_prefetch_buffer_header (p3, LOAD);
 943
 944             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 945             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 946           }
 947
 948           /* speculatively enqueue b0 and b1 to the current next frame */
 949           to_next[0] = bi0 = from[0];
 950           to_next[1] = bi1 = from[1];
 951           from += 2;
 952           to_next += 2;
 953           n_left_from -= 2;
 954           n_left_to_next -= 2;
 955
 956           b0 = vlib_get_buffer (vm, bi0);
 957           b1 = vlib_get_buffer (vm, bi1);
 958
 959           ip0 = vlib_buffer_get_current (b0);
 960           udp0 = ip4_next_header (ip0);
 961           tcp0 = (tcp_header_t *) udp0;
 962           icmp0 = (icmp46_header_t *) udp0;
 963
 964           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 965           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 966                                    sw_if_index0);
 967
 968           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 969
 970           if (PREDICT_FALSE(ip0->ttl == 1))
 971             {
 972               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 973               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 974                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 975                                            0);
 976               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 977               goto trace00;
 978             }
 979
 980           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 981
 982           /* Next configured feature, probably ip4-lookup */
 983           if (is_slow_path)
 984             {
 985               if (PREDICT_FALSE (proto0 == ~0))
 986                 goto trace00;
 987
 988               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 989                 {
 990                   next0 = icmp_in2out_slow_path
 991                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 992                      node, next0, now, thread_index, &s0);
 993                   goto trace00;
 994                 }
 995             }
 996           else
 997             {
 998               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 999                 {
1000                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1001                   goto trace00;
1002                 }
1003             }
1004
1005           key0.addr = ip0->src_address;
1006           key0.port = udp0->src_port;
1007           key0.protocol = proto0;
1008           key0.fib_index = rx_fib_index0;
1009
1010           kv0.key = key0.as_u64;
1011
1012           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
1013             {
1014               if (is_slow_path)
1015                 {
1016                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
1017                       proto0, rx_fib_index0)))
1018                     goto trace00;
1019
1020                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1021                                      &s0, node, next0, thread_index);
1022                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1023                     goto trace00;
1024                 }
1025               else
1026                 {
1027                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1028                   goto trace00;
1029                 }
1030             }
1031           else
1032             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1033                                     value0.value);
1034
1035           old_addr0 = ip0->src_address.as_u32;
1036           ip0->src_address = s0->out2in.addr;
1037           new_addr0 = ip0->src_address.as_u32;
1038           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1039
1040           sum0 = ip0->checksum;
1041           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1042                                  ip4_header_t,
1043                                  src_address /* changed member */);
1044           ip0->checksum = ip_csum_fold (sum0);
1045
1046           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1047             {
1048               old_port0 = tcp0->src_port;
1049               tcp0->src_port = s0->out2in.port;
1050               new_port0 = tcp0->src_port;
1051
1052               sum0 = tcp0->checksum;
1053               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1054                                      ip4_header_t,
1055                                      dst_address /* changed member */);
1056               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1057                                      ip4_header_t /* cheat */,
1058                                      length /* changed member */);
1059               tcp0->checksum = ip_csum_fold(sum0);
1060             }
1061           else
1062             {
1063               old_port0 = udp0->src_port;
1064               udp0->src_port = s0->out2in.port;
1065               udp0->checksum = 0;
1066             }
1067
1068           /* Hairpinning */
1069           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1070
1071           /* Accounting */
1072           s0->last_heard = now;
1073           s0->total_pkts++;
1074           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1075           /* Per-user LRU list maintenance for dynamic translation */
1076           if (!snat_is_session_static (s0))
1077             {
1078               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1079                                  s0->per_user_index);
1080               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1081                                   s0->per_user_list_head_index,
1082                                   s0->per_user_index);
1083             }
1084         trace00:
1085
1086           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1087                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1088             {
1089               snat_in2out_trace_t *t =
1090                  vlib_add_trace (vm, node, b0, sizeof (*t));
1091               t->is_slow_path = is_slow_path;
1092               t->sw_if_index = sw_if_index0;
1093               t->next_index = next0;
1094                   t->session_index = ~0;
1095               if (s0)
1096                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1097             }
1098
1099           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1100
1101           ip1 = vlib_buffer_get_current (b1);
1102           udp1 = ip4_next_header (ip1);
1103           tcp1 = (tcp_header_t *) udp1;
1104           icmp1 = (icmp46_header_t *) udp1;
1105
1106           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1107           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1108                                    sw_if_index1);
1109
1110           if (PREDICT_FALSE(ip1->ttl == 1))
1111             {
1112               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1113               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1114                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1115                                            0);
1116               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1117               goto trace01;
1118             }
1119
1120           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1121
1122           /* Next configured feature, probably ip4-lookup */
1123           if (is_slow_path)
1124             {
1125               if (PREDICT_FALSE (proto1 == ~0))
1126                 goto trace01;
1127
1128               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1129                 {
1130                   next1 = icmp_in2out_slow_path
1131                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1132                      next1, now, thread_index, &s1);
1133                   goto trace01;
1134                 }
1135             }
1136           else
1137             {
1138               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1139                 {
1140                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1141                   goto trace01;
1142                 }
1143             }
1144
1145           key1.addr = ip1->src_address;
1146           key1.port = udp1->src_port;
1147           key1.protocol = proto1;
1148           key1.fib_index = rx_fib_index1;
1149
1150           kv1.key = key1.as_u64;
1151
1152             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
1153             {
1154               if (is_slow_path)
1155                 {
1156                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1, ip1,
1157                       proto1, rx_fib_index1)))
1158                     goto trace01;
1159
1160                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1161                                      &s1, node, next1, thread_index);
1162                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1163                     goto trace01;
1164                 }
1165               else
1166                 {
1167                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1168                   goto trace01;
1169                 }
1170             }
1171           else
1172             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1173                                     value1.value);
1174
1175           old_addr1 = ip1->src_address.as_u32;
1176           ip1->src_address = s1->out2in.addr;
1177           new_addr1 = ip1->src_address.as_u32;
1178           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1179
1180           sum1 = ip1->checksum;
1181           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1182                                  ip4_header_t,
1183                                  src_address /* changed member */);
1184           ip1->checksum = ip_csum_fold (sum1);
1185
1186           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1187             {
1188               old_port1 = tcp1->src_port;
1189               tcp1->src_port = s1->out2in.port;
1190               new_port1 = tcp1->src_port;
1191
1192               sum1 = tcp1->checksum;
1193               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1194                                      ip4_header_t,
1195                                      dst_address /* changed member */);
1196               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1197                                      ip4_header_t /* cheat */,
1198                                      length /* changed member */);
1199               tcp1->checksum = ip_csum_fold(sum1);
1200             }
1201           else
1202             {
1203               old_port1 = udp1->src_port;
1204               udp1->src_port = s1->out2in.port;
1205               udp1->checksum = 0;
1206             }
1207
1208           /* Hairpinning */
1209           snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1210
1211           /* Accounting */
1212           s1->last_heard = now;
1213           s1->total_pkts++;
1214           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1215           /* Per-user LRU list maintenance for dynamic translation */
1216           if (!snat_is_session_static (s1))
1217             {
1218               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1219                                  s1->per_user_index);
1220               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1221                                   s1->per_user_list_head_index,
1222                                   s1->per_user_index);
1223             }
1224         trace01:
1225
1226           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1227                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1228             {
1229               snat_in2out_trace_t *t =
1230                  vlib_add_trace (vm, node, b1, sizeof (*t));
1231               t->sw_if_index = sw_if_index1;
1232               t->next_index = next1;
1233               t->session_index = ~0;
1234               if (s1)
1235                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1236             }
1237
1238           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1239
1240           /* verify speculative enqueues, maybe switch current next frame */
1241           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1242                                            to_next, n_left_to_next,
1243                                            bi0, bi1, next0, next1);
1244         }
1245
1246       while (n_left_from > 0 && n_left_to_next > 0)
1247         {
1248           u32 bi0;
1249           vlib_buffer_t * b0;
1250           u32 next0;
1251           u32 sw_if_index0;
1252           ip4_header_t * ip0;
1253           ip_csum_t sum0;
1254           u32 new_addr0, old_addr0;
1255           u16 old_port0, new_port0;
1256           udp_header_t * udp0;
1257           tcp_header_t * tcp0;
1258           icmp46_header_t * icmp0;
1259           snat_session_key_t key0;
1260           u32 rx_fib_index0;
1261           u32 proto0;
1262           snat_session_t * s0 = 0;
1263           clib_bihash_kv_8_8_t kv0, value0;
1264
1265           /* speculatively enqueue b0 to the current next frame */
1266           bi0 = from[0];
1267           to_next[0] = bi0;
1268           from += 1;
1269           to_next += 1;
1270           n_left_from -= 1;
1271           n_left_to_next -= 1;
1272
1273           b0 = vlib_get_buffer (vm, bi0);
1274           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1275
1276           ip0 = vlib_buffer_get_current (b0);
1277           udp0 = ip4_next_header (ip0);
1278           tcp0 = (tcp_header_t *) udp0;
1279           icmp0 = (icmp46_header_t *) udp0;
1280
1281           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1282           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1283                                    sw_if_index0);
1284
1285           if (PREDICT_FALSE(ip0->ttl == 1))
1286             {
1287               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1288               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1289                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1290                                            0);
1291               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1292               goto trace0;
1293             }
1294
1295           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1296
1297           /* Next configured feature, probably ip4-lookup */
1298           if (is_slow_path)
1299             {
1300               if (PREDICT_FALSE (proto0 == ~0))
1301                 goto trace0;
1302
1303               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1304                 {
1305                   next0 = icmp_in2out_slow_path
1306                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1307                      next0, now, thread_index, &s0);
1308                   goto trace0;
1309                 }
1310             }
1311           else
1312             {
1313               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1314                 {
1315                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1316                   goto trace0;
1317                 }
1318             }
1319
1320           key0.addr = ip0->src_address;
1321           key0.port = udp0->src_port;
1322           key0.protocol = proto0;
1323           key0.fib_index = rx_fib_index0;
1324
1325           kv0.key = key0.as_u64;
1326
1327           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
1328             {
1329               if (is_slow_path)
1330                 {
1331                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
1332                       proto0, rx_fib_index0)))
1333                     goto trace0;
1334
1335                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1336                                      &s0, node, next0, thread_index);
1337
1338                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1339                     goto trace0;
1340                 }
1341               else
1342                 {
1343                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1344                   goto trace0;
1345                 }
1346             }
1347           else
1348             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1349                                     value0.value);
1350
1351           old_addr0 = ip0->src_address.as_u32;
1352           ip0->src_address = s0->out2in.addr;
1353           new_addr0 = ip0->src_address.as_u32;
1354           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1355
1356           sum0 = ip0->checksum;
1357           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1358                                  ip4_header_t,
1359                                  src_address /* changed member */);
1360           ip0->checksum = ip_csum_fold (sum0);
1361
1362           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1363             {
1364               old_port0 = tcp0->src_port;
1365               tcp0->src_port = s0->out2in.port;
1366               new_port0 = tcp0->src_port;
1367
1368               sum0 = tcp0->checksum;
1369               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1370                                      ip4_header_t,
1371                                      dst_address /* changed member */);
1372               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1373                                      ip4_header_t /* cheat */,
1374                                      length /* changed member */);
1375               tcp0->checksum = ip_csum_fold(sum0);
1376             }
1377           else
1378             {
1379               old_port0 = udp0->src_port;
1380               udp0->src_port = s0->out2in.port;
1381               udp0->checksum = 0;
1382             }
1383
1384           /* Hairpinning */
1385           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1386
1387           /* Accounting */
1388           s0->last_heard = now;
1389           s0->total_pkts++;
1390           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1391           /* Per-user LRU list maintenance for dynamic translation */
1392           if (!snat_is_session_static (s0))
1393             {
1394               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1395                                  s0->per_user_index);
1396               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1397                                   s0->per_user_list_head_index,
1398                                   s0->per_user_index);
1399             }
1400
1401         trace0:
1402           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1403                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1404             {
1405               snat_in2out_trace_t *t =
1406                  vlib_add_trace (vm, node, b0, sizeof (*t));
1407               t->is_slow_path = is_slow_path;
1408               t->sw_if_index = sw_if_index0;
1409               t->next_index = next0;
1410                   t->session_index = ~0;
1411               if (s0)
1412                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1413             }
1414
1415           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1416
1417           /* verify speculative enqueue, maybe switch current next frame */
1418           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1419                                            to_next, n_left_to_next,
1420                                            bi0, next0);
1421         }
1422
1423       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1424     }
1425
1426   vlib_node_increment_counter (vm, stats_node_index,
1427                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1428                                pkts_processed);
1429   return frame->n_vectors;
1430 }
1431
1432 static uword
1433 snat_in2out_fast_path_fn (vlib_main_t * vm,
1434                           vlib_node_runtime_t * node,
1435                           vlib_frame_t * frame)
1436 {
1437   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
1438 }
1439
1440 VLIB_REGISTER_NODE (snat_in2out_node) = {
1441   .function = snat_in2out_fast_path_fn,
1442   .name = "snat-in2out",
1443   .vector_size = sizeof (u32),
1444   .format_trace = format_snat_in2out_trace,
1445   .type = VLIB_NODE_TYPE_INTERNAL,
1446
1447   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1448   .error_strings = snat_in2out_error_strings,
1449
1450   .runtime_data_bytes = sizeof (snat_runtime_t),
1451
1452   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1453
1454   /* edit / add dispositions here */
1455   .next_nodes = {
1456     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1457     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1458     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1459     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1460   },
1461 };
1462
1463 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1464
1465 static uword
1466 snat_in2out_slow_path_fn (vlib_main_t * vm,
1467                           vlib_node_runtime_t * node,
1468                           vlib_frame_t * frame)
1469 {
1470   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
1471 }
1472
1473 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1474   .function = snat_in2out_slow_path_fn,
1475   .name = "snat-in2out-slowpath",
1476   .vector_size = sizeof (u32),
1477   .format_trace = format_snat_in2out_trace,
1478   .type = VLIB_NODE_TYPE_INTERNAL,
1479
1480   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1481   .error_strings = snat_in2out_error_strings,
1482
1483   .runtime_data_bytes = sizeof (snat_runtime_t),
1484
1485   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1486
1487   /* edit / add dispositions here */
1488   .next_nodes = {
1489     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1490     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1491     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1492     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1493   },
1494 };
1495
1496 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1497
1498 /**************************/
1499 /*** deterministic mode ***/
1500 /**************************/
1501 static uword
1502 snat_det_in2out_node_fn (vlib_main_t * vm,
1503                          vlib_node_runtime_t * node,
1504                          vlib_frame_t * frame)
1505 {
1506   u32 n_left_from, * from, * to_next;
1507   snat_in2out_next_t next_index;
1508   u32 pkts_processed = 0;
1509   snat_main_t * sm = &snat_main;
1510   u32 now = (u32) vlib_time_now (vm);
1511
1512   from = vlib_frame_vector_args (frame);
1513   n_left_from = frame->n_vectors;
1514   next_index = node->cached_next_index;
1515
1516   while (n_left_from > 0)
1517     {
1518       u32 n_left_to_next;
1519
1520       vlib_get_next_frame (vm, node, next_index,
1521                            to_next, n_left_to_next);
1522
1523       while (n_left_from >= 4 && n_left_to_next >= 2)
1524         {
1525           u32 bi0, bi1;
1526           vlib_buffer_t * b0, * b1;
1527           u32 next0, next1;
1528           u32 sw_if_index0, sw_if_index1;
1529           ip4_header_t * ip0, * ip1;
1530           ip_csum_t sum0, sum1;
1531           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1532           u16 old_port0, new_port0, lo_port0, i0;
1533           u16 old_port1, new_port1, lo_port1, i1;
1534           udp_header_t * udp0, * udp1;
1535           tcp_header_t * tcp0, * tcp1;
1536           u32 proto0, proto1;
1537           snat_det_out_key_t key0, key1;
1538           snat_det_map_t * dm0, * dm1;
1539           snat_det_session_t * ses0 = 0, * ses1 = 0;
1540
1541           /* Prefetch next iteration. */
1542           {
1543             vlib_buffer_t * p2, * p3;
1544
1545             p2 = vlib_get_buffer (vm, from[2]);
1546             p3 = vlib_get_buffer (vm, from[3]);
1547
1548             vlib_prefetch_buffer_header (p2, LOAD);
1549             vlib_prefetch_buffer_header (p3, LOAD);
1550
1551             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1552             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1553           }
1554
1555           /* speculatively enqueue b0 and b1 to the current next frame */
1556           to_next[0] = bi0 = from[0];
1557           to_next[1] = bi1 = from[1];
1558           from += 2;
1559           to_next += 2;
1560           n_left_from -= 2;
1561           n_left_to_next -= 2;
1562
1563           b0 = vlib_get_buffer (vm, bi0);
1564           b1 = vlib_get_buffer (vm, bi1);
1565
1566           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1567           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1568
1569           ip0 = vlib_buffer_get_current (b0);
1570           udp0 = ip4_next_header (ip0);
1571           tcp0 = (tcp_header_t *) udp0;
1572
1573           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1574
1575           if (PREDICT_FALSE(ip0->ttl == 1))
1576             {
1577               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1578               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1579                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1580                                            0);
1581               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1582               goto trace0;
1583             }
1584
1585           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
1586           if (PREDICT_FALSE(!dm0))
1587             {
1588               clib_warning("no match for internal host %U",
1589                            format_ip4_address, &ip0->src_address);
1590               next0 = SNAT_IN2OUT_NEXT_DROP;
1591               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1592               goto trace0;
1593             }
1594
1595           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
1596
1597           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src);
1598           if (PREDICT_FALSE(!ses0))
1599             {
1600               key0.ext_host_addr = ip0->dst_address;
1601               key0.ext_host_port = tcp0->dst;
1602               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
1603                 {
1604                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
1605                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
1606
1607                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
1608                     continue;
1609
1610                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
1611                   break;
1612                 }
1613               if (PREDICT_FALSE(!ses0))
1614                 {
1615                   next0 = SNAT_IN2OUT_NEXT_DROP;
1616                   b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
1617                   goto trace0;
1618                 }
1619             }
1620
1621           new_port0 = ses0->out.out_port;
1622           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1623
1624           old_addr0.as_u32 = ip0->src_address.as_u32;
1625           ip0->src_address.as_u32 = new_addr0.as_u32;
1626           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1627
1628           sum0 = ip0->checksum;
1629           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1630                                  ip4_header_t,
1631                                  src_address /* changed member */);
1632           ip0->checksum = ip_csum_fold (sum0);
1633
1634           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1635             {
1636               if (tcp0->flags & TCP_FLAG_SYN)
1637                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
1638               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
1639                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1640               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1641                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
1642               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
1643                 snat_det_ses_close(dm0, ses0);
1644               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
1645                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
1646               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
1647                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1648
1649               old_port0 = tcp0->src;
1650               tcp0->src = new_port0;
1651
1652               sum0 = tcp0->checksum;
1653               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1654                                      ip4_header_t,
1655                                      dst_address /* changed member */);
1656               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1657                                      ip4_header_t /* cheat */,
1658                                      length /* changed member */);
1659               tcp0->checksum = ip_csum_fold(sum0);
1660             }
1661           else
1662             {
1663               ses0->state = SNAT_SESSION_UDP_ACTIVE;
1664               old_port0 = udp0->src_port;
1665               udp0->src_port = new_port0;
1666               udp0->checksum = 0;
1667             }
1668
1669           switch(ses0->state)
1670             {
1671             case SNAT_SESSION_UDP_ACTIVE:
1672                 ses0->expire = now + SNAT_UDP_TIMEOUT;
1673                 break;
1674             case SNAT_SESSION_TCP_SYN_SENT:
1675             case SNAT_SESSION_TCP_FIN_WAIT:
1676             case SNAT_SESSION_TCP_CLOSE_WAIT:
1677             case SNAT_SESSION_TCP_LAST_ACK:
1678                 ses0->expire = now + SNAT_TCP_TRANSITORY_TIMEOUT;
1679                 break;
1680             case SNAT_SESSION_TCP_ESTABLISHED:
1681                 ses0->expire = now + SNAT_TCP_ESTABLISHED_TIMEOUT;
1682                 break;
1683             }
1684
1685         trace0:
1686           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1687                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1688             {
1689               snat_in2out_trace_t *t =
1690                  vlib_add_trace (vm, node, b0, sizeof (*t));
1691               t->is_slow_path = 0;
1692               t->sw_if_index = sw_if_index0;
1693               t->next_index = next0;
1694               t->session_index = ~0;
1695               if (ses0)
1696                 t->session_index = ses0 - dm0->sessions;
1697             }
1698
1699           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1700
1701           ip1 = vlib_buffer_get_current (b1);
1702           udp1 = ip4_next_header (ip1);
1703           tcp1 = (tcp_header_t *) udp1;
1704
1705           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1706
1707           if (PREDICT_FALSE(ip1->ttl == 1))
1708             {
1709               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1710               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1711                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1712                                            0);
1713               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1714               goto trace1;
1715             }
1716
1717           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
1718           if (PREDICT_FALSE(!dm1))
1719             {
1720               clib_warning("no match for internal host %U",
1721                            format_ip4_address, &ip0->src_address);
1722               next1 = SNAT_IN2OUT_NEXT_DROP;
1723               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1724               goto trace1;
1725             }
1726
1727           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
1728
1729           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src);
1730           if (PREDICT_FALSE(!ses1))
1731             {
1732               key1.ext_host_addr = ip1->dst_address;
1733               key1.ext_host_port = tcp1->dst;
1734               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
1735                 {
1736                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
1737                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
1738
1739                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
1740                     continue;
1741
1742                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
1743                   break;
1744                 }
1745               if (PREDICT_FALSE(!ses1))
1746                 {
1747                   next1 = SNAT_IN2OUT_NEXT_DROP;
1748                   b1->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
1749                   goto trace1;
1750                 }
1751             }
1752
1753           new_port1 = ses1->out.out_port;
1754           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1755
1756           old_addr1.as_u32 = ip1->src_address.as_u32;
1757           ip1->src_address.as_u32 = new_addr1.as_u32;
1758           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1759
1760           sum1 = ip1->checksum;
1761           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1762                                  ip4_header_t,
1763                                  src_address /* changed member */);
1764           ip1->checksum = ip_csum_fold (sum1);
1765
1766           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1767             {
1768               if (tcp1->flags & TCP_FLAG_SYN)
1769                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
1770               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
1771                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
1772               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1773                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
1774               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
1775                 snat_det_ses_close(dm1, ses1);
1776               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
1777                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
1778               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
1779                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
1780
1781               old_port1 = tcp1->src;
1782               tcp1->src = new_port1;
1783
1784               sum1 = tcp1->checksum;
1785               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1786                                      ip4_header_t,
1787                                      dst_address /* changed member */);
1788               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1789                                      ip4_header_t /* cheat */,
1790                                      length /* changed member */);
1791               tcp1->checksum = ip_csum_fold(sum1);
1792             }
1793           else
1794             {
1795               ses1->state = SNAT_SESSION_UDP_ACTIVE;
1796               old_port1 = udp1->src_port;
1797               udp1->src_port = new_port1;
1798               udp1->checksum = 0;
1799             }
1800
1801           switch(ses1->state)
1802             {
1803             case SNAT_SESSION_UDP_ACTIVE:
1804                 ses1->expire = now + SNAT_UDP_TIMEOUT;
1805                 break;
1806             case SNAT_SESSION_TCP_SYN_SENT:
1807             case SNAT_SESSION_TCP_FIN_WAIT:
1808             case SNAT_SESSION_TCP_CLOSE_WAIT:
1809             case SNAT_SESSION_TCP_LAST_ACK:
1810                 ses1->expire = now + SNAT_TCP_TRANSITORY_TIMEOUT;
1811                 break;
1812             case SNAT_SESSION_TCP_ESTABLISHED:
1813                 ses1->expire = now + SNAT_TCP_ESTABLISHED_TIMEOUT;
1814                 break;
1815             }
1816
1817         trace1:
1818           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1819                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1820             {
1821               snat_in2out_trace_t *t =
1822                  vlib_add_trace (vm, node, b1, sizeof (*t));
1823               t->is_slow_path = 0;
1824               t->sw_if_index = sw_if_index1;
1825               t->next_index = next1;
1826               t->session_index = ~0;
1827               if (ses1)
1828                 t->session_index = ses1 - dm1->sessions;
1829             }
1830
1831           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1832
1833           /* verify speculative enqueues, maybe switch current next frame */
1834           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1835                                            to_next, n_left_to_next,
1836                                            bi0, bi1, next0, next1);
1837          }
1838
1839       while (n_left_from > 0 && n_left_to_next > 0)
1840         {
1841           u32 bi0;
1842           vlib_buffer_t * b0;
1843           u32 next0;
1844           u32 sw_if_index0;
1845           ip4_header_t * ip0;
1846           ip_csum_t sum0;
1847           ip4_address_t new_addr0, old_addr0;
1848           u16 old_port0, new_port0, lo_port0, i0;
1849           udp_header_t * udp0;
1850           tcp_header_t * tcp0;
1851           u32 proto0;
1852           snat_det_out_key_t key0;
1853           snat_det_map_t * dm0;
1854           snat_det_session_t * ses0 = 0;
1855
1856           /* speculatively enqueue b0 to the current next frame */
1857           bi0 = from[0];
1858           to_next[0] = bi0;
1859           from += 1;
1860           to_next += 1;
1861           n_left_from -= 1;
1862           n_left_to_next -= 1;
1863
1864           b0 = vlib_get_buffer (vm, bi0);
1865           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1866
1867           ip0 = vlib_buffer_get_current (b0);
1868           udp0 = ip4_next_header (ip0);
1869           tcp0 = (tcp_header_t *) udp0;
1870
1871           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1872
1873           if (PREDICT_FALSE(ip0->ttl == 1))
1874             {
1875               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1876               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1877                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1878                                            0);
1879               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1880               goto trace00;
1881             }
1882
1883           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
1884           if (PREDICT_FALSE(!dm0))
1885             {
1886               clib_warning("no match for internal host %U",
1887                            format_ip4_address, &ip0->src_address);
1888               next0 = SNAT_IN2OUT_NEXT_DROP;
1889               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1890               goto trace00;
1891             }
1892
1893           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
1894
1895           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src);
1896           if (PREDICT_FALSE(!ses0))
1897             {
1898               key0.ext_host_addr = ip0->dst_address;
1899               key0.ext_host_port = tcp0->dst;
1900               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
1901                 {
1902                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
1903                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
1904
1905                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
1906                     continue;
1907
1908                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
1909                   break;
1910                 }
1911               if (PREDICT_FALSE(!ses0))
1912                 {
1913                   next0 = SNAT_IN2OUT_NEXT_DROP;
1914                   b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
1915                   goto trace00;
1916                 }
1917             }
1918
1919           new_port0 = ses0->out.out_port;
1920           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1921
1922           old_addr0.as_u32 = ip0->src_address.as_u32;
1923           ip0->src_address.as_u32 = new_addr0.as_u32;
1924           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1925
1926           sum0 = ip0->checksum;
1927           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1928                                  ip4_header_t,
1929                                  src_address /* changed member */);
1930           ip0->checksum = ip_csum_fold (sum0);
1931
1932           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1933             {
1934               if (tcp0->flags & TCP_FLAG_SYN)
1935                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
1936               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
1937                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1938               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1939                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
1940               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
1941                 snat_det_ses_close(dm0, ses0);
1942               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
1943                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
1944               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
1945                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
1946
1947               old_port0 = tcp0->src;
1948               tcp0->src = new_port0;
1949
1950               sum0 = tcp0->checksum;
1951               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1952                                      ip4_header_t,
1953                                      dst_address /* changed member */);
1954               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1955                                      ip4_header_t /* cheat */,
1956                                      length /* changed member */);
1957               tcp0->checksum = ip_csum_fold(sum0);
1958             }
1959           else
1960             {
1961               ses0->state = SNAT_SESSION_UDP_ACTIVE;
1962               old_port0 = udp0->src_port;
1963               udp0->src_port = new_port0;
1964               udp0->checksum = 0;
1965             }
1966
1967           switch(ses0->state)
1968             {
1969             case SNAT_SESSION_UDP_ACTIVE:
1970                 ses0->expire = now + SNAT_UDP_TIMEOUT;
1971                 break;
1972             case SNAT_SESSION_TCP_SYN_SENT:
1973             case SNAT_SESSION_TCP_FIN_WAIT:
1974             case SNAT_SESSION_TCP_CLOSE_WAIT:
1975             case SNAT_SESSION_TCP_LAST_ACK:
1976                 ses0->expire = now + SNAT_TCP_TRANSITORY_TIMEOUT;
1977                 break;
1978             case SNAT_SESSION_TCP_ESTABLISHED:
1979                 ses0->expire = now + SNAT_TCP_ESTABLISHED_TIMEOUT;
1980                 break;
1981             }
1982
1983         trace00:
1984           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1985                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1986             {
1987               snat_in2out_trace_t *t =
1988                  vlib_add_trace (vm, node, b0, sizeof (*t));
1989               t->is_slow_path = 0;
1990               t->sw_if_index = sw_if_index0;
1991               t->next_index = next0;
1992               t->session_index = ~0;
1993               if (ses0)
1994                 t->session_index = ses0 - dm0->sessions;
1995             }
1996
1997           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1998
1999           /* verify speculative enqueue, maybe switch current next frame */
2000           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2001                                            to_next, n_left_to_next,
2002                                            bi0, next0);
2003         }
2004
2005       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2006     }
2007
2008   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2009                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2010                                pkts_processed);
2011   return frame->n_vectors;
2012 }
2013
2014 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2015   .function = snat_det_in2out_node_fn,
2016   .name = "snat-det-in2out",
2017   .vector_size = sizeof (u32),
2018   .format_trace = format_snat_in2out_trace,
2019   .type = VLIB_NODE_TYPE_INTERNAL,
2020
2021   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2022   .error_strings = snat_in2out_error_strings,
2023
2024   .runtime_data_bytes = sizeof (snat_runtime_t),
2025
2026   .n_next_nodes = 3,
2027
2028   /* edit / add dispositions here */
2029   .next_nodes = {
2030     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2031     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2032     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2033   },
2034 };
2035
2036 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2037
2038 /**********************/
2039 /*** worker handoff ***/
2040 /**********************/
2041 static uword
2042 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
2043                                vlib_node_runtime_t * node,
2044                                vlib_frame_t * frame)
2045 {
2046   snat_main_t *sm = &snat_main;
2047   vlib_thread_main_t *tm = vlib_get_thread_main ();
2048   u32 n_left_from, *from, *to_next = 0;
2049   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2050   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2051     = 0;
2052   vlib_frame_queue_elt_t *hf = 0;
2053   vlib_frame_t *f = 0;
2054   int i;
2055   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2056   u32 next_worker_index = 0;
2057   u32 current_worker_index = ~0;
2058   u32 thread_index = vlib_get_thread_index ();
2059
2060   ASSERT (vec_len (sm->workers));
2061
2062   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2063     {
2064       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2065
2066       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2067                                sm->first_worker_index + sm->num_workers - 1,
2068                                (vlib_frame_queue_t *) (~0));
2069     }
2070
2071   from = vlib_frame_vector_args (frame);
2072   n_left_from = frame->n_vectors;
2073
2074   while (n_left_from > 0)
2075     {
2076       u32 bi0;
2077       vlib_buffer_t *b0;
2078       u32 sw_if_index0;
2079       u32 rx_fib_index0;
2080       ip4_header_t * ip0;
2081       u8 do_handoff;
2082
2083       bi0 = from[0];
2084       from += 1;
2085       n_left_from -= 1;
2086
2087       b0 = vlib_get_buffer (vm, bi0);
2088
2089       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2090       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2091
2092       ip0 = vlib_buffer_get_current (b0);
2093
2094       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
2095
2096       if (PREDICT_FALSE (next_worker_index != thread_index))
2097         {
2098           do_handoff = 1;
2099
2100           if (next_worker_index != current_worker_index)
2101             {
2102               if (hf)
2103                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2104
2105               hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index,
2106                                                       next_worker_index,
2107                                                       handoff_queue_elt_by_worker_index);
2108
2109               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2110               to_next_worker = &hf->buffer_index[hf->n_vectors];
2111               current_worker_index = next_worker_index;
2112             }
2113
2114           /* enqueue to correct worker thread */
2115           to_next_worker[0] = bi0;
2116           to_next_worker++;
2117           n_left_to_next_worker--;
2118
2119           if (n_left_to_next_worker == 0)
2120             {
2121               hf->n_vectors = VLIB_FRAME_SIZE;
2122               vlib_put_frame_queue_elt (hf);
2123               current_worker_index = ~0;
2124               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2125               hf = 0;
2126             }
2127         }
2128       else
2129         {
2130           do_handoff = 0;
2131           /* if this is 1st frame */
2132           if (!f)
2133             {
2134               f = vlib_get_frame_to_node (vm, sm->in2out_node_index);
2135               to_next = vlib_frame_vector_args (f);
2136             }
2137
2138           to_next[0] = bi0;
2139           to_next += 1;
2140           f->n_vectors++;
2141         }
2142
2143       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2144                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2145         {
2146           snat_in2out_worker_handoff_trace_t *t =
2147             vlib_add_trace (vm, node, b0, sizeof (*t));
2148           t->next_worker_index = next_worker_index;
2149           t->do_handoff = do_handoff;
2150         }
2151     }
2152
2153   if (f)
2154     vlib_put_frame_to_node (vm, sm->in2out_node_index, f);
2155
2156   if (hf)
2157     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2158
2159   /* Ship frames to the worker nodes */
2160   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2161     {
2162       if (handoff_queue_elt_by_worker_index[i])
2163         {
2164           hf = handoff_queue_elt_by_worker_index[i];
2165           /*
2166            * It works better to let the handoff node
2167            * rate-adapt, always ship the handoff queue element.
2168            */
2169           if (1 || hf->n_vectors == hf->last_n_vectors)
2170             {
2171               vlib_put_frame_queue_elt (hf);
2172               handoff_queue_elt_by_worker_index[i] = 0;
2173             }
2174           else
2175             hf->last_n_vectors = hf->n_vectors;
2176         }
2177       congested_handoff_queue_by_worker_index[i] =
2178         (vlib_frame_queue_t *) (~0);
2179     }
2180   hf = 0;
2181   current_worker_index = ~0;
2182   return frame->n_vectors;
2183 }
2184
2185 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
2186   .function = snat_in2out_worker_handoff_fn,
2187   .name = "snat-in2out-worker-handoff",
2188   .vector_size = sizeof (u32),
2189   .format_trace = format_snat_in2out_worker_handoff_trace,
2190   .type = VLIB_NODE_TYPE_INTERNAL,
2191
2192   .n_next_nodes = 1,
2193
2194   .next_nodes = {
2195     [0] = "error-drop",
2196   },
2197 };
2198
2199 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn);
2200
2201 static uword
2202 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
2203                                 vlib_node_runtime_t * node,
2204                                 vlib_frame_t * frame)
2205 {
2206   u32 n_left_from, * from, * to_next;
2207   snat_in2out_next_t next_index;
2208   u32 pkts_processed = 0;
2209   snat_main_t * sm = &snat_main;
2210   u32 stats_node_index;
2211
2212   stats_node_index = snat_in2out_fast_node.index;
2213
2214   from = vlib_frame_vector_args (frame);
2215   n_left_from = frame->n_vectors;
2216   next_index = node->cached_next_index;
2217
2218   while (n_left_from > 0)
2219     {
2220       u32 n_left_to_next;
2221
2222       vlib_get_next_frame (vm, node, next_index,
2223                            to_next, n_left_to_next);
2224
2225       while (n_left_from > 0 && n_left_to_next > 0)
2226         {
2227           u32 bi0;
2228           vlib_buffer_t * b0;
2229           u32 next0;
2230           u32 sw_if_index0;
2231           ip4_header_t * ip0;
2232           ip_csum_t sum0;
2233           u32 new_addr0, old_addr0;
2234           u16 old_port0, new_port0;
2235           udp_header_t * udp0;
2236           tcp_header_t * tcp0;
2237           icmp46_header_t * icmp0;
2238           snat_session_key_t key0, sm0;
2239           u32 proto0;
2240           u32 rx_fib_index0;
2241
2242           /* speculatively enqueue b0 to the current next frame */
2243           bi0 = from[0];
2244           to_next[0] = bi0;
2245           from += 1;
2246           to_next += 1;
2247           n_left_from -= 1;
2248           n_left_to_next -= 1;
2249
2250           b0 = vlib_get_buffer (vm, bi0);
2251           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2252
2253           ip0 = vlib_buffer_get_current (b0);
2254           udp0 = ip4_next_header (ip0);
2255           tcp0 = (tcp_header_t *) udp0;
2256           icmp0 = (icmp46_header_t *) udp0;
2257
2258           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2259           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2260
2261           if (PREDICT_FALSE(ip0->ttl == 1))
2262             {
2263               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2264               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2265                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2266                                            0);
2267               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2268               goto trace0;
2269             }
2270
2271           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2272
2273           if (PREDICT_FALSE (proto0 == ~0))
2274               goto trace0;
2275
2276           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2277             {
2278               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2279                                   rx_fib_index0, node, next0, ~0, 0);
2280               goto trace0;
2281             }
2282
2283           key0.addr = ip0->src_address;
2284           key0.port = udp0->src_port;
2285           key0.fib_index = rx_fib_index0;
2286
2287           if (snat_static_mapping_match(sm, key0, &sm0, 0))
2288             {
2289               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2290               next0= SNAT_IN2OUT_NEXT_DROP;
2291               goto trace0;
2292             }
2293
2294           new_addr0 = sm0.addr.as_u32;
2295           new_port0 = sm0.port;
2296           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2297           old_addr0 = ip0->src_address.as_u32;
2298           ip0->src_address.as_u32 = new_addr0;
2299
2300           sum0 = ip0->checksum;
2301           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2302                                  ip4_header_t,
2303                                  src_address /* changed member */);
2304           ip0->checksum = ip_csum_fold (sum0);
2305
2306           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2307             {
2308               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2309                 {
2310                   old_port0 = tcp0->src_port;
2311                   tcp0->src_port = new_port0;
2312
2313                   sum0 = tcp0->checksum;
2314                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2315                                          ip4_header_t,
2316                                          dst_address /* changed member */);
2317                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2318                                          ip4_header_t /* cheat */,
2319                                          length /* changed member */);
2320                   tcp0->checksum = ip_csum_fold(sum0);
2321                 }
2322               else
2323                 {
2324                   old_port0 = udp0->src_port;
2325                   udp0->src_port = new_port0;
2326                   udp0->checksum = 0;
2327                 }
2328             }
2329           else
2330             {
2331               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2332                 {
2333                   sum0 = tcp0->checksum;
2334                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2335                                          ip4_header_t,
2336                                          dst_address /* changed member */);
2337                   tcp0->checksum = ip_csum_fold(sum0);
2338                 }
2339             }
2340
2341           /* Hairpinning */
2342           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2343
2344         trace0:
2345           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2346                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2347             {
2348               snat_in2out_trace_t *t =
2349                  vlib_add_trace (vm, node, b0, sizeof (*t));
2350               t->sw_if_index = sw_if_index0;
2351               t->next_index = next0;
2352             }
2353
2354           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2355
2356           /* verify speculative enqueue, maybe switch current next frame */
2357           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2358                                            to_next, n_left_to_next,
2359                                            bi0, next0);
2360         }
2361
2362       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2363     }
2364
2365   vlib_node_increment_counter (vm, stats_node_index,
2366                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2367                                pkts_processed);
2368   return frame->n_vectors;
2369 }
2370
2371
2372 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2373   .function = snat_in2out_fast_static_map_fn,
2374   .name = "snat-in2out-fast",
2375   .vector_size = sizeof (u32),
2376   .format_trace = format_snat_in2out_fast_trace,
2377   .type = VLIB_NODE_TYPE_INTERNAL,
2378
2379   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2380   .error_strings = snat_in2out_error_strings,
2381
2382   .runtime_data_bytes = sizeof (snat_runtime_t),
2383
2384   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2385
2386   /* edit / add dispositions here */
2387   .next_nodes = {
2388     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2389     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2390     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
2391     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2392   },
2393 };
2394
2395 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);