src/plugins/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25 #include <snat/snat_ipfix_logging.h>
  26 #include <snat/snat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91
  92 #define foreach_snat_in2out_error                       \
  93 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  94 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  95 _(OUT_OF_PORTS, "Out of ports")                         \
  96 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  97 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  98 _(NO_TRANSLATION, "No translation")
  99
 100 typedef enum {
 101 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 102   foreach_snat_in2out_error
 103 #undef _
 104   SNAT_IN2OUT_N_ERROR,
 105 } snat_in2out_error_t;
 106
 107 static char * snat_in2out_error_strings[] = {
 108 #define _(sym,string) string,
 109   foreach_snat_in2out_error
 110 #undef _
 111 };
 112
 113 typedef enum {
 114   SNAT_IN2OUT_NEXT_LOOKUP,
 115   SNAT_IN2OUT_NEXT_DROP,
 116   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 117   SNAT_IN2OUT_NEXT_SLOW_PATH,
 118   SNAT_IN2OUT_N_NEXT,
 119 } snat_in2out_next_t;
 120
 121 /**
 122  * @brief Check if packet should be translated
 123  *
 124  * Packets aimed at outside interface and external addresss with active session
 125  * should be translated.
 126  *
 127  * @param sm            SNAT main
 128  * @param rt            SNAT runtime data
 129  * @param sw_if_index0  index of the inside interface
 130  * @param ip0           IPv4 header
 131  * @param proto0        SNAT protocol
 132  * @param rx_fib_index0 RX FIB index
 133  *
 134  * @returns 0 if packet should be translated otherwise 1
 135  */
 136 static inline int
 137 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 138                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 139                          u32 rx_fib_index0)
 140 {
 141   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 142   fib_prefix_t pfx = {
 143     .fp_proto = FIB_PROTOCOL_IP4,
 144     .fp_len = 32,
 145     .fp_addr = {
 146         .ip4.as_u32 = ip0->dst_address.as_u32,
 147     },
 148   };
 149
 150   /* Don't NAT packet aimed at the intfc address */
 151   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 152                                       ip0->dst_address.as_u32)))
 153     return 1;
 154
 155   fei = fib_table_lookup (rx_fib_index0, &pfx);
 156   if (FIB_NODE_INDEX_INVALID != fei)
 157     {
 158       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 159       if (sw_if_index == ~0)
 160         {
 161           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 162           if (FIB_NODE_INDEX_INVALID != fei)
 163             sw_if_index = fib_entry_get_resolving_interface (fei);
 164         }
 165       snat_interface_t *i;
 166       pool_foreach (i, sm->interfaces,
 167       ({
 168         /* NAT packet aimed at outside interface */
 169         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 170           return 0;
 171       }));
 172     }
 173
 174   return 1;
 175 }
 176
 177 static inline int
 178 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 179                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                     u32 rx_fib_index0)
 181 {
 182   udp_header_t * udp0 = ip4_next_header (ip0);
 183   snat_session_key_t key0, sm0;
 184   clib_bihash_kv_8_8_t kv0, value0;
 185
 186   key0.addr = ip0->dst_address;
 187   key0.port = udp0->dst_port;
 188   key0.protocol = proto0;
 189   key0.fib_index = sm->outside_fib_index;
 190   kv0.key = key0.as_u64;
 191
 192   /* NAT packet aimed at external address if */
 193   /* has active sessions */
 194   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 195     {
 196       /* or is static mappings */
 197       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 198         return 0;
 199     }
 200   else
 201     return 0;
 202
 203   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 204                                  rx_fib_index0);
 205 }
 206
 207 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 208                       ip4_header_t * ip0,
 209                       u32 rx_fib_index0,
 210                       snat_session_key_t * key0,
 211                       snat_session_t ** sessionp,
 212                       vlib_node_runtime_t * node,
 213                       u32 next0,
 214                       u32 thread_index)
 215 {
 216   snat_user_t *u;
 217   snat_user_key_t user_key;
 218   snat_session_t *s;
 219   clib_bihash_kv_8_8_t kv0, value0;
 220   u32 oldest_per_user_translation_list_index;
 221   dlist_elt_t * oldest_per_user_translation_list_elt;
 222   dlist_elt_t * per_user_translation_list_elt;
 223   dlist_elt_t * per_user_list_head_elt;
 224   u32 session_index;
 225   snat_session_key_t key1;
 226   u32 address_index = ~0;
 227   u32 outside_fib_index;
 228   uword * p;
 229   snat_worker_key_t worker_by_out_key;
 230
 231   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 232   if (! p)
 233     {
 234       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 235       return SNAT_IN2OUT_NEXT_DROP;
 236     }
 237   outside_fib_index = p[0];
 238
 239   key1.protocol = key0->protocol;
 240   user_key.addr = ip0->src_address;
 241   user_key.fib_index = rx_fib_index0;
 242   kv0.key = user_key.as_u64;
 243
 244   /* Ever heard of the "user" = src ip4 address before? */
 245   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 246     {
 247       /* no, make a new one */
 248       pool_get (sm->per_thread_data[thread_index].users, u);
 249       memset (u, 0, sizeof (*u));
 250       u->addr = ip0->src_address;
 251       u->fib_index = rx_fib_index0;
 252
 253       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 254
 255       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 256         sm->per_thread_data[thread_index].list_pool;
 257
 258       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 259                        u->sessions_per_user_list_head_index);
 260
 261       kv0.value = u - sm->per_thread_data[thread_index].users;
 262
 263       /* add user */
 264       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 265     }
 266   else
 267     {
 268       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 269                              value0.value);
 270     }
 271
 272   /* Over quota? Recycle the least recently used dynamic translation */
 273   if (u->nsessions >= sm->max_translations_per_user)
 274     {
 275       /* Remove the oldest dynamic translation */
 276       do {
 277           oldest_per_user_translation_list_index =
 278             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 279                                     u->sessions_per_user_list_head_index);
 280
 281           ASSERT (oldest_per_user_translation_list_index != ~0);
 282
 283           /* add it back to the end of the LRU list */
 284           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 285                               u->sessions_per_user_list_head_index,
 286                               oldest_per_user_translation_list_index);
 287           /* Get the list element */
 288           oldest_per_user_translation_list_elt =
 289             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 290                                oldest_per_user_translation_list_index);
 291
 292           /* Get the session index from the list element */
 293           session_index = oldest_per_user_translation_list_elt->value;
 294
 295           /* Get the session */
 296           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 297                                  session_index);
 298       } while (snat_is_session_static (s));
 299
 300       if (snat_is_unk_proto_session (s))
 301         {
 302           clib_bihash_kv_16_8_t up_kv;
 303           snat_unk_proto_ses_key_t key;
 304
 305           /* Remove from lookup tables */
 306           key.l_addr = s->in2out.addr;
 307           key.r_addr = s->ext_host_addr;
 308           key.fib_index = s->in2out.fib_index;
 309           key.proto = s->in2out.port;
 310           up_kv.key[0] = key.as_u64[0];
 311           up_kv.key[1] = key.as_u64[1];
 312           if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &up_kv, 0))
 313             clib_warning ("in2out key del failed");
 314
 315           key.l_addr = s->out2in.addr;
 316           key.fib_index = s->out2in.fib_index;
 317           up_kv.key[0] = key.as_u64[0];
 318           up_kv.key[1] = key.as_u64[1];
 319           if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &up_kv, 0))
 320             clib_warning ("out2in key del failed");
 321         }
 322       else
 323         {
 324           /* Remove in2out, out2in keys */
 325           kv0.key = s->in2out.as_u64;
 326           if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 327               clib_warning ("in2out key delete failed");
 328           kv0.key = s->out2in.as_u64;
 329           if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 330               clib_warning ("out2in key delete failed");
 331
 332           /* log NAT event */
 333           snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 334                                               s->out2in.addr.as_u32,
 335                                               s->in2out.protocol,
 336                                               s->in2out.port,
 337                                               s->out2in.port,
 338                                               s->in2out.fib_index);
 339
 340           snat_free_outside_address_and_port
 341             (sm, &s->out2in, s->outside_address_index);
 342         }
 343       s->outside_address_index = ~0;
 344
 345       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 346                                                &address_index))
 347         {
 348           ASSERT(0);
 349
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353       s->outside_address_index = address_index;
 354     }
 355   else
 356     {
 357       u8 static_mapping = 1;
 358
 359       /* First try to match static mapping by local address and port */
 360       if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
 361         {
 362           static_mapping = 0;
 363           /* Try to create dynamic translation */
 364           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 365                                                    &address_index))
 366             {
 367               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 368               return SNAT_IN2OUT_NEXT_DROP;
 369             }
 370         }
 371
 372       /* Create a new session */
 373       pool_get (sm->per_thread_data[thread_index].sessions, s);
 374       memset (s, 0, sizeof (*s));
 375
 376       s->outside_address_index = address_index;
 377
 378       if (static_mapping)
 379         {
 380           u->nstaticsessions++;
 381           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 382         }
 383       else
 384         {
 385           u->nsessions++;
 386         }
 387
 388       /* Create list elts */
 389       pool_get (sm->per_thread_data[thread_index].list_pool,
 390                 per_user_translation_list_elt);
 391       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 392                        per_user_translation_list_elt -
 393                        sm->per_thread_data[thread_index].list_pool);
 394
 395       per_user_translation_list_elt->value =
 396         s - sm->per_thread_data[thread_index].sessions;
 397       s->per_user_index = per_user_translation_list_elt -
 398                           sm->per_thread_data[thread_index].list_pool;
 399       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 400
 401       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 402                           s->per_user_list_head_index,
 403                           per_user_translation_list_elt -
 404                           sm->per_thread_data[thread_index].list_pool);
 405    }
 406
 407   s->in2out = *key0;
 408   s->out2in = key1;
 409   s->out2in.protocol = key0->protocol;
 410   s->out2in.fib_index = outside_fib_index;
 411   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 412   *sessionp = s;
 413
 414   /* Add to translation hashes */
 415   kv0.key = s->in2out.as_u64;
 416   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 417   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 418       clib_warning ("in2out key add failed");
 419
 420   kv0.key = s->out2in.as_u64;
 421   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 422
 423   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 424       clib_warning ("out2in key add failed");
 425
 426   /* Add to translated packets worker lookup */
 427   worker_by_out_key.addr = s->out2in.addr;
 428   worker_by_out_key.port = s->out2in.port;
 429   worker_by_out_key.fib_index = s->out2in.fib_index;
 430   kv0.key = worker_by_out_key.as_u64;
 431   kv0.value = thread_index;
 432   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 433
 434   /* log NAT event */
 435   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 436                                       s->out2in.addr.as_u32,
 437                                       s->in2out.protocol,
 438                                       s->in2out.port,
 439                                       s->out2in.port,
 440                                       s->in2out.fib_index);
 441   return next0;
 442 }
 443
 444 static_always_inline
 445 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 446                                  snat_session_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   snat_session_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.protocol = SNAT_PROTOCOL_ICMP;
 461       key0.addr = ip0->src_address;
 462       key0.port = echo0->identifier;
 463     }
 464   else
 465     {
 466       inner_ip0 = (ip4_header_t *)(echo0+1);
 467       l4_header = ip4_next_header (inner_ip0);
 468       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 469       key0.addr = inner_ip0->dst_address;
 470       switch (key0.protocol)
 471         {
 472         case SNAT_PROTOCOL_ICMP:
 473           inner_icmp0 = (icmp46_header_t*)l4_header;
 474           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 475           key0.port = inner_echo0->identifier;
 476           break;
 477         case SNAT_PROTOCOL_UDP:
 478         case SNAT_PROTOCOL_TCP:
 479           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 480           break;
 481         default:
 482           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 483         }
 484     }
 485   *p_key0 = key0;
 486   return -1; /* success */
 487 }
 488
 489 /**
 490  * Get address and port values to be used for packet SNAT translation
 491  * and create session if needed
 492  *
 493  * @param[in,out] sm             SNAT main
 494  * @param[in,out] node           SNAT node runtime
 495  * @param[in] thread_index       thread index
 496  * @param[in,out] b0             buffer containing packet to be translated
 497  * @param[out] p_proto           protocol used for matching
 498  * @param[out] p_value           address and port after NAT translation
 499  * @param[out] p_dont_translate  if packet should not be translated
 500  * @param d                      optional parameter
 501  * @param e                      optional parameter
 502  */
 503 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 504                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 505                            snat_session_key_t *p_value,
 506                            u8 *p_dont_translate, void *d, void *e)
 507 {
 508   ip4_header_t *ip0;
 509   icmp46_header_t *icmp0;
 510   u32 sw_if_index0;
 511   u32 rx_fib_index0;
 512   snat_session_key_t key0;
 513   snat_session_t *s0 = 0;
 514   u8 dont_translate = 0;
 515   clib_bihash_kv_8_8_t kv0, value0;
 516   u32 next0 = ~0;
 517   int err;
 518
 519   ip0 = vlib_buffer_get_current (b0);
 520   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 521   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 522   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 523
 524   err = icmp_get_key (ip0, &key0);
 525   if (err != -1)
 526     {
 527       b0->error = node->errors[err];
 528       next0 = SNAT_IN2OUT_NEXT_DROP;
 529       goto out;
 530     }
 531   key0.fib_index = rx_fib_index0;
 532
 533   kv0.key = key0.as_u64;
 534
 535   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 536     {
 537       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 538           IP_PROTOCOL_ICMP, rx_fib_index0)))
 539         {
 540           dont_translate = 1;
 541           goto out;
 542         }
 543
 544       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 545         {
 546           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 547           next0 = SNAT_IN2OUT_NEXT_DROP;
 548           goto out;
 549         }
 550
 551       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 552                          &s0, node, next0, thread_index);
 553
 554       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 555         goto out;
 556     }
 557   else
 558     {
 559       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 560                         icmp0->type != ICMP4_echo_reply &&
 561                         !icmp_is_error_message (icmp0)))
 562         {
 563           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 564           next0 = SNAT_IN2OUT_NEXT_DROP;
 565           goto out;
 566         }
 567
 568       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 569                               value0.value);
 570     }
 571
 572 out:
 573   *p_proto = key0.protocol;
 574   if (s0)
 575     *p_value = s0->out2in;
 576   *p_dont_translate = dont_translate;
 577   if (d)
 578     *(snat_session_t**)d = s0;
 579   return next0;
 580 }
 581
 582 /**
 583  * Get address and port values to be used for packet SNAT translation
 584  *
 585  * @param[in] sm                 SNAT main
 586  * @param[in,out] node           SNAT node runtime
 587  * @param[in] thread_index       thread index
 588  * @param[in,out] b0             buffer containing packet to be translated
 589  * @param[out] p_proto           protocol used for matching
 590  * @param[out] p_value           address and port after NAT translation
 591  * @param[out] p_dont_translate  if packet should not be translated
 592  * @param d                      optional parameter
 593  * @param e                      optional parameter
 594  */
 595 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 596                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 597                            snat_session_key_t *p_value,
 598                            u8 *p_dont_translate, void *d, void *e)
 599 {
 600   ip4_header_t *ip0;
 601   icmp46_header_t *icmp0;
 602   u32 sw_if_index0;
 603   u32 rx_fib_index0;
 604   snat_session_key_t key0;
 605   snat_session_key_t sm0;
 606   u8 dont_translate = 0;
 607   u8 is_addr_only;
 608   u32 next0 = ~0;
 609   int err;
 610
 611   ip0 = vlib_buffer_get_current (b0);
 612   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 613   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 614   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 615
 616   err = icmp_get_key (ip0, &key0);
 617   if (err != -1)
 618     {
 619       b0->error = node->errors[err];
 620       next0 = SNAT_IN2OUT_NEXT_DROP;
 621       goto out2;
 622     }
 623   key0.fib_index = rx_fib_index0;
 624
 625   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
 626     {
 627       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 628           IP_PROTOCOL_ICMP, rx_fib_index0)))
 629         {
 630           dont_translate = 1;
 631           goto out;
 632         }
 633
 634       if (icmp_is_error_message (icmp0))
 635         {
 636           next0 = SNAT_IN2OUT_NEXT_DROP;
 637           goto out;
 638         }
 639
 640       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 641       next0 = SNAT_IN2OUT_NEXT_DROP;
 642       goto out;
 643     }
 644
 645   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 646                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 647                     !icmp_is_error_message (icmp0)))
 648     {
 649       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 650       next0 = SNAT_IN2OUT_NEXT_DROP;
 651       goto out;
 652     }
 653
 654 out:
 655   *p_value = sm0;
 656 out2:
 657   *p_proto = key0.protocol;
 658   *p_dont_translate = dont_translate;
 659   return next0;
 660 }
 661
 662 static inline u32 icmp_in2out (snat_main_t *sm,
 663                                vlib_buffer_t * b0,
 664                                ip4_header_t * ip0,
 665                                icmp46_header_t * icmp0,
 666                                u32 sw_if_index0,
 667                                u32 rx_fib_index0,
 668                                vlib_node_runtime_t * node,
 669                                u32 next0,
 670                                u32 thread_index,
 671                                void *d,
 672                                void *e)
 673 {
 674   snat_session_key_t sm0;
 675   u8 protocol;
 676   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 677   ip4_header_t *inner_ip0;
 678   void *l4_header = 0;
 679   icmp46_header_t *inner_icmp0;
 680   u8 dont_translate;
 681   u32 new_addr0, old_addr0;
 682   u16 old_id0, new_id0;
 683   ip_csum_t sum0;
 684   u16 checksum0;
 685   u32 next0_tmp;
 686
 687   echo0 = (icmp_echo_header_t *)(icmp0+1);
 688
 689   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 690                                        &protocol, &sm0, &dont_translate, d, e);
 691   if (next0_tmp != ~0)
 692     next0 = next0_tmp;
 693   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 694     goto out;
 695
 696   sum0 = ip_incremental_checksum (0, icmp0,
 697                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 698   checksum0 = ~ip_csum_fold (sum0);
 699   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 700     {
 701       next0 = SNAT_IN2OUT_NEXT_DROP;
 702       goto out;
 703     }
 704
 705   old_addr0 = ip0->src_address.as_u32;
 706   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 707   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 708
 709   sum0 = ip0->checksum;
 710   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 711                          src_address /* changed member */);
 712   ip0->checksum = ip_csum_fold (sum0);
 713
 714   if (!icmp_is_error_message (icmp0))
 715     {
 716       new_id0 = sm0.port;
 717       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 718         {
 719           old_id0 = echo0->identifier;
 720           new_id0 = sm0.port;
 721           echo0->identifier = new_id0;
 722
 723           sum0 = icmp0->checksum;
 724           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 725                                  identifier);
 726           icmp0->checksum = ip_csum_fold (sum0);
 727         }
 728     }
 729   else
 730     {
 731       inner_ip0 = (ip4_header_t *)(echo0+1);
 732       l4_header = ip4_next_header (inner_ip0);
 733
 734       if (!ip4_header_checksum_is_valid (inner_ip0))
 735         {
 736           next0 = SNAT_IN2OUT_NEXT_DROP;
 737           goto out;
 738         }
 739
 740       old_addr0 = inner_ip0->dst_address.as_u32;
 741       inner_ip0->dst_address = sm0.addr;
 742       new_addr0 = inner_ip0->dst_address.as_u32;
 743
 744       sum0 = icmp0->checksum;
 745       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 746                              dst_address /* changed member */);
 747       icmp0->checksum = ip_csum_fold (sum0);
 748
 749       switch (protocol)
 750         {
 751           case SNAT_PROTOCOL_ICMP:
 752             inner_icmp0 = (icmp46_header_t*)l4_header;
 753             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 754
 755             old_id0 = inner_echo0->identifier;
 756             new_id0 = sm0.port;
 757             inner_echo0->identifier = new_id0;
 758
 759             sum0 = icmp0->checksum;
 760             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 761                                    identifier);
 762             icmp0->checksum = ip_csum_fold (sum0);
 763             break;
 764           case SNAT_PROTOCOL_UDP:
 765           case SNAT_PROTOCOL_TCP:
 766             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 767             new_id0 = sm0.port;
 768             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 769
 770             sum0 = icmp0->checksum;
 771             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 772                                    dst_port);
 773             icmp0->checksum = ip_csum_fold (sum0);
 774             break;
 775           default:
 776             ASSERT(0);
 777         }
 778     }
 779
 780 out:
 781   return next0;
 782 }
 783
 784 /**
 785  * @brief Hairpinning
 786  *
 787  * Hairpinning allows two endpoints on the internal side of the NAT to
 788  * communicate even if they only use each other's external IP addresses
 789  * and ports.
 790  *
 791  * @param sm     SNAT main.
 792  * @param b0     Vlib buffer.
 793  * @param ip0    IP header.
 794  * @param udp0   UDP header.
 795  * @param tcp0   TCP header.
 796  * @param proto0 SNAT protocol.
 797  */
 798 static inline void
 799 snat_hairpinning (snat_main_t *sm,
 800                   vlib_buffer_t * b0,
 801                   ip4_header_t * ip0,
 802                   udp_header_t * udp0,
 803                   tcp_header_t * tcp0,
 804                   u32 proto0)
 805 {
 806   snat_session_key_t key0, sm0;
 807   snat_worker_key_t k0;
 808   snat_session_t * s0;
 809   clib_bihash_kv_8_8_t kv0, value0;
 810   ip_csum_t sum0;
 811   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 812   u16 new_dst_port0, old_dst_port0;
 813
 814   key0.addr = ip0->dst_address;
 815   key0.port = udp0->dst_port;
 816   key0.protocol = proto0;
 817   key0.fib_index = sm->outside_fib_index;
 818   kv0.key = key0.as_u64;
 819
 820   /* Check if destination is in active sessions */
 821   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 822     {
 823       /* or static mappings */
 824       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 825         {
 826           new_dst_addr0 = sm0.addr.as_u32;
 827           new_dst_port0 = sm0.port;
 828           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 829         }
 830     }
 831   else
 832     {
 833       si = value0.value;
 834       if (sm->num_workers > 1)
 835         {
 836           k0.addr = ip0->dst_address;
 837           k0.port = udp0->dst_port;
 838           k0.fib_index = sm->outside_fib_index;
 839           kv0.key = k0.as_u64;
 840           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 841             ASSERT(0);
 842           else
 843             ti = value0.value;
 844         }
 845       else
 846         ti = sm->num_workers;
 847
 848       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 849       new_dst_addr0 = s0->in2out.addr.as_u32;
 850       new_dst_port0 = s0->in2out.port;
 851       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 852     }
 853
 854   /* Destination is behind the same NAT, use internal address and port */
 855   if (new_dst_addr0)
 856     {
 857       old_dst_addr0 = ip0->dst_address.as_u32;
 858       ip0->dst_address.as_u32 = new_dst_addr0;
 859       sum0 = ip0->checksum;
 860       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 861                              ip4_header_t, dst_address);
 862       ip0->checksum = ip_csum_fold (sum0);
 863
 864       old_dst_port0 = tcp0->dst;
 865       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 866         {
 867           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 868             {
 869               tcp0->dst = new_dst_port0;
 870               sum0 = tcp0->checksum;
 871               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 872                                      ip4_header_t, dst_address);
 873               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 874                                      ip4_header_t /* cheat */, length);
 875               tcp0->checksum = ip_csum_fold(sum0);
 876             }
 877           else
 878             {
 879               udp0->dst_port = new_dst_port0;
 880               udp0->checksum = 0;
 881             }
 882         }
 883       else
 884         {
 885           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 886             {
 887               sum0 = tcp0->checksum;
 888               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 889                                      ip4_header_t, dst_address);
 890               tcp0->checksum = ip_csum_fold(sum0);
 891             }
 892         }
 893     }
 894 }
 895
 896 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 897                                          vlib_buffer_t * b0,
 898                                          ip4_header_t * ip0,
 899                                          icmp46_header_t * icmp0,
 900                                          u32 sw_if_index0,
 901                                          u32 rx_fib_index0,
 902                                          vlib_node_runtime_t * node,
 903                                          u32 next0,
 904                                          f64 now,
 905                                          u32 thread_index,
 906                                          snat_session_t ** p_s0)
 907 {
 908   snat_session_key_t key0, sm0;
 909   clib_bihash_kv_8_8_t kv0, value0;
 910   snat_worker_key_t k0;
 911   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 912   ip_csum_t sum0;
 913
 914   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 915                       next0, thread_index, p_s0, 0);
 916   snat_session_t * s0 = *p_s0;
 917   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 918     {
 919       /* Hairpinning */
 920       if (!icmp_is_error_message (icmp0))
 921         {
 922           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 923           u16 icmp_id0 = echo0->identifier;
 924           key0.addr = ip0->dst_address;
 925           key0.port = icmp_id0;
 926           key0.protocol = SNAT_PROTOCOL_ICMP;
 927           key0.fib_index = sm->outside_fib_index;
 928           kv0.key = key0.as_u64;
 929
 930           /* Check if destination is in active sessions */
 931           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 932             {
 933               /* or static mappings */
 934               if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 935                 {
 936                   new_dst_addr0 = sm0.addr.as_u32;
 937                   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 938                 }
 939             }
 940           else
 941             {
 942               si = value0.value;
 943               if (sm->num_workers > 1)
 944                 {
 945                   k0.addr = ip0->dst_address;
 946                   k0.port = icmp_id0;
 947                   k0.fib_index = sm->outside_fib_index;
 948                   kv0.key = k0.as_u64;
 949                   if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 950                     ASSERT(0);
 951                   else
 952                     ti = value0.value;
 953                 }
 954               else
 955                 ti = sm->num_workers;
 956
 957               s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 958               new_dst_addr0 = s0->in2out.addr.as_u32;
 959               vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 960               echo0->identifier = s0->in2out.port;
 961               sum0 = icmp0->checksum;
 962               sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 963                                      icmp_echo_header_t, identifier);
 964               icmp0->checksum = ip_csum_fold (sum0);
 965             }
 966
 967           /* Destination is behind the same NAT, use internal address and port */
 968           if (new_dst_addr0)
 969             {
 970               old_dst_addr0 = ip0->dst_address.as_u32;
 971               ip0->dst_address.as_u32 = new_dst_addr0;
 972               sum0 = ip0->checksum;
 973               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 974                                      ip4_header_t, dst_address);
 975               ip0->checksum = ip_csum_fold (sum0);
 976             }
 977         }
 978
 979       /* Accounting */
 980       s0->last_heard = now;
 981       s0->total_pkts++;
 982       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 983       /* Per-user LRU list maintenance for dynamic translations */
 984       if (!snat_is_session_static (s0))
 985         {
 986           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 987                              s0->per_user_index);
 988           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 989                               s0->per_user_list_head_index,
 990                               s0->per_user_index);
 991         }
 992     }
 993   return next0;
 994 }
 995
 996 static void
 997 snat_in2out_unknown_proto (snat_main_t *sm,
 998                            vlib_buffer_t * b,
 999                            ip4_header_t * ip,
1000                            u32 rx_fib_index,
1001                            u32 thread_index,
1002                            f64 now,
1003                            vlib_main_t * vm)
1004 {
1005   clib_bihash_kv_8_8_t kv, value;
1006   clib_bihash_kv_16_8_t s_kv, s_value;
1007   snat_static_mapping_t *m;
1008   snat_session_key_t m_key;
1009   u32 old_addr, new_addr = 0;
1010   ip_csum_t sum;
1011   snat_user_key_t u_key;
1012   snat_user_t *u;
1013   dlist_elt_t *head, *elt, *oldest;
1014   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1015   u32 elt_index, head_index, ses_index, oldest_index;
1016   snat_session_t * s;
1017   snat_unk_proto_ses_key_t key;
1018   u32 address_index = ~0;
1019   int i;
1020   u8 is_sm = 0;
1021
1022   old_addr = ip->src_address.as_u32;
1023
1024   key.l_addr = ip->src_address;
1025   key.r_addr = ip->dst_address;
1026   key.fib_index = rx_fib_index;
1027   key.proto = ip->protocol;
1028   key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
1029   s_kv.key[0] = key.as_u64[0];
1030   s_kv.key[1] = key.as_u64[1];
1031
1032   if (!clib_bihash_search_16_8 (&sm->in2out_unk_proto, &s_kv, &s_value))
1033     {
1034       s = pool_elt_at_index (tsm->sessions, s_value.value);
1035       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1036     }
1037   else
1038     {
1039       u_key.addr = ip->src_address;
1040       u_key.fib_index = rx_fib_index;
1041       kv.key = u_key.as_u64;
1042
1043       /* Ever heard of the "user" = src ip4 address before? */
1044       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1045         {
1046           /* no, make a new one */
1047           pool_get (tsm->users, u);
1048           memset (u, 0, sizeof (*u));
1049           u->addr = ip->src_address;
1050           u->fib_index = rx_fib_index;
1051
1052           pool_get (tsm->list_pool, head);
1053           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1054
1055           clib_dlist_init (tsm->list_pool,
1056                            u->sessions_per_user_list_head_index);
1057
1058           kv.value = u - tsm->users;
1059
1060           /* add user */
1061           clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
1062         }
1063       else
1064         {
1065           u = pool_elt_at_index (tsm->users, value.value);
1066         }
1067
1068       m_key.addr = ip->src_address;
1069       m_key.port = 0;
1070       m_key.protocol = 0;
1071       m_key.fib_index = rx_fib_index;
1072       kv.key = m_key.as_u64;
1073
1074       /* Try to find static mapping first */
1075       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1076         {
1077           m = pool_elt_at_index (sm->static_mappings, value.value);
1078           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1079           is_sm = 1;
1080           goto create_ses;
1081         }
1082       /* Fallback to 3-tuple key */
1083       else
1084         {
1085           /* Choose same out address as for TCP/UDP session to same destination */
1086           if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1087             {
1088               head_index = u->sessions_per_user_list_head_index;
1089               head = pool_elt_at_index (tsm->list_pool, head_index);
1090               elt_index = head->next;
1091               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1092               ses_index = elt->value;
1093               while (ses_index != ~0)
1094                 {
1095                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1096                   elt_index = elt->next;
1097                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1098                   ses_index = elt->value;
1099
1100                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1101                     {
1102                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1103                       address_index = s->outside_address_index;
1104
1105                       key.fib_index = sm->outside_fib_index;
1106                       key.l_addr.as_u32 = new_addr;
1107                       s_kv.key[0] = key.as_u64[0];
1108                       s_kv.key[1] = key.as_u64[1];
1109                       if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1110                         break;
1111
1112                       goto create_ses;
1113                     }
1114                 }
1115             }
1116           key.fib_index = sm->outside_fib_index;
1117           for (i = 0; i < vec_len (sm->addresses); i++)
1118             {
1119               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1120               s_kv.key[0] = key.as_u64[0];
1121               s_kv.key[1] = key.as_u64[1];
1122               if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1123                 {
1124                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1125                   address_index = i;
1126                   goto create_ses;
1127                 }
1128             }
1129           return;
1130         }
1131
1132 create_ses:
1133       /* Over quota? Recycle the least recently used dynamic translation */
1134       if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1135         {
1136           /* Remove the oldest dynamic translation */
1137           do {
1138               oldest_index = clib_dlist_remove_head (
1139                 tsm->list_pool, u->sessions_per_user_list_head_index);
1140
1141               ASSERT (oldest_index != ~0);
1142
1143               /* add it back to the end of the LRU list */
1144               clib_dlist_addtail (tsm->list_pool,
1145                                   u->sessions_per_user_list_head_index,
1146                                   oldest_index);
1147               /* Get the list element */
1148               oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1149
1150               /* Get the session index from the list element */
1151               ses_index = oldest->value;
1152
1153               /* Get the session */
1154               s = pool_elt_at_index (tsm->sessions, ses_index);
1155           } while (snat_is_session_static (s));
1156
1157           if (snat_is_unk_proto_session (s))
1158             {
1159               /* Remove from lookup tables */
1160               key.l_addr = s->in2out.addr;
1161               key.r_addr = s->ext_host_addr;
1162               key.fib_index = s->in2out.fib_index;
1163               key.proto = s->in2out.port;
1164               s_kv.key[0] = key.as_u64[0];
1165               s_kv.key[1] = key.as_u64[1];
1166               if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 0))
1167                 clib_warning ("in2out key del failed");
1168
1169               key.l_addr = s->out2in.addr;
1170               key.fib_index = s->out2in.fib_index;
1171               s_kv.key[0] = key.as_u64[0];
1172               s_kv.key[1] = key.as_u64[1];
1173               if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 0))
1174                 clib_warning ("out2in key del failed");
1175             }
1176           else
1177             {
1178               /* log NAT event */
1179               snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1180                                                   s->out2in.addr.as_u32,
1181                                                   s->in2out.protocol,
1182                                                   s->in2out.port,
1183                                                   s->out2in.port,
1184                                                   s->in2out.fib_index);
1185
1186               snat_free_outside_address_and_port (sm, &s->out2in,
1187                                                   s->outside_address_index);
1188
1189               /* Remove in2out, out2in keys */
1190               kv.key = s->in2out.as_u64;
1191               if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
1192                 clib_warning ("in2out key del failed");
1193               kv.key = s->out2in.as_u64;
1194               if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
1195                 clib_warning ("out2in key del failed");
1196             }
1197         }
1198       else
1199         {
1200           /* Create a new session */
1201           pool_get (tsm->sessions, s);
1202           memset (s, 0, sizeof (*s));
1203
1204           /* Create list elts */
1205           pool_get (tsm->list_pool, elt);
1206           clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1207           elt->value = s - tsm->sessions;
1208           s->per_user_index = elt - tsm->list_pool;
1209           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1210           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1211                               s->per_user_index);
1212         }
1213
1214       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1215       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1216       s->outside_address_index = address_index;
1217       s->out2in.addr.as_u32 = new_addr;
1218       s->out2in.fib_index = sm->outside_fib_index;
1219       s->in2out.addr.as_u32 = old_addr;
1220       s->in2out.fib_index = rx_fib_index;
1221       s->in2out.port = s->out2in.port = ip->protocol;
1222       if (is_sm)
1223         {
1224           u->nstaticsessions++;
1225           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1226         }
1227       else
1228         {
1229           u->nsessions++;
1230         }
1231
1232       /* Add to lookup tables */
1233       key.l_addr.as_u32 = old_addr;
1234       key.r_addr = ip->dst_address;
1235       key.proto = ip->protocol;
1236       key.fib_index = rx_fib_index;
1237       s_kv.key[0] = key.as_u64[0];
1238       s_kv.key[1] = key.as_u64[1];
1239       s_kv.value = s - tsm->sessions;
1240       if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
1241         clib_warning ("in2out key add failed");
1242
1243       key.l_addr.as_u32 = new_addr;
1244       key.fib_index = sm->outside_fib_index;
1245       s_kv.key[0] = key.as_u64[0];
1246       s_kv.key[1] = key.as_u64[1];
1247       if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
1248         clib_warning ("out2in key add failed");
1249   }
1250
1251   /* Update IP checksum */
1252   sum = ip->checksum;
1253   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1254   ip->checksum = ip_csum_fold (sum);
1255
1256   /* Accounting */
1257   s->last_heard = now;
1258   s->total_pkts++;
1259   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1260   /* Per-user LRU list maintenance */
1261   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1262   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1263                       s->per_user_index);
1264
1265   /* Hairpinning */
1266   old_addr = ip->dst_address.as_u32;
1267   key.l_addr.as_u32 = ip->dst_address.as_u32;
1268   key.r_addr.as_u32 = new_addr;
1269   key.fib_index = sm->outside_fib_index;
1270   s_kv.key[0] = key.as_u64[0];
1271   s_kv.key[1] = key.as_u64[1];
1272   if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1273     {
1274       m_key.addr = ip->dst_address;
1275       m_key.fib_index = sm->outside_fib_index;
1276       kv.key = m_key.as_u64;
1277       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1278         {
1279           vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1280           return;
1281         }
1282
1283       m = pool_elt_at_index (sm->static_mappings, value.value);
1284       vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1285       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1286     }
1287   else
1288     {
1289       s = pool_elt_at_index (tsm->sessions, s_value.value);
1290       vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1291       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1292     }
1293   sum = ip->checksum;
1294   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1295   ip->checksum = ip_csum_fold (sum);
1296 }
1297
1298 static inline uword
1299 snat_in2out_node_fn_inline (vlib_main_t * vm,
1300                             vlib_node_runtime_t * node,
1301                             vlib_frame_t * frame, int is_slow_path)
1302 {
1303   u32 n_left_from, * from, * to_next;
1304   snat_in2out_next_t next_index;
1305   u32 pkts_processed = 0;
1306   snat_main_t * sm = &snat_main;
1307   f64 now = vlib_time_now (vm);
1308   u32 stats_node_index;
1309   u32 thread_index = vlib_get_thread_index ();
1310
1311   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1312     snat_in2out_node.index;
1313
1314   from = vlib_frame_vector_args (frame);
1315   n_left_from = frame->n_vectors;
1316   next_index = node->cached_next_index;
1317
1318   while (n_left_from > 0)
1319     {
1320       u32 n_left_to_next;
1321
1322       vlib_get_next_frame (vm, node, next_index,
1323                            to_next, n_left_to_next);
1324
1325       while (n_left_from >= 4 && n_left_to_next >= 2)
1326         {
1327           u32 bi0, bi1;
1328           vlib_buffer_t * b0, * b1;
1329           u32 next0, next1;
1330           u32 sw_if_index0, sw_if_index1;
1331           ip4_header_t * ip0, * ip1;
1332           ip_csum_t sum0, sum1;
1333           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1334           u16 old_port0, new_port0, old_port1, new_port1;
1335           udp_header_t * udp0, * udp1;
1336           tcp_header_t * tcp0, * tcp1;
1337           icmp46_header_t * icmp0, * icmp1;
1338           snat_session_key_t key0, key1;
1339           u32 rx_fib_index0, rx_fib_index1;
1340           u32 proto0, proto1;
1341           snat_session_t * s0 = 0, * s1 = 0;
1342           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1343
1344           /* Prefetch next iteration. */
1345           {
1346             vlib_buffer_t * p2, * p3;
1347
1348             p2 = vlib_get_buffer (vm, from[2]);
1349             p3 = vlib_get_buffer (vm, from[3]);
1350
1351             vlib_prefetch_buffer_header (p2, LOAD);
1352             vlib_prefetch_buffer_header (p3, LOAD);
1353
1354             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1355             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1356           }
1357
1358           /* speculatively enqueue b0 and b1 to the current next frame */
1359           to_next[0] = bi0 = from[0];
1360           to_next[1] = bi1 = from[1];
1361           from += 2;
1362           to_next += 2;
1363           n_left_from -= 2;
1364           n_left_to_next -= 2;
1365
1366           b0 = vlib_get_buffer (vm, bi0);
1367           b1 = vlib_get_buffer (vm, bi1);
1368
1369           ip0 = vlib_buffer_get_current (b0);
1370           udp0 = ip4_next_header (ip0);
1371           tcp0 = (tcp_header_t *) udp0;
1372           icmp0 = (icmp46_header_t *) udp0;
1373
1374           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1375           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1376                                    sw_if_index0);
1377
1378           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1379
1380           if (PREDICT_FALSE(ip0->ttl == 1))
1381             {
1382               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1383               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1384                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1385                                            0);
1386               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1387               goto trace00;
1388             }
1389
1390           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1391
1392           /* Next configured feature, probably ip4-lookup */
1393           if (is_slow_path)
1394             {
1395               if (PREDICT_FALSE (proto0 == ~0))
1396                 {
1397                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1398                                              thread_index, now, vm);
1399                   goto trace00;
1400                 }
1401
1402               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1403                 {
1404                   next0 = icmp_in2out_slow_path
1405                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1406                      node, next0, now, thread_index, &s0);
1407                   goto trace00;
1408                 }
1409             }
1410           else
1411             {
1412               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1413                 {
1414                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1415                   goto trace00;
1416                 }
1417             }
1418
1419           key0.addr = ip0->src_address;
1420           key0.port = udp0->src_port;
1421           key0.protocol = proto0;
1422           key0.fib_index = rx_fib_index0;
1423
1424           kv0.key = key0.as_u64;
1425
1426           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
1427             {
1428               if (is_slow_path)
1429                 {
1430                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
1431                       proto0, rx_fib_index0)))
1432                     goto trace00;
1433
1434                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1435                                      &s0, node, next0, thread_index);
1436                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1437                     goto trace00;
1438                 }
1439               else
1440                 {
1441                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1442                   goto trace00;
1443                 }
1444             }
1445           else
1446             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1447                                     value0.value);
1448
1449           old_addr0 = ip0->src_address.as_u32;
1450           ip0->src_address = s0->out2in.addr;
1451           new_addr0 = ip0->src_address.as_u32;
1452           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1453
1454           sum0 = ip0->checksum;
1455           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1456                                  ip4_header_t,
1457                                  src_address /* changed member */);
1458           ip0->checksum = ip_csum_fold (sum0);
1459
1460           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1461             {
1462               old_port0 = tcp0->src_port;
1463               tcp0->src_port = s0->out2in.port;
1464               new_port0 = tcp0->src_port;
1465
1466               sum0 = tcp0->checksum;
1467               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1468                                      ip4_header_t,
1469                                      dst_address /* changed member */);
1470               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1471                                      ip4_header_t /* cheat */,
1472                                      length /* changed member */);
1473               tcp0->checksum = ip_csum_fold(sum0);
1474             }
1475           else
1476             {
1477               old_port0 = udp0->src_port;
1478               udp0->src_port = s0->out2in.port;
1479               udp0->checksum = 0;
1480             }
1481
1482           /* Hairpinning */
1483           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1484
1485           /* Accounting */
1486           s0->last_heard = now;
1487           s0->total_pkts++;
1488           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1489           /* Per-user LRU list maintenance for dynamic translation */
1490           if (!snat_is_session_static (s0))
1491             {
1492               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1493                                  s0->per_user_index);
1494               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1495                                   s0->per_user_list_head_index,
1496                                   s0->per_user_index);
1497             }
1498         trace00:
1499
1500           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1501                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1502             {
1503               snat_in2out_trace_t *t =
1504                  vlib_add_trace (vm, node, b0, sizeof (*t));
1505               t->is_slow_path = is_slow_path;
1506               t->sw_if_index = sw_if_index0;
1507               t->next_index = next0;
1508                   t->session_index = ~0;
1509               if (s0)
1510                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1511             }
1512
1513           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1514
1515           ip1 = vlib_buffer_get_current (b1);
1516           udp1 = ip4_next_header (ip1);
1517           tcp1 = (tcp_header_t *) udp1;
1518           icmp1 = (icmp46_header_t *) udp1;
1519
1520           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1521           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1522                                    sw_if_index1);
1523
1524           if (PREDICT_FALSE(ip1->ttl == 1))
1525             {
1526               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1527               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1528                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1529                                            0);
1530               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1531               goto trace01;
1532             }
1533
1534           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1535
1536           /* Next configured feature, probably ip4-lookup */
1537           if (is_slow_path)
1538             {
1539               if (PREDICT_FALSE (proto1 == ~0))
1540                 {
1541                   snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1542                                              thread_index, now, vm);
1543                   goto trace01;
1544                 }
1545
1546               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1547                 {
1548                   next1 = icmp_in2out_slow_path
1549                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1550                      next1, now, thread_index, &s1);
1551                   goto trace01;
1552                 }
1553             }
1554           else
1555             {
1556               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1557                 {
1558                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1559                   goto trace01;
1560                 }
1561             }
1562
1563           key1.addr = ip1->src_address;
1564           key1.port = udp1->src_port;
1565           key1.protocol = proto1;
1566           key1.fib_index = rx_fib_index1;
1567
1568           kv1.key = key1.as_u64;
1569
1570             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
1571             {
1572               if (is_slow_path)
1573                 {
1574                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1, ip1,
1575                       proto1, rx_fib_index1)))
1576                     goto trace01;
1577
1578                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1579                                      &s1, node, next1, thread_index);
1580                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1581                     goto trace01;
1582                 }
1583               else
1584                 {
1585                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1586                   goto trace01;
1587                 }
1588             }
1589           else
1590             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1591                                     value1.value);
1592
1593           old_addr1 = ip1->src_address.as_u32;
1594           ip1->src_address = s1->out2in.addr;
1595           new_addr1 = ip1->src_address.as_u32;
1596           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1597
1598           sum1 = ip1->checksum;
1599           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1600                                  ip4_header_t,
1601                                  src_address /* changed member */);
1602           ip1->checksum = ip_csum_fold (sum1);
1603
1604           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1605             {
1606               old_port1 = tcp1->src_port;
1607               tcp1->src_port = s1->out2in.port;
1608               new_port1 = tcp1->src_port;
1609
1610               sum1 = tcp1->checksum;
1611               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1612                                      ip4_header_t,
1613                                      dst_address /* changed member */);
1614               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1615                                      ip4_header_t /* cheat */,
1616                                      length /* changed member */);
1617               tcp1->checksum = ip_csum_fold(sum1);
1618             }
1619           else
1620             {
1621               old_port1 = udp1->src_port;
1622               udp1->src_port = s1->out2in.port;
1623               udp1->checksum = 0;
1624             }
1625
1626           /* Hairpinning */
1627           snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1628
1629           /* Accounting */
1630           s1->last_heard = now;
1631           s1->total_pkts++;
1632           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1633           /* Per-user LRU list maintenance for dynamic translation */
1634           if (!snat_is_session_static (s1))
1635             {
1636               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1637                                  s1->per_user_index);
1638               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1639                                   s1->per_user_list_head_index,
1640                                   s1->per_user_index);
1641             }
1642         trace01:
1643
1644           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1645                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1646             {
1647               snat_in2out_trace_t *t =
1648                  vlib_add_trace (vm, node, b1, sizeof (*t));
1649               t->sw_if_index = sw_if_index1;
1650               t->next_index = next1;
1651               t->session_index = ~0;
1652               if (s1)
1653                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1654             }
1655
1656           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1657
1658           /* verify speculative enqueues, maybe switch current next frame */
1659           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1660                                            to_next, n_left_to_next,
1661                                            bi0, bi1, next0, next1);
1662         }
1663
1664       while (n_left_from > 0 && n_left_to_next > 0)
1665         {
1666           u32 bi0;
1667           vlib_buffer_t * b0;
1668           u32 next0;
1669           u32 sw_if_index0;
1670           ip4_header_t * ip0;
1671           ip_csum_t sum0;
1672           u32 new_addr0, old_addr0;
1673           u16 old_port0, new_port0;
1674           udp_header_t * udp0;
1675           tcp_header_t * tcp0;
1676           icmp46_header_t * icmp0;
1677           snat_session_key_t key0;
1678           u32 rx_fib_index0;
1679           u32 proto0;
1680           snat_session_t * s0 = 0;
1681           clib_bihash_kv_8_8_t kv0, value0;
1682
1683           /* speculatively enqueue b0 to the current next frame */
1684           bi0 = from[0];
1685           to_next[0] = bi0;
1686           from += 1;
1687           to_next += 1;
1688           n_left_from -= 1;
1689           n_left_to_next -= 1;
1690
1691           b0 = vlib_get_buffer (vm, bi0);
1692           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1693
1694           ip0 = vlib_buffer_get_current (b0);
1695           udp0 = ip4_next_header (ip0);
1696           tcp0 = (tcp_header_t *) udp0;
1697           icmp0 = (icmp46_header_t *) udp0;
1698
1699           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1700           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1701                                    sw_if_index0);
1702
1703           if (PREDICT_FALSE(ip0->ttl == 1))
1704             {
1705               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1706               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1707                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1708                                            0);
1709               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1710               goto trace0;
1711             }
1712
1713           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1714
1715           /* Next configured feature, probably ip4-lookup */
1716           if (is_slow_path)
1717             {
1718               if (PREDICT_FALSE (proto0 == ~0))
1719                 {
1720                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1721                                              thread_index, now, vm);
1722                   goto trace0;
1723                 }
1724
1725               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1726                 {
1727                   next0 = icmp_in2out_slow_path
1728                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1729                      next0, now, thread_index, &s0);
1730                   goto trace0;
1731                 }
1732             }
1733           else
1734             {
1735               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1736                 {
1737                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1738                   goto trace0;
1739                 }
1740             }
1741
1742           key0.addr = ip0->src_address;
1743           key0.port = udp0->src_port;
1744           key0.protocol = proto0;
1745           key0.fib_index = rx_fib_index0;
1746
1747           kv0.key = key0.as_u64;
1748
1749           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
1750             {
1751               if (is_slow_path)
1752                 {
1753                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
1754                       proto0, rx_fib_index0)))
1755                     goto trace0;
1756
1757                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1758                                      &s0, node, next0, thread_index);
1759
1760                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1761                     goto trace0;
1762                 }
1763               else
1764                 {
1765                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1766                   goto trace0;
1767                 }
1768             }
1769           else
1770             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1771                                     value0.value);
1772
1773           old_addr0 = ip0->src_address.as_u32;
1774           ip0->src_address = s0->out2in.addr;
1775           new_addr0 = ip0->src_address.as_u32;
1776           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1777
1778           sum0 = ip0->checksum;
1779           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1780                                  ip4_header_t,
1781                                  src_address /* changed member */);
1782           ip0->checksum = ip_csum_fold (sum0);
1783
1784           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1785             {
1786               old_port0 = tcp0->src_port;
1787               tcp0->src_port = s0->out2in.port;
1788               new_port0 = tcp0->src_port;
1789
1790               sum0 = tcp0->checksum;
1791               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1792                                      ip4_header_t,
1793                                      dst_address /* changed member */);
1794               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1795                                      ip4_header_t /* cheat */,
1796                                      length /* changed member */);
1797               tcp0->checksum = ip_csum_fold(sum0);
1798             }
1799           else
1800             {
1801               old_port0 = udp0->src_port;
1802               udp0->src_port = s0->out2in.port;
1803               udp0->checksum = 0;
1804             }
1805
1806           /* Hairpinning */
1807           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1808
1809           /* Accounting */
1810           s0->last_heard = now;
1811           s0->total_pkts++;
1812           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1813           /* Per-user LRU list maintenance for dynamic translation */
1814           if (!snat_is_session_static (s0))
1815             {
1816               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1817                                  s0->per_user_index);
1818               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1819                                   s0->per_user_list_head_index,
1820                                   s0->per_user_index);
1821             }
1822
1823         trace0:
1824           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1825                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1826             {
1827               snat_in2out_trace_t *t =
1828                  vlib_add_trace (vm, node, b0, sizeof (*t));
1829               t->is_slow_path = is_slow_path;
1830               t->sw_if_index = sw_if_index0;
1831               t->next_index = next0;
1832                   t->session_index = ~0;
1833               if (s0)
1834                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1835             }
1836
1837           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1838
1839           /* verify speculative enqueue, maybe switch current next frame */
1840           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1841                                            to_next, n_left_to_next,
1842                                            bi0, next0);
1843         }
1844
1845       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1846     }
1847
1848   vlib_node_increment_counter (vm, stats_node_index,
1849                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1850                                pkts_processed);
1851   return frame->n_vectors;
1852 }
1853
1854 static uword
1855 snat_in2out_fast_path_fn (vlib_main_t * vm,
1856                           vlib_node_runtime_t * node,
1857                           vlib_frame_t * frame)
1858 {
1859   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
1860 }
1861
1862 VLIB_REGISTER_NODE (snat_in2out_node) = {
1863   .function = snat_in2out_fast_path_fn,
1864   .name = "snat-in2out",
1865   .vector_size = sizeof (u32),
1866   .format_trace = format_snat_in2out_trace,
1867   .type = VLIB_NODE_TYPE_INTERNAL,
1868
1869   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1870   .error_strings = snat_in2out_error_strings,
1871
1872   .runtime_data_bytes = sizeof (snat_runtime_t),
1873
1874   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1875
1876   /* edit / add dispositions here */
1877   .next_nodes = {
1878     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1879     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1880     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1881     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1882   },
1883 };
1884
1885 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1886
1887 static uword
1888 snat_in2out_slow_path_fn (vlib_main_t * vm,
1889                           vlib_node_runtime_t * node,
1890                           vlib_frame_t * frame)
1891 {
1892   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
1893 }
1894
1895 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1896   .function = snat_in2out_slow_path_fn,
1897   .name = "snat-in2out-slowpath",
1898   .vector_size = sizeof (u32),
1899   .format_trace = format_snat_in2out_trace,
1900   .type = VLIB_NODE_TYPE_INTERNAL,
1901
1902   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1903   .error_strings = snat_in2out_error_strings,
1904
1905   .runtime_data_bytes = sizeof (snat_runtime_t),
1906
1907   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1908
1909   /* edit / add dispositions here */
1910   .next_nodes = {
1911     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1912     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1913     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1914     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1915   },
1916 };
1917
1918 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1919
1920 /**************************/
1921 /*** deterministic mode ***/
1922 /**************************/
1923 static uword
1924 snat_det_in2out_node_fn (vlib_main_t * vm,
1925                          vlib_node_runtime_t * node,
1926                          vlib_frame_t * frame)
1927 {
1928   u32 n_left_from, * from, * to_next;
1929   snat_in2out_next_t next_index;
1930   u32 pkts_processed = 0;
1931   snat_main_t * sm = &snat_main;
1932   u32 now = (u32) vlib_time_now (vm);
1933   u32 thread_index = vlib_get_thread_index ();
1934
1935   from = vlib_frame_vector_args (frame);
1936   n_left_from = frame->n_vectors;
1937   next_index = node->cached_next_index;
1938
1939   while (n_left_from > 0)
1940     {
1941       u32 n_left_to_next;
1942
1943       vlib_get_next_frame (vm, node, next_index,
1944                            to_next, n_left_to_next);
1945
1946       while (n_left_from >= 4 && n_left_to_next >= 2)
1947         {
1948           u32 bi0, bi1;
1949           vlib_buffer_t * b0, * b1;
1950           u32 next0, next1;
1951           u32 sw_if_index0, sw_if_index1;
1952           ip4_header_t * ip0, * ip1;
1953           ip_csum_t sum0, sum1;
1954           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1955           u16 old_port0, new_port0, lo_port0, i0;
1956           u16 old_port1, new_port1, lo_port1, i1;
1957           udp_header_t * udp0, * udp1;
1958           tcp_header_t * tcp0, * tcp1;
1959           u32 proto0, proto1;
1960           snat_det_out_key_t key0, key1;
1961           snat_det_map_t * dm0, * dm1;
1962           snat_det_session_t * ses0 = 0, * ses1 = 0;
1963           u32 rx_fib_index0, rx_fib_index1;
1964           icmp46_header_t * icmp0, * icmp1;
1965
1966           /* Prefetch next iteration. */
1967           {
1968             vlib_buffer_t * p2, * p3;
1969
1970             p2 = vlib_get_buffer (vm, from[2]);
1971             p3 = vlib_get_buffer (vm, from[3]);
1972
1973             vlib_prefetch_buffer_header (p2, LOAD);
1974             vlib_prefetch_buffer_header (p3, LOAD);
1975
1976             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1977             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1978           }
1979
1980           /* speculatively enqueue b0 and b1 to the current next frame */
1981           to_next[0] = bi0 = from[0];
1982           to_next[1] = bi1 = from[1];
1983           from += 2;
1984           to_next += 2;
1985           n_left_from -= 2;
1986           n_left_to_next -= 2;
1987
1988           b0 = vlib_get_buffer (vm, bi0);
1989           b1 = vlib_get_buffer (vm, bi1);
1990
1991           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1992           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1993
1994           ip0 = vlib_buffer_get_current (b0);
1995           udp0 = ip4_next_header (ip0);
1996           tcp0 = (tcp_header_t *) udp0;
1997
1998           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1999
2000           if (PREDICT_FALSE(ip0->ttl == 1))
2001             {
2002               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2003               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2004                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2005                                            0);
2006               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2007               goto trace0;
2008             }
2009
2010           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2011
2012           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2013             {
2014               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2015               icmp0 = (icmp46_header_t *) udp0;
2016
2017               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2018                                   rx_fib_index0, node, next0, thread_index,
2019                                   &ses0, &dm0);
2020               goto trace0;
2021             }
2022
2023           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2024           if (PREDICT_FALSE(!dm0))
2025             {
2026               clib_warning("no match for internal host %U",
2027                            format_ip4_address, &ip0->src_address);
2028               next0 = SNAT_IN2OUT_NEXT_DROP;
2029               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2030               goto trace0;
2031             }
2032
2033           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2034
2035           key0.ext_host_addr = ip0->dst_address;
2036           key0.ext_host_port = tcp0->dst;
2037
2038           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2039           if (PREDICT_FALSE(!ses0))
2040             {
2041               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2042                 {
2043                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2044                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2045
2046                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2047                     continue;
2048
2049                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2050                   break;
2051                 }
2052               if (PREDICT_FALSE(!ses0))
2053                 {
2054                   /* too many sessions for user, send ICMP error packet */
2055
2056                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2057                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2058                                                ICMP4_destination_unreachable_destination_unreachable_host,
2059                                                0);
2060                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2061                   goto trace0;
2062                 }
2063             }
2064
2065           new_port0 = ses0->out.out_port;
2066
2067           old_addr0.as_u32 = ip0->src_address.as_u32;
2068           ip0->src_address.as_u32 = new_addr0.as_u32;
2069           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2070
2071           sum0 = ip0->checksum;
2072           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2073                                  ip4_header_t,
2074                                  src_address /* changed member */);
2075           ip0->checksum = ip_csum_fold (sum0);
2076
2077           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2078             {
2079               if (tcp0->flags & TCP_FLAG_SYN)
2080                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2081               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2082                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2083               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2084                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2085               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2086                 snat_det_ses_close(dm0, ses0);
2087               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2088                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2089               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2090                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2091
2092               old_port0 = tcp0->src;
2093               tcp0->src = new_port0;
2094
2095               sum0 = tcp0->checksum;
2096               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2097                                      ip4_header_t,
2098                                      dst_address /* changed member */);
2099               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2100                                      ip4_header_t /* cheat */,
2101                                      length /* changed member */);
2102               tcp0->checksum = ip_csum_fold(sum0);
2103             }
2104           else
2105             {
2106               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2107               old_port0 = udp0->src_port;
2108               udp0->src_port = new_port0;
2109               udp0->checksum = 0;
2110             }
2111
2112           switch(ses0->state)
2113             {
2114             case SNAT_SESSION_UDP_ACTIVE:
2115                 ses0->expire = now + sm->udp_timeout;
2116                 break;
2117             case SNAT_SESSION_TCP_SYN_SENT:
2118             case SNAT_SESSION_TCP_FIN_WAIT:
2119             case SNAT_SESSION_TCP_CLOSE_WAIT:
2120             case SNAT_SESSION_TCP_LAST_ACK:
2121                 ses0->expire = now + sm->tcp_transitory_timeout;
2122                 break;
2123             case SNAT_SESSION_TCP_ESTABLISHED:
2124                 ses0->expire = now + sm->tcp_established_timeout;
2125                 break;
2126             }
2127
2128         trace0:
2129           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2130                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2131             {
2132               snat_in2out_trace_t *t =
2133                  vlib_add_trace (vm, node, b0, sizeof (*t));
2134               t->is_slow_path = 0;
2135               t->sw_if_index = sw_if_index0;
2136               t->next_index = next0;
2137               t->session_index = ~0;
2138               if (ses0)
2139                 t->session_index = ses0 - dm0->sessions;
2140             }
2141
2142           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2143
2144           ip1 = vlib_buffer_get_current (b1);
2145           udp1 = ip4_next_header (ip1);
2146           tcp1 = (tcp_header_t *) udp1;
2147
2148           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2149
2150           if (PREDICT_FALSE(ip1->ttl == 1))
2151             {
2152               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2153               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2154                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2155                                            0);
2156               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2157               goto trace1;
2158             }
2159
2160           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2161
2162           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2163             {
2164               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2165               icmp1 = (icmp46_header_t *) udp1;
2166
2167               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2168                                   rx_fib_index1, node, next1, thread_index,
2169                                   &ses1, &dm1);
2170               goto trace1;
2171             }
2172
2173           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2174           if (PREDICT_FALSE(!dm1))
2175             {
2176               clib_warning("no match for internal host %U",
2177                            format_ip4_address, &ip0->src_address);
2178               next1 = SNAT_IN2OUT_NEXT_DROP;
2179               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2180               goto trace1;
2181             }
2182
2183           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2184
2185           key1.ext_host_addr = ip1->dst_address;
2186           key1.ext_host_port = tcp1->dst;
2187
2188           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2189           if (PREDICT_FALSE(!ses1))
2190             {
2191               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2192                 {
2193                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2194                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2195
2196                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2197                     continue;
2198
2199                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2200                   break;
2201                 }
2202               if (PREDICT_FALSE(!ses1))
2203                 {
2204                   /* too many sessions for user, send ICMP error packet */
2205
2206                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2207                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2208                                                ICMP4_destination_unreachable_destination_unreachable_host,
2209                                                0);
2210                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2211                   goto trace1;
2212                 }
2213             }
2214
2215           new_port1 = ses1->out.out_port;
2216
2217           old_addr1.as_u32 = ip1->src_address.as_u32;
2218           ip1->src_address.as_u32 = new_addr1.as_u32;
2219           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2220
2221           sum1 = ip1->checksum;
2222           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2223                                  ip4_header_t,
2224                                  src_address /* changed member */);
2225           ip1->checksum = ip_csum_fold (sum1);
2226
2227           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2228             {
2229               if (tcp1->flags & TCP_FLAG_SYN)
2230                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2231               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2232                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2233               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2234                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2235               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2236                 snat_det_ses_close(dm1, ses1);
2237               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2238                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2239               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2240                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2241
2242               old_port1 = tcp1->src;
2243               tcp1->src = new_port1;
2244
2245               sum1 = tcp1->checksum;
2246               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2247                                      ip4_header_t,
2248                                      dst_address /* changed member */);
2249               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2250                                      ip4_header_t /* cheat */,
2251                                      length /* changed member */);
2252               tcp1->checksum = ip_csum_fold(sum1);
2253             }
2254           else
2255             {
2256               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2257               old_port1 = udp1->src_port;
2258               udp1->src_port = new_port1;
2259               udp1->checksum = 0;
2260             }
2261
2262           switch(ses1->state)
2263             {
2264             case SNAT_SESSION_UDP_ACTIVE:
2265                 ses1->expire = now + sm->udp_timeout;
2266                 break;
2267             case SNAT_SESSION_TCP_SYN_SENT:
2268             case SNAT_SESSION_TCP_FIN_WAIT:
2269             case SNAT_SESSION_TCP_CLOSE_WAIT:
2270             case SNAT_SESSION_TCP_LAST_ACK:
2271                 ses1->expire = now + sm->tcp_transitory_timeout;
2272                 break;
2273             case SNAT_SESSION_TCP_ESTABLISHED:
2274                 ses1->expire = now + sm->tcp_established_timeout;
2275                 break;
2276             }
2277
2278         trace1:
2279           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2280                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2281             {
2282               snat_in2out_trace_t *t =
2283                  vlib_add_trace (vm, node, b1, sizeof (*t));
2284               t->is_slow_path = 0;
2285               t->sw_if_index = sw_if_index1;
2286               t->next_index = next1;
2287               t->session_index = ~0;
2288               if (ses1)
2289                 t->session_index = ses1 - dm1->sessions;
2290             }
2291
2292           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2293
2294           /* verify speculative enqueues, maybe switch current next frame */
2295           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2296                                            to_next, n_left_to_next,
2297                                            bi0, bi1, next0, next1);
2298          }
2299
2300       while (n_left_from > 0 && n_left_to_next > 0)
2301         {
2302           u32 bi0;
2303           vlib_buffer_t * b0;
2304           u32 next0;
2305           u32 sw_if_index0;
2306           ip4_header_t * ip0;
2307           ip_csum_t sum0;
2308           ip4_address_t new_addr0, old_addr0;
2309           u16 old_port0, new_port0, lo_port0, i0;
2310           udp_header_t * udp0;
2311           tcp_header_t * tcp0;
2312           u32 proto0;
2313           snat_det_out_key_t key0;
2314           snat_det_map_t * dm0;
2315           snat_det_session_t * ses0 = 0;
2316           u32 rx_fib_index0;
2317           icmp46_header_t * icmp0;
2318
2319           /* speculatively enqueue b0 to the current next frame */
2320           bi0 = from[0];
2321           to_next[0] = bi0;
2322           from += 1;
2323           to_next += 1;
2324           n_left_from -= 1;
2325           n_left_to_next -= 1;
2326
2327           b0 = vlib_get_buffer (vm, bi0);
2328           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2329
2330           ip0 = vlib_buffer_get_current (b0);
2331           udp0 = ip4_next_header (ip0);
2332           tcp0 = (tcp_header_t *) udp0;
2333
2334           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2335
2336           if (PREDICT_FALSE(ip0->ttl == 1))
2337             {
2338               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2339               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2340                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2341                                            0);
2342               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2343               goto trace00;
2344             }
2345
2346           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2347
2348           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2349             {
2350               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2351               icmp0 = (icmp46_header_t *) udp0;
2352
2353               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2354                                   rx_fib_index0, node, next0, thread_index,
2355                                   &ses0, &dm0);
2356               goto trace00;
2357             }
2358
2359           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2360           if (PREDICT_FALSE(!dm0))
2361             {
2362               clib_warning("no match for internal host %U",
2363                            format_ip4_address, &ip0->src_address);
2364               next0 = SNAT_IN2OUT_NEXT_DROP;
2365               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2366               goto trace00;
2367             }
2368
2369           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2370
2371           key0.ext_host_addr = ip0->dst_address;
2372           key0.ext_host_port = tcp0->dst;
2373
2374           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2375           if (PREDICT_FALSE(!ses0))
2376             {
2377               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2378                 {
2379                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2380                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2381
2382                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2383                     continue;
2384
2385                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2386                   break;
2387                 }
2388               if (PREDICT_FALSE(!ses0))
2389                 {
2390                   /* too many sessions for user, send ICMP error packet */
2391
2392                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2393                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2394                                                ICMP4_destination_unreachable_destination_unreachable_host,
2395                                                0);
2396                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2397                   goto trace00;
2398                 }
2399             }
2400
2401           new_port0 = ses0->out.out_port;
2402
2403           old_addr0.as_u32 = ip0->src_address.as_u32;
2404           ip0->src_address.as_u32 = new_addr0.as_u32;
2405           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2406
2407           sum0 = ip0->checksum;
2408           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2409                                  ip4_header_t,
2410                                  src_address /* changed member */);
2411           ip0->checksum = ip_csum_fold (sum0);
2412
2413           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2414             {
2415               if (tcp0->flags & TCP_FLAG_SYN)
2416                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2417               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2418                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2419               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2420                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2421               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2422                 snat_det_ses_close(dm0, ses0);
2423               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2424                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2425               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2426                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2427
2428               old_port0 = tcp0->src;
2429               tcp0->src = new_port0;
2430
2431               sum0 = tcp0->checksum;
2432               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2433                                      ip4_header_t,
2434                                      dst_address /* changed member */);
2435               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2436                                      ip4_header_t /* cheat */,
2437                                      length /* changed member */);
2438               tcp0->checksum = ip_csum_fold(sum0);
2439             }
2440           else
2441             {
2442               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2443               old_port0 = udp0->src_port;
2444               udp0->src_port = new_port0;
2445               udp0->checksum = 0;
2446             }
2447
2448           switch(ses0->state)
2449             {
2450             case SNAT_SESSION_UDP_ACTIVE:
2451                 ses0->expire = now + sm->udp_timeout;
2452                 break;
2453             case SNAT_SESSION_TCP_SYN_SENT:
2454             case SNAT_SESSION_TCP_FIN_WAIT:
2455             case SNAT_SESSION_TCP_CLOSE_WAIT:
2456             case SNAT_SESSION_TCP_LAST_ACK:
2457                 ses0->expire = now + sm->tcp_transitory_timeout;
2458                 break;
2459             case SNAT_SESSION_TCP_ESTABLISHED:
2460                 ses0->expire = now + sm->tcp_established_timeout;
2461                 break;
2462             }
2463
2464         trace00:
2465           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2466                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2467             {
2468               snat_in2out_trace_t *t =
2469                  vlib_add_trace (vm, node, b0, sizeof (*t));
2470               t->is_slow_path = 0;
2471               t->sw_if_index = sw_if_index0;
2472               t->next_index = next0;
2473               t->session_index = ~0;
2474               if (ses0)
2475                 t->session_index = ses0 - dm0->sessions;
2476             }
2477
2478           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2479
2480           /* verify speculative enqueue, maybe switch current next frame */
2481           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2482                                            to_next, n_left_to_next,
2483                                            bi0, next0);
2484         }
2485
2486       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2487     }
2488
2489   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2490                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2491                                pkts_processed);
2492   return frame->n_vectors;
2493 }
2494
2495 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2496   .function = snat_det_in2out_node_fn,
2497   .name = "snat-det-in2out",
2498   .vector_size = sizeof (u32),
2499   .format_trace = format_snat_in2out_trace,
2500   .type = VLIB_NODE_TYPE_INTERNAL,
2501
2502   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2503   .error_strings = snat_in2out_error_strings,
2504
2505   .runtime_data_bytes = sizeof (snat_runtime_t),
2506
2507   .n_next_nodes = 3,
2508
2509   /* edit / add dispositions here */
2510   .next_nodes = {
2511     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2512     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2513     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2514   },
2515 };
2516
2517 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2518
2519 /**
2520  * Get address and port values to be used for packet SNAT translation
2521  * and create session if needed
2522  *
2523  * @param[in,out] sm             SNAT main
2524  * @param[in,out] node           SNAT node runtime
2525  * @param[in] thread_index       thread index
2526  * @param[in,out] b0             buffer containing packet to be translated
2527  * @param[out] p_proto           protocol used for matching
2528  * @param[out] p_value           address and port after NAT translation
2529  * @param[out] p_dont_translate  if packet should not be translated
2530  * @param d                      optional parameter
2531  * @param e                      optional parameter
2532  */
2533 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
2534                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2535                           snat_session_key_t *p_value,
2536                           u8 *p_dont_translate, void *d, void *e)
2537 {
2538   ip4_header_t *ip0;
2539   icmp46_header_t *icmp0;
2540   u32 sw_if_index0;
2541   u32 rx_fib_index0;
2542   u8 protocol;
2543   snat_det_out_key_t key0;
2544   u8 dont_translate = 0;
2545   u32 next0 = ~0;
2546   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2547   ip4_header_t *inner_ip0;
2548   void *l4_header = 0;
2549   icmp46_header_t *inner_icmp0;
2550   snat_det_map_t * dm0 = 0;
2551   ip4_address_t new_addr0;
2552   u16 lo_port0, i0;
2553   snat_det_session_t * ses0 = 0;
2554   ip4_address_t in_addr;
2555   u16 in_port;
2556
2557   ip0 = vlib_buffer_get_current (b0);
2558   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2559   echo0 = (icmp_echo_header_t *)(icmp0+1);
2560   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2561   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2562
2563   if (!icmp_is_error_message (icmp0))
2564     {
2565       protocol = SNAT_PROTOCOL_ICMP;
2566       in_addr = ip0->src_address;
2567       in_port = echo0->identifier;
2568     }
2569   else
2570     {
2571       inner_ip0 = (ip4_header_t *)(echo0+1);
2572       l4_header = ip4_next_header (inner_ip0);
2573       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2574       in_addr = inner_ip0->dst_address;
2575       switch (protocol)
2576         {
2577         case SNAT_PROTOCOL_ICMP:
2578           inner_icmp0 = (icmp46_header_t*)l4_header;
2579           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2580           in_port = inner_echo0->identifier;
2581           break;
2582         case SNAT_PROTOCOL_UDP:
2583         case SNAT_PROTOCOL_TCP:
2584           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2585           break;
2586         default:
2587           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
2588           next0 = SNAT_IN2OUT_NEXT_DROP;
2589           goto out;
2590         }
2591     }
2592
2593   dm0 = snat_det_map_by_user(sm, &in_addr);
2594   if (PREDICT_FALSE(!dm0))
2595     {
2596       clib_warning("no match for internal host %U",
2597                    format_ip4_address, &in_addr);
2598       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2599           IP_PROTOCOL_ICMP, rx_fib_index0)))
2600         {
2601           dont_translate = 1;
2602           goto out;
2603         }
2604       next0 = SNAT_IN2OUT_NEXT_DROP;
2605       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2606       goto out;
2607     }
2608
2609   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
2610
2611   key0.ext_host_addr = ip0->dst_address;
2612   key0.ext_host_port = 0;
2613
2614   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
2615   if (PREDICT_FALSE(!ses0))
2616     {
2617       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2618           IP_PROTOCOL_ICMP, rx_fib_index0)))
2619         {
2620           dont_translate = 1;
2621           goto out;
2622         }
2623       if (icmp0->type != ICMP4_echo_request)
2624         {
2625           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2626           next0 = SNAT_IN2OUT_NEXT_DROP;
2627           goto out;
2628         }
2629       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2630         {
2631           key0.out_port = clib_host_to_net_u16 (lo_port0 +
2632             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
2633
2634           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
2635             continue;
2636
2637           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
2638           break;
2639         }
2640       if (PREDICT_FALSE(!ses0))
2641         {
2642           next0 = SNAT_IN2OUT_NEXT_DROP;
2643           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2644           goto out;
2645         }
2646     }
2647
2648   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
2649                     !icmp_is_error_message (icmp0)))
2650     {
2651       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2652       next0 = SNAT_IN2OUT_NEXT_DROP;
2653       goto out;
2654     }
2655
2656   u32 now = (u32) vlib_time_now (sm->vlib_main);
2657
2658   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
2659   ses0->expire = now + sm->icmp_timeout;
2660
2661 out:
2662   *p_proto = protocol;
2663   if (ses0)
2664     {
2665       p_value->addr = new_addr0;
2666       p_value->fib_index = sm->outside_fib_index;
2667       p_value->port = ses0->out.out_port;
2668     }
2669   *p_dont_translate = dont_translate;
2670   if (d)
2671     *(snat_det_session_t**)d = ses0;
2672   if (e)
2673     *(snat_det_map_t**)e = dm0;
2674   return next0;
2675 }
2676
2677 /**********************/
2678 /*** worker handoff ***/
2679 /**********************/
2680 static uword
2681 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
2682                                vlib_node_runtime_t * node,
2683                                vlib_frame_t * frame)
2684 {
2685   snat_main_t *sm = &snat_main;
2686   vlib_thread_main_t *tm = vlib_get_thread_main ();
2687   u32 n_left_from, *from, *to_next = 0;
2688   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2689   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2690     = 0;
2691   vlib_frame_queue_elt_t *hf = 0;
2692   vlib_frame_t *f = 0;
2693   int i;
2694   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2695   u32 next_worker_index = 0;
2696   u32 current_worker_index = ~0;
2697   u32 thread_index = vlib_get_thread_index ();
2698
2699   ASSERT (vec_len (sm->workers));
2700
2701   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2702     {
2703       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2704
2705       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2706                                sm->first_worker_index + sm->num_workers - 1,
2707                                (vlib_frame_queue_t *) (~0));
2708     }
2709
2710   from = vlib_frame_vector_args (frame);
2711   n_left_from = frame->n_vectors;
2712
2713   while (n_left_from > 0)
2714     {
2715       u32 bi0;
2716       vlib_buffer_t *b0;
2717       u32 sw_if_index0;
2718       u32 rx_fib_index0;
2719       ip4_header_t * ip0;
2720       u8 do_handoff;
2721
2722       bi0 = from[0];
2723       from += 1;
2724       n_left_from -= 1;
2725
2726       b0 = vlib_get_buffer (vm, bi0);
2727
2728       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2729       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2730
2731       ip0 = vlib_buffer_get_current (b0);
2732
2733       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
2734
2735       if (PREDICT_FALSE (next_worker_index != thread_index))
2736         {
2737           do_handoff = 1;
2738
2739           if (next_worker_index != current_worker_index)
2740             {
2741               if (hf)
2742                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2743
2744               hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index,
2745                                                       next_worker_index,
2746                                                       handoff_queue_elt_by_worker_index);
2747
2748               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2749               to_next_worker = &hf->buffer_index[hf->n_vectors];
2750               current_worker_index = next_worker_index;
2751             }
2752
2753           /* enqueue to correct worker thread */
2754           to_next_worker[0] = bi0;
2755           to_next_worker++;
2756           n_left_to_next_worker--;
2757
2758           if (n_left_to_next_worker == 0)
2759             {
2760               hf->n_vectors = VLIB_FRAME_SIZE;
2761               vlib_put_frame_queue_elt (hf);
2762               current_worker_index = ~0;
2763               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2764               hf = 0;
2765             }
2766         }
2767       else
2768         {
2769           do_handoff = 0;
2770           /* if this is 1st frame */
2771           if (!f)
2772             {
2773               f = vlib_get_frame_to_node (vm, sm->in2out_node_index);
2774               to_next = vlib_frame_vector_args (f);
2775             }
2776
2777           to_next[0] = bi0;
2778           to_next += 1;
2779           f->n_vectors++;
2780         }
2781
2782       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2783                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2784         {
2785           snat_in2out_worker_handoff_trace_t *t =
2786             vlib_add_trace (vm, node, b0, sizeof (*t));
2787           t->next_worker_index = next_worker_index;
2788           t->do_handoff = do_handoff;
2789         }
2790     }
2791
2792   if (f)
2793     vlib_put_frame_to_node (vm, sm->in2out_node_index, f);
2794
2795   if (hf)
2796     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2797
2798   /* Ship frames to the worker nodes */
2799   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2800     {
2801       if (handoff_queue_elt_by_worker_index[i])
2802         {
2803           hf = handoff_queue_elt_by_worker_index[i];
2804           /*
2805            * It works better to let the handoff node
2806            * rate-adapt, always ship the handoff queue element.
2807            */
2808           if (1 || hf->n_vectors == hf->last_n_vectors)
2809             {
2810               vlib_put_frame_queue_elt (hf);
2811               handoff_queue_elt_by_worker_index[i] = 0;
2812             }
2813           else
2814             hf->last_n_vectors = hf->n_vectors;
2815         }
2816       congested_handoff_queue_by_worker_index[i] =
2817         (vlib_frame_queue_t *) (~0);
2818     }
2819   hf = 0;
2820   current_worker_index = ~0;
2821   return frame->n_vectors;
2822 }
2823
2824 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
2825   .function = snat_in2out_worker_handoff_fn,
2826   .name = "snat-in2out-worker-handoff",
2827   .vector_size = sizeof (u32),
2828   .format_trace = format_snat_in2out_worker_handoff_trace,
2829   .type = VLIB_NODE_TYPE_INTERNAL,
2830
2831   .n_next_nodes = 1,
2832
2833   .next_nodes = {
2834     [0] = "error-drop",
2835   },
2836 };
2837
2838 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn);
2839
2840 static uword
2841 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
2842                                 vlib_node_runtime_t * node,
2843                                 vlib_frame_t * frame)
2844 {
2845   u32 n_left_from, * from, * to_next;
2846   snat_in2out_next_t next_index;
2847   u32 pkts_processed = 0;
2848   snat_main_t * sm = &snat_main;
2849   u32 stats_node_index;
2850
2851   stats_node_index = snat_in2out_fast_node.index;
2852
2853   from = vlib_frame_vector_args (frame);
2854   n_left_from = frame->n_vectors;
2855   next_index = node->cached_next_index;
2856
2857   while (n_left_from > 0)
2858     {
2859       u32 n_left_to_next;
2860
2861       vlib_get_next_frame (vm, node, next_index,
2862                            to_next, n_left_to_next);
2863
2864       while (n_left_from > 0 && n_left_to_next > 0)
2865         {
2866           u32 bi0;
2867           vlib_buffer_t * b0;
2868           u32 next0;
2869           u32 sw_if_index0;
2870           ip4_header_t * ip0;
2871           ip_csum_t sum0;
2872           u32 new_addr0, old_addr0;
2873           u16 old_port0, new_port0;
2874           udp_header_t * udp0;
2875           tcp_header_t * tcp0;
2876           icmp46_header_t * icmp0;
2877           snat_session_key_t key0, sm0;
2878           u32 proto0;
2879           u32 rx_fib_index0;
2880
2881           /* speculatively enqueue b0 to the current next frame */
2882           bi0 = from[0];
2883           to_next[0] = bi0;
2884           from += 1;
2885           to_next += 1;
2886           n_left_from -= 1;
2887           n_left_to_next -= 1;
2888
2889           b0 = vlib_get_buffer (vm, bi0);
2890           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2891
2892           ip0 = vlib_buffer_get_current (b0);
2893           udp0 = ip4_next_header (ip0);
2894           tcp0 = (tcp_header_t *) udp0;
2895           icmp0 = (icmp46_header_t *) udp0;
2896
2897           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2898           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2899
2900           if (PREDICT_FALSE(ip0->ttl == 1))
2901             {
2902               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2903               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2904                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2905                                            0);
2906               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2907               goto trace0;
2908             }
2909
2910           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2911
2912           if (PREDICT_FALSE (proto0 == ~0))
2913               goto trace0;
2914
2915           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2916             {
2917               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2918                                   rx_fib_index0, node, next0, ~0, 0, 0);
2919               goto trace0;
2920             }
2921
2922           key0.addr = ip0->src_address;
2923           key0.port = udp0->src_port;
2924           key0.fib_index = rx_fib_index0;
2925
2926           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
2927             {
2928               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2929               next0= SNAT_IN2OUT_NEXT_DROP;
2930               goto trace0;
2931             }
2932
2933           new_addr0 = sm0.addr.as_u32;
2934           new_port0 = sm0.port;
2935           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2936           old_addr0 = ip0->src_address.as_u32;
2937           ip0->src_address.as_u32 = new_addr0;
2938
2939           sum0 = ip0->checksum;
2940           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2941                                  ip4_header_t,
2942                                  src_address /* changed member */);
2943           ip0->checksum = ip_csum_fold (sum0);
2944
2945           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2946             {
2947               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2948                 {
2949                   old_port0 = tcp0->src_port;
2950                   tcp0->src_port = new_port0;
2951
2952                   sum0 = tcp0->checksum;
2953                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2954                                          ip4_header_t,
2955                                          dst_address /* changed member */);
2956                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2957                                          ip4_header_t /* cheat */,
2958                                          length /* changed member */);
2959                   tcp0->checksum = ip_csum_fold(sum0);
2960                 }
2961               else
2962                 {
2963                   old_port0 = udp0->src_port;
2964                   udp0->src_port = new_port0;
2965                   udp0->checksum = 0;
2966                 }
2967             }
2968           else
2969             {
2970               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2971                 {
2972                   sum0 = tcp0->checksum;
2973                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2974                                          ip4_header_t,
2975                                          dst_address /* changed member */);
2976                   tcp0->checksum = ip_csum_fold(sum0);
2977                 }
2978             }
2979
2980           /* Hairpinning */
2981           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2982
2983         trace0:
2984           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2985                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2986             {
2987               snat_in2out_trace_t *t =
2988                  vlib_add_trace (vm, node, b0, sizeof (*t));
2989               t->sw_if_index = sw_if_index0;
2990               t->next_index = next0;
2991             }
2992
2993           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2994
2995           /* verify speculative enqueue, maybe switch current next frame */
2996           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2997                                            to_next, n_left_to_next,
2998                                            bi0, next0);
2999         }
3000
3001       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3002     }
3003
3004   vlib_node_increment_counter (vm, stats_node_index,
3005                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3006                                pkts_processed);
3007   return frame->n_vectors;
3008 }
3009
3010
3011 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3012   .function = snat_in2out_fast_static_map_fn,
3013   .name = "snat-in2out-fast",
3014   .vector_size = sizeof (u32),
3015   .format_trace = format_snat_in2out_fast_trace,
3016   .type = VLIB_NODE_TYPE_INTERNAL,
3017
3018   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3019   .error_strings = snat_in2out_error_strings,
3020
3021   .runtime_data_bytes = sizeof (snat_runtime_t),
3022
3023   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3024
3025   /* edit / add dispositions here */
3026   .next_nodes = {
3027     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3028     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3029     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
3030     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3031   },
3032 };
3033
3034 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);