src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91 vlib_node_registration_t snat_in2out_output_node;
  92 vlib_node_registration_t snat_in2out_output_slowpath_node;
  93 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
  94 vlib_node_registration_t snat_hairpin_dst_node;
  95 vlib_node_registration_t snat_hairpin_src_node;
  96
  97
  98 #define foreach_snat_in2out_error                       \
  99 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 100 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 101 _(OUT_OF_PORTS, "Out of ports")                         \
 102 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 103 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 104 _(NO_TRANSLATION, "No translation")
 105
 106 typedef enum {
 107 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 108   foreach_snat_in2out_error
 109 #undef _
 110   SNAT_IN2OUT_N_ERROR,
 111 } snat_in2out_error_t;
 112
 113 static char * snat_in2out_error_strings[] = {
 114 #define _(sym,string) string,
 115   foreach_snat_in2out_error
 116 #undef _
 117 };
 118
 119 typedef enum {
 120   SNAT_IN2OUT_NEXT_LOOKUP,
 121   SNAT_IN2OUT_NEXT_DROP,
 122   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 123   SNAT_IN2OUT_NEXT_SLOW_PATH,
 124   SNAT_IN2OUT_N_NEXT,
 125 } snat_in2out_next_t;
 126
 127 typedef enum {
 128   SNAT_HAIRPIN_SRC_NEXT_DROP,
 129   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 130   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 131   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 132   SNAT_HAIRPIN_SRC_N_NEXT,
 133 } snat_hairpin_next_t;
 134
 135 /**
 136  * @brief Check if packet should be translated
 137  *
 138  * Packets aimed at outside interface and external addresss with active session
 139  * should be translated.
 140  *
 141  * @param sm            NAT main
 142  * @param rt            NAT runtime data
 143  * @param sw_if_index0  index of the inside interface
 144  * @param ip0           IPv4 header
 145  * @param proto0        NAT protocol
 146  * @param rx_fib_index0 RX FIB index
 147  *
 148  * @returns 0 if packet should be translated otherwise 1
 149  */
 150 static inline int
 151 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 152                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 153                          u32 rx_fib_index0)
 154 {
 155   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 156   fib_prefix_t pfx = {
 157     .fp_proto = FIB_PROTOCOL_IP4,
 158     .fp_len = 32,
 159     .fp_addr = {
 160         .ip4.as_u32 = ip0->dst_address.as_u32,
 161     },
 162   };
 163
 164   /* Don't NAT packet aimed at the intfc address */
 165   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 166                                       ip0->dst_address.as_u32)))
 167     return 1;
 168
 169   fei = fib_table_lookup (rx_fib_index0, &pfx);
 170   if (FIB_NODE_INDEX_INVALID != fei)
 171     {
 172       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 173       if (sw_if_index == ~0)
 174         {
 175           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 176           if (FIB_NODE_INDEX_INVALID != fei)
 177             sw_if_index = fib_entry_get_resolving_interface (fei);
 178         }
 179       snat_interface_t *i;
 180       pool_foreach (i, sm->interfaces,
 181       ({
 182         /* NAT packet aimed at outside interface */
 183         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 184           return 0;
 185       }));
 186     }
 187
 188   return 1;
 189 }
 190
 191 static inline int
 192 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 193                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 194                     u32 rx_fib_index0, u32 thread_index)
 195 {
 196   udp_header_t * udp0 = ip4_next_header (ip0);
 197   snat_session_key_t key0, sm0;
 198   clib_bihash_kv_8_8_t kv0, value0;
 199
 200   key0.addr = ip0->dst_address;
 201   key0.port = udp0->dst_port;
 202   key0.protocol = proto0;
 203   key0.fib_index = sm->outside_fib_index;
 204   kv0.key = key0.as_u64;
 205
 206   /* NAT packet aimed at external address if */
 207   /* has active sessions */
 208   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 209                               &value0))
 210     {
 211       /* or is static mappings */
 212       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 213         return 0;
 214     }
 215   else
 216     return 0;
 217
 218   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 219                                  rx_fib_index0);
 220 }
 221
 222 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 223                       ip4_header_t * ip0,
 224                       u32 rx_fib_index0,
 225                       snat_session_key_t * key0,
 226                       snat_session_t ** sessionp,
 227                       vlib_node_runtime_t * node,
 228                       u32 next0,
 229                       u32 thread_index)
 230 {
 231   snat_user_t *u;
 232   snat_user_key_t user_key;
 233   snat_session_t *s;
 234   clib_bihash_kv_8_8_t kv0, value0;
 235   u32 oldest_per_user_translation_list_index;
 236   dlist_elt_t * oldest_per_user_translation_list_elt;
 237   dlist_elt_t * per_user_translation_list_elt;
 238   dlist_elt_t * per_user_list_head_elt;
 239   u32 session_index;
 240   snat_session_key_t key1;
 241   u32 address_index = ~0;
 242   u32 outside_fib_index;
 243   uword * p;
 244
 245   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 246   if (! p)
 247     {
 248       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 249       return SNAT_IN2OUT_NEXT_DROP;
 250     }
 251   outside_fib_index = p[0];
 252
 253   key1.protocol = key0->protocol;
 254   user_key.addr = ip0->src_address;
 255   user_key.fib_index = rx_fib_index0;
 256   kv0.key = user_key.as_u64;
 257
 258   /* Ever heard of the "user" = src ip4 address before? */
 259   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 260                               &kv0, &value0))
 261     {
 262       /* no, make a new one */
 263       pool_get (sm->per_thread_data[thread_index].users, u);
 264       memset (u, 0, sizeof (*u));
 265       u->addr = ip0->src_address;
 266       u->fib_index = rx_fib_index0;
 267
 268       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 269
 270       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 271         sm->per_thread_data[thread_index].list_pool;
 272
 273       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 274                        u->sessions_per_user_list_head_index);
 275
 276       kv0.value = u - sm->per_thread_data[thread_index].users;
 277
 278       /* add user */
 279       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 280                                &kv0, 1 /* is_add */);
 281     }
 282   else
 283     {
 284       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 285                              value0.value);
 286     }
 287
 288   /* Over quota? Recycle the least recently used dynamic translation */
 289   if (u->nsessions >= sm->max_translations_per_user)
 290     {
 291       /* Remove the oldest dynamic translation */
 292       do {
 293           oldest_per_user_translation_list_index =
 294             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 295                                     u->sessions_per_user_list_head_index);
 296
 297           ASSERT (oldest_per_user_translation_list_index != ~0);
 298
 299           /* add it back to the end of the LRU list */
 300           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 301                               u->sessions_per_user_list_head_index,
 302                               oldest_per_user_translation_list_index);
 303           /* Get the list element */
 304           oldest_per_user_translation_list_elt =
 305             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 306                                oldest_per_user_translation_list_index);
 307
 308           /* Get the session index from the list element */
 309           session_index = oldest_per_user_translation_list_elt->value;
 310
 311           /* Get the session */
 312           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 313                                  session_index);
 314       } while (snat_is_session_static (s));
 315
 316       if (snat_is_unk_proto_session (s))
 317         {
 318           clib_bihash_kv_16_8_t up_kv;
 319           nat_ed_ses_key_t key;
 320
 321           /* Remove from lookup tables */
 322           key.l_addr = s->in2out.addr;
 323           key.r_addr = s->ext_host_addr;
 324           key.fib_index = s->in2out.fib_index;
 325           key.proto = s->in2out.port;
 326           key.rsvd = 0;
 327           key.l_port = 0;
 328           up_kv.key[0] = key.as_u64[0];
 329           up_kv.key[1] = key.as_u64[1];
 330           if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &up_kv, 0))
 331             clib_warning ("in2out key del failed");
 332
 333           key.l_addr = s->out2in.addr;
 334           key.fib_index = s->out2in.fib_index;
 335           up_kv.key[0] = key.as_u64[0];
 336           up_kv.key[1] = key.as_u64[1];
 337           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &up_kv, 0))
 338             clib_warning ("out2in key del failed");
 339         }
 340       else
 341         {
 342           /* Remove in2out, out2in keys */
 343           kv0.key = s->in2out.as_u64;
 344           if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out,
 345                                        &kv0, 0 /* is_add */))
 346               clib_warning ("in2out key delete failed");
 347           kv0.key = s->out2in.as_u64;
 348           if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in,
 349                                        &kv0, 0 /* is_add */))
 350               clib_warning ("out2in key delete failed");
 351
 352           /* log NAT event */
 353           snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 354                                               s->out2in.addr.as_u32,
 355                                               s->in2out.protocol,
 356                                               s->in2out.port,
 357                                               s->out2in.port,
 358                                               s->in2out.fib_index);
 359
 360           snat_free_outside_address_and_port
 361             (sm, thread_index, &s->out2in, s->outside_address_index);
 362         }
 363       s->outside_address_index = ~0;
 364
 365       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, thread_index,
 366                                                &key1, &address_index))
 367         {
 368           ASSERT(0);
 369
 370           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 371           return SNAT_IN2OUT_NEXT_DROP;
 372         }
 373       s->outside_address_index = address_index;
 374     }
 375   else
 376     {
 377       u8 static_mapping = 1;
 378
 379       /* First try to match static mapping by local address and port */
 380       if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
 381         {
 382           static_mapping = 0;
 383           /* Try to create dynamic translation */
 384           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0,
 385                                                    thread_index, &key1,
 386                                                    &address_index))
 387             {
 388               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 389               return SNAT_IN2OUT_NEXT_DROP;
 390             }
 391         }
 392
 393       /* Create a new session */
 394       pool_get (sm->per_thread_data[thread_index].sessions, s);
 395       memset (s, 0, sizeof (*s));
 396
 397       s->outside_address_index = address_index;
 398
 399       if (static_mapping)
 400         {
 401           u->nstaticsessions++;
 402           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 403         }
 404       else
 405         {
 406           u->nsessions++;
 407         }
 408
 409       /* Create list elts */
 410       pool_get (sm->per_thread_data[thread_index].list_pool,
 411                 per_user_translation_list_elt);
 412       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 413                        per_user_translation_list_elt -
 414                        sm->per_thread_data[thread_index].list_pool);
 415
 416       per_user_translation_list_elt->value =
 417         s - sm->per_thread_data[thread_index].sessions;
 418       s->per_user_index = per_user_translation_list_elt -
 419                           sm->per_thread_data[thread_index].list_pool;
 420       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 421
 422       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 423                           s->per_user_list_head_index,
 424                           per_user_translation_list_elt -
 425                           sm->per_thread_data[thread_index].list_pool);
 426    }
 427
 428   s->in2out = *key0;
 429   s->out2in = key1;
 430   s->out2in.protocol = key0->protocol;
 431   s->out2in.fib_index = outside_fib_index;
 432   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 433   *sessionp = s;
 434
 435   /* Add to translation hashes */
 436   kv0.key = s->in2out.as_u64;
 437   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 438   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 439                                1 /* is_add */))
 440       clib_warning ("in2out key add failed");
 441
 442   kv0.key = s->out2in.as_u64;
 443   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 444
 445   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 446                                1 /* is_add */))
 447       clib_warning ("out2in key add failed");
 448
 449   /* log NAT event */
 450   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 451                                       s->out2in.addr.as_u32,
 452                                       s->in2out.protocol,
 453                                       s->in2out.port,
 454                                       s->out2in.port,
 455                                       s->in2out.fib_index);
 456   return next0;
 457 }
 458
 459 static_always_inline
 460 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 461                                  snat_session_key_t *p_key0)
 462 {
 463   icmp46_header_t *icmp0;
 464   snat_session_key_t key0;
 465   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 466   ip4_header_t *inner_ip0 = 0;
 467   void *l4_header = 0;
 468   icmp46_header_t *inner_icmp0;
 469
 470   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 471   echo0 = (icmp_echo_header_t *)(icmp0+1);
 472
 473   if (!icmp_is_error_message (icmp0))
 474     {
 475       key0.protocol = SNAT_PROTOCOL_ICMP;
 476       key0.addr = ip0->src_address;
 477       key0.port = echo0->identifier;
 478     }
 479   else
 480     {
 481       inner_ip0 = (ip4_header_t *)(echo0+1);
 482       l4_header = ip4_next_header (inner_ip0);
 483       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 484       key0.addr = inner_ip0->dst_address;
 485       switch (key0.protocol)
 486         {
 487         case SNAT_PROTOCOL_ICMP:
 488           inner_icmp0 = (icmp46_header_t*)l4_header;
 489           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 490           key0.port = inner_echo0->identifier;
 491           break;
 492         case SNAT_PROTOCOL_UDP:
 493         case SNAT_PROTOCOL_TCP:
 494           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 495           break;
 496         default:
 497           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 498         }
 499     }
 500   *p_key0 = key0;
 501   return -1; /* success */
 502 }
 503
 504 /**
 505  * Get address and port values to be used for ICMP packet translation
 506  * and create session if needed
 507  *
 508  * @param[in,out] sm             NAT main
 509  * @param[in,out] node           NAT node runtime
 510  * @param[in] thread_index       thread index
 511  * @param[in,out] b0             buffer containing packet to be translated
 512  * @param[out] p_proto           protocol used for matching
 513  * @param[out] p_value           address and port after NAT translation
 514  * @param[out] p_dont_translate  if packet should not be translated
 515  * @param d                      optional parameter
 516  * @param e                      optional parameter
 517  */
 518 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 519                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 520                            snat_session_key_t *p_value,
 521                            u8 *p_dont_translate, void *d, void *e)
 522 {
 523   ip4_header_t *ip0;
 524   icmp46_header_t *icmp0;
 525   u32 sw_if_index0;
 526   u32 rx_fib_index0;
 527   snat_session_key_t key0;
 528   snat_session_t *s0 = 0;
 529   u8 dont_translate = 0;
 530   clib_bihash_kv_8_8_t kv0, value0;
 531   u32 next0 = ~0;
 532   int err;
 533   u32 iph_offset0 = 0;
 534
 535   if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
 536     {
 537       iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 538     }
 539   ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
 540   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 541   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 542   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 543
 544   err = icmp_get_key (ip0, &key0);
 545   if (err != -1)
 546     {
 547       b0->error = node->errors[err];
 548       next0 = SNAT_IN2OUT_NEXT_DROP;
 549       goto out;
 550     }
 551   key0.fib_index = rx_fib_index0;
 552
 553   kv0.key = key0.as_u64;
 554
 555   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 556                               &value0))
 557     {
 558       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 559           IP_PROTOCOL_ICMP, rx_fib_index0, thread_index) &&
 560           vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
 561         {
 562           dont_translate = 1;
 563           goto out;
 564         }
 565
 566       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 567         {
 568           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 569           next0 = SNAT_IN2OUT_NEXT_DROP;
 570           goto out;
 571         }
 572
 573       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 574                          &s0, node, next0, thread_index);
 575
 576       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 577         goto out;
 578     }
 579   else
 580     {
 581       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 582                         icmp0->type != ICMP4_echo_reply &&
 583                         !icmp_is_error_message (icmp0)))
 584         {
 585           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 586           next0 = SNAT_IN2OUT_NEXT_DROP;
 587           goto out;
 588         }
 589
 590       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 591                               value0.value);
 592     }
 593
 594 out:
 595   *p_proto = key0.protocol;
 596   if (s0)
 597     *p_value = s0->out2in;
 598   *p_dont_translate = dont_translate;
 599   if (d)
 600     *(snat_session_t**)d = s0;
 601   return next0;
 602 }
 603
 604 /**
 605  * Get address and port values to be used for ICMP packet translation
 606  *
 607  * @param[in] sm                 NAT main
 608  * @param[in,out] node           NAT node runtime
 609  * @param[in] thread_index       thread index
 610  * @param[in,out] b0             buffer containing packet to be translated
 611  * @param[out] p_proto           protocol used for matching
 612  * @param[out] p_value           address and port after NAT translation
 613  * @param[out] p_dont_translate  if packet should not be translated
 614  * @param d                      optional parameter
 615  * @param e                      optional parameter
 616  */
 617 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 618                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 619                            snat_session_key_t *p_value,
 620                            u8 *p_dont_translate, void *d, void *e)
 621 {
 622   ip4_header_t *ip0;
 623   icmp46_header_t *icmp0;
 624   u32 sw_if_index0;
 625   u32 rx_fib_index0;
 626   snat_session_key_t key0;
 627   snat_session_key_t sm0;
 628   u8 dont_translate = 0;
 629   u8 is_addr_only;
 630   u32 next0 = ~0;
 631   int err;
 632
 633   ip0 = vlib_buffer_get_current (b0);
 634   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 635   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 636   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 637
 638   err = icmp_get_key (ip0, &key0);
 639   if (err != -1)
 640     {
 641       b0->error = node->errors[err];
 642       next0 = SNAT_IN2OUT_NEXT_DROP;
 643       goto out2;
 644     }
 645   key0.fib_index = rx_fib_index0;
 646
 647   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
 648     {
 649       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 650           IP_PROTOCOL_ICMP, rx_fib_index0)))
 651         {
 652           dont_translate = 1;
 653           goto out;
 654         }
 655
 656       if (icmp_is_error_message (icmp0))
 657         {
 658           next0 = SNAT_IN2OUT_NEXT_DROP;
 659           goto out;
 660         }
 661
 662       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 663       next0 = SNAT_IN2OUT_NEXT_DROP;
 664       goto out;
 665     }
 666
 667   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 668                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 669                     !icmp_is_error_message (icmp0)))
 670     {
 671       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 672       next0 = SNAT_IN2OUT_NEXT_DROP;
 673       goto out;
 674     }
 675
 676 out:
 677   *p_value = sm0;
 678 out2:
 679   *p_proto = key0.protocol;
 680   *p_dont_translate = dont_translate;
 681   return next0;
 682 }
 683
 684 static inline u32 icmp_in2out (snat_main_t *sm,
 685                                vlib_buffer_t * b0,
 686                                ip4_header_t * ip0,
 687                                icmp46_header_t * icmp0,
 688                                u32 sw_if_index0,
 689                                u32 rx_fib_index0,
 690                                vlib_node_runtime_t * node,
 691                                u32 next0,
 692                                u32 thread_index,
 693                                void *d,
 694                                void *e)
 695 {
 696   snat_session_key_t sm0;
 697   u8 protocol;
 698   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 699   ip4_header_t *inner_ip0;
 700   void *l4_header = 0;
 701   icmp46_header_t *inner_icmp0;
 702   u8 dont_translate;
 703   u32 new_addr0, old_addr0;
 704   u16 old_id0, new_id0;
 705   ip_csum_t sum0;
 706   u16 checksum0;
 707   u32 next0_tmp;
 708
 709   echo0 = (icmp_echo_header_t *)(icmp0+1);
 710
 711   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 712                                        &protocol, &sm0, &dont_translate, d, e);
 713   if (next0_tmp != ~0)
 714     next0 = next0_tmp;
 715   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 716     goto out;
 717
 718   sum0 = ip_incremental_checksum (0, icmp0,
 719                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 720   checksum0 = ~ip_csum_fold (sum0);
 721   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 722     {
 723       next0 = SNAT_IN2OUT_NEXT_DROP;
 724       goto out;
 725     }
 726
 727   old_addr0 = ip0->src_address.as_u32;
 728   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 729   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 730     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 731
 732   sum0 = ip0->checksum;
 733   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 734                          src_address /* changed member */);
 735   ip0->checksum = ip_csum_fold (sum0);
 736
 737   if (!icmp_is_error_message (icmp0))
 738     {
 739       new_id0 = sm0.port;
 740       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 741         {
 742           old_id0 = echo0->identifier;
 743           new_id0 = sm0.port;
 744           echo0->identifier = new_id0;
 745
 746           sum0 = icmp0->checksum;
 747           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 748                                  identifier);
 749           icmp0->checksum = ip_csum_fold (sum0);
 750         }
 751     }
 752   else
 753     {
 754       inner_ip0 = (ip4_header_t *)(echo0+1);
 755       l4_header = ip4_next_header (inner_ip0);
 756
 757       if (!ip4_header_checksum_is_valid (inner_ip0))
 758         {
 759           next0 = SNAT_IN2OUT_NEXT_DROP;
 760           goto out;
 761         }
 762
 763       old_addr0 = inner_ip0->dst_address.as_u32;
 764       inner_ip0->dst_address = sm0.addr;
 765       new_addr0 = inner_ip0->dst_address.as_u32;
 766
 767       sum0 = icmp0->checksum;
 768       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 769                              dst_address /* changed member */);
 770       icmp0->checksum = ip_csum_fold (sum0);
 771
 772       switch (protocol)
 773         {
 774           case SNAT_PROTOCOL_ICMP:
 775             inner_icmp0 = (icmp46_header_t*)l4_header;
 776             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 777
 778             old_id0 = inner_echo0->identifier;
 779             new_id0 = sm0.port;
 780             inner_echo0->identifier = new_id0;
 781
 782             sum0 = icmp0->checksum;
 783             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 784                                    identifier);
 785             icmp0->checksum = ip_csum_fold (sum0);
 786             break;
 787           case SNAT_PROTOCOL_UDP:
 788           case SNAT_PROTOCOL_TCP:
 789             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 790             new_id0 = sm0.port;
 791             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 792
 793             sum0 = icmp0->checksum;
 794             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 795                                    dst_port);
 796             icmp0->checksum = ip_csum_fold (sum0);
 797             break;
 798           default:
 799             ASSERT(0);
 800         }
 801     }
 802
 803 out:
 804   return next0;
 805 }
 806
 807 /**
 808  * @brief Hairpinning
 809  *
 810  * Hairpinning allows two endpoints on the internal side of the NAT to
 811  * communicate even if they only use each other's external IP addresses
 812  * and ports.
 813  *
 814  * @param sm     NAT main.
 815  * @param b0     Vlib buffer.
 816  * @param ip0    IP header.
 817  * @param udp0   UDP header.
 818  * @param tcp0   TCP header.
 819  * @param proto0 NAT protocol.
 820  */
 821 static inline void
 822 snat_hairpinning (snat_main_t *sm,
 823                   vlib_buffer_t * b0,
 824                   ip4_header_t * ip0,
 825                   udp_header_t * udp0,
 826                   tcp_header_t * tcp0,
 827                   u32 proto0)
 828 {
 829   snat_session_key_t key0, sm0;
 830   snat_session_t * s0;
 831   clib_bihash_kv_8_8_t kv0, value0;
 832   ip_csum_t sum0;
 833   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 834   u16 new_dst_port0, old_dst_port0;
 835
 836   key0.addr = ip0->dst_address;
 837   key0.port = udp0->dst_port;
 838   key0.protocol = proto0;
 839   key0.fib_index = sm->outside_fib_index;
 840   kv0.key = key0.as_u64;
 841
 842   /* Check if destination is static mappings */
 843   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 844     {
 845       new_dst_addr0 = sm0.addr.as_u32;
 846       new_dst_port0 = sm0.port;
 847       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 848     }
 849   /* or active session */
 850   else
 851     {
 852       if (sm->num_workers > 1)
 853         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 854       else
 855         ti = sm->num_workers;
 856
 857       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 858         {
 859           si = value0.value;
 860
 861           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 862           new_dst_addr0 = s0->in2out.addr.as_u32;
 863           new_dst_port0 = s0->in2out.port;
 864           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 865         }
 866     }
 867
 868   /* Destination is behind the same NAT, use internal address and port */
 869   if (new_dst_addr0)
 870     {
 871       old_dst_addr0 = ip0->dst_address.as_u32;
 872       ip0->dst_address.as_u32 = new_dst_addr0;
 873       sum0 = ip0->checksum;
 874       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 875                              ip4_header_t, dst_address);
 876       ip0->checksum = ip_csum_fold (sum0);
 877
 878       old_dst_port0 = tcp0->dst;
 879       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 880         {
 881           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 882             {
 883               tcp0->dst = new_dst_port0;
 884               sum0 = tcp0->checksum;
 885               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 886                                      ip4_header_t, dst_address);
 887               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 888                                      ip4_header_t /* cheat */, length);
 889               tcp0->checksum = ip_csum_fold(sum0);
 890             }
 891           else
 892             {
 893               udp0->dst_port = new_dst_port0;
 894               udp0->checksum = 0;
 895             }
 896         }
 897       else
 898         {
 899           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 900             {
 901               sum0 = tcp0->checksum;
 902               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 903                                      ip4_header_t, dst_address);
 904               tcp0->checksum = ip_csum_fold(sum0);
 905             }
 906         }
 907     }
 908 }
 909
 910 static inline void
 911 snat_icmp_hairpinning (snat_main_t *sm,
 912                        vlib_buffer_t * b0,
 913                        ip4_header_t * ip0,
 914                        icmp46_header_t * icmp0)
 915 {
 916   snat_session_key_t key0, sm0;
 917   clib_bihash_kv_8_8_t kv0, value0;
 918   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 919   ip_csum_t sum0;
 920   snat_session_t *s0;
 921
 922   if (!icmp_is_error_message (icmp0))
 923     {
 924       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 925       u16 icmp_id0 = echo0->identifier;
 926       key0.addr = ip0->dst_address;
 927       key0.port = icmp_id0;
 928       key0.protocol = SNAT_PROTOCOL_ICMP;
 929       key0.fib_index = sm->outside_fib_index;
 930       kv0.key = key0.as_u64;
 931
 932       if (sm->num_workers > 1)
 933         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 934       else
 935         ti = sm->num_workers;
 936
 937       /* Check if destination is in active sessions */
 938       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 939                                   &value0))
 940         {
 941           /* or static mappings */
 942           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 943             {
 944               new_dst_addr0 = sm0.addr.as_u32;
 945               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 946             }
 947         }
 948       else
 949         {
 950           si = value0.value;
 951
 952           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 953           new_dst_addr0 = s0->in2out.addr.as_u32;
 954           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 955           echo0->identifier = s0->in2out.port;
 956           sum0 = icmp0->checksum;
 957           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 958                                  icmp_echo_header_t, identifier);
 959           icmp0->checksum = ip_csum_fold (sum0);
 960         }
 961
 962       /* Destination is behind the same NAT, use internal address and port */
 963       if (new_dst_addr0)
 964         {
 965           old_dst_addr0 = ip0->dst_address.as_u32;
 966           ip0->dst_address.as_u32 = new_dst_addr0;
 967           sum0 = ip0->checksum;
 968           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 969                                  ip4_header_t, dst_address);
 970           ip0->checksum = ip_csum_fold (sum0);
 971         }
 972     }
 973
 974 }
 975
 976 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 977                                          vlib_buffer_t * b0,
 978                                          ip4_header_t * ip0,
 979                                          icmp46_header_t * icmp0,
 980                                          u32 sw_if_index0,
 981                                          u32 rx_fib_index0,
 982                                          vlib_node_runtime_t * node,
 983                                          u32 next0,
 984                                          f64 now,
 985                                          u32 thread_index,
 986                                          snat_session_t ** p_s0)
 987 {
 988   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 989                       next0, thread_index, p_s0, 0);
 990   snat_session_t * s0 = *p_s0;
 991   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 992     {
 993       /* Hairpinning */
 994       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
 995         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
 996       /* Accounting */
 997       s0->last_heard = now;
 998       s0->total_pkts++;
 999       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1000       /* Per-user LRU list maintenance for dynamic translations */
1001       if (!snat_is_session_static (s0))
1002         {
1003           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1004                              s0->per_user_index);
1005           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1006                               s0->per_user_list_head_index,
1007                               s0->per_user_index);
1008         }
1009     }
1010   return next0;
1011 }
1012 static inline void
1013 snat_hairpinning_unknown_proto (snat_main_t *sm,
1014                                 vlib_buffer_t * b,
1015                                 ip4_header_t * ip)
1016 {
1017   u32 old_addr, new_addr = 0, ti = 0;
1018   clib_bihash_kv_8_8_t kv, value;
1019   clib_bihash_kv_16_8_t s_kv, s_value;
1020   nat_ed_ses_key_t key;
1021   snat_session_key_t m_key;
1022   snat_static_mapping_t *m;
1023   ip_csum_t sum;
1024   snat_session_t *s;
1025
1026   old_addr = ip->dst_address.as_u32;
1027   key.l_addr.as_u32 = ip->dst_address.as_u32;
1028   key.r_addr.as_u32 = ip->src_address.as_u32;
1029   key.fib_index = sm->outside_fib_index;
1030   key.proto = ip->protocol;
1031   key.rsvd = 0;
1032   key.l_port = 0;
1033   s_kv.key[0] = key.as_u64[0];
1034   s_kv.key[1] = key.as_u64[1];
1035   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1036     {
1037       m_key.addr = ip->dst_address;
1038       m_key.fib_index = sm->outside_fib_index;
1039       m_key.port = 0;
1040       m_key.protocol = 0;
1041       kv.key = m_key.as_u64;
1042       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1043         return;
1044
1045       m = pool_elt_at_index (sm->static_mappings, value.value);
1046       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1047         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1048       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1049     }
1050   else
1051     {
1052       if (sm->num_workers > 1)
1053         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1054       else
1055         ti = sm->num_workers;
1056
1057       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1058       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1059         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1060       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1061     }
1062   sum = ip->checksum;
1063   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1064   ip->checksum = ip_csum_fold (sum);
1065 }
1066
1067 static void
1068 snat_in2out_unknown_proto (snat_main_t *sm,
1069                            vlib_buffer_t * b,
1070                            ip4_header_t * ip,
1071                            u32 rx_fib_index,
1072                            u32 thread_index,
1073                            f64 now,
1074                            vlib_main_t * vm)
1075 {
1076   clib_bihash_kv_8_8_t kv, value;
1077   clib_bihash_kv_16_8_t s_kv, s_value;
1078   snat_static_mapping_t *m;
1079   snat_session_key_t m_key;
1080   u32 old_addr, new_addr = 0;
1081   ip_csum_t sum;
1082   snat_user_key_t u_key;
1083   snat_user_t *u;
1084   dlist_elt_t *head, *elt, *oldest;
1085   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1086   u32 elt_index, head_index, ses_index, oldest_index;
1087   snat_session_t * s;
1088   nat_ed_ses_key_t key;
1089   u32 address_index = ~0;
1090   int i;
1091   u8 is_sm = 0;
1092
1093   old_addr = ip->src_address.as_u32;
1094
1095   key.l_addr = ip->src_address;
1096   key.r_addr = ip->dst_address;
1097   key.fib_index = rx_fib_index;
1098   key.proto = ip->protocol;
1099   key.rsvd = 0;
1100   key.l_port = 0;
1101   s_kv.key[0] = key.as_u64[0];
1102   s_kv.key[1] = key.as_u64[1];
1103
1104   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1105     {
1106       s = pool_elt_at_index (tsm->sessions, s_value.value);
1107       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1108     }
1109   else
1110     {
1111       u_key.addr = ip->src_address;
1112       u_key.fib_index = rx_fib_index;
1113       kv.key = u_key.as_u64;
1114
1115       /* Ever heard of the "user" = src ip4 address before? */
1116       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1117         {
1118           /* no, make a new one */
1119           pool_get (tsm->users, u);
1120           memset (u, 0, sizeof (*u));
1121           u->addr = ip->src_address;
1122           u->fib_index = rx_fib_index;
1123
1124           pool_get (tsm->list_pool, head);
1125           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1126
1127           clib_dlist_init (tsm->list_pool,
1128                            u->sessions_per_user_list_head_index);
1129
1130           kv.value = u - tsm->users;
1131
1132           /* add user */
1133           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
1134         }
1135       else
1136         {
1137           u = pool_elt_at_index (tsm->users, value.value);
1138         }
1139
1140       m_key.addr = ip->src_address;
1141       m_key.port = 0;
1142       m_key.protocol = 0;
1143       m_key.fib_index = rx_fib_index;
1144       kv.key = m_key.as_u64;
1145
1146       /* Try to find static mapping first */
1147       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1148         {
1149           m = pool_elt_at_index (sm->static_mappings, value.value);
1150           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1151           is_sm = 1;
1152           goto create_ses;
1153         }
1154       /* Fallback to 3-tuple key */
1155       else
1156         {
1157           /* Choose same out address as for TCP/UDP session to same destination */
1158           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1159             {
1160               head_index = u->sessions_per_user_list_head_index;
1161               head = pool_elt_at_index (tsm->list_pool, head_index);
1162               elt_index = head->next;
1163               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1164               ses_index = elt->value;
1165               while (ses_index != ~0)
1166                 {
1167                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1168                   elt_index = elt->next;
1169                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1170                   ses_index = elt->value;
1171
1172                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1173                     {
1174                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1175                       address_index = s->outside_address_index;
1176
1177                       key.fib_index = sm->outside_fib_index;
1178                       key.l_addr.as_u32 = new_addr;
1179                       s_kv.key[0] = key.as_u64[0];
1180                       s_kv.key[1] = key.as_u64[1];
1181                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1182                         break;
1183
1184                       goto create_ses;
1185                     }
1186                 }
1187             }
1188           key.fib_index = sm->outside_fib_index;
1189           for (i = 0; i < vec_len (sm->addresses); i++)
1190             {
1191               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1192               s_kv.key[0] = key.as_u64[0];
1193               s_kv.key[1] = key.as_u64[1];
1194               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1195                 {
1196                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1197                   address_index = i;
1198                   goto create_ses;
1199                 }
1200             }
1201           return;
1202         }
1203
1204 create_ses:
1205       /* Over quota? Recycle the least recently used dynamic translation */
1206       if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1207         {
1208           /* Remove the oldest dynamic translation */
1209           do {
1210               oldest_index = clib_dlist_remove_head (
1211                 tsm->list_pool, u->sessions_per_user_list_head_index);
1212
1213               ASSERT (oldest_index != ~0);
1214
1215               /* add it back to the end of the LRU list */
1216               clib_dlist_addtail (tsm->list_pool,
1217                                   u->sessions_per_user_list_head_index,
1218                                   oldest_index);
1219               /* Get the list element */
1220               oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1221
1222               /* Get the session index from the list element */
1223               ses_index = oldest->value;
1224
1225               /* Get the session */
1226               s = pool_elt_at_index (tsm->sessions, ses_index);
1227           } while (snat_is_session_static (s));
1228
1229           if (snat_is_unk_proto_session (s))
1230             {
1231               /* Remove from lookup tables */
1232               key.l_addr = s->in2out.addr;
1233               key.r_addr = s->ext_host_addr;
1234               key.fib_index = s->in2out.fib_index;
1235               key.proto = s->in2out.port;
1236               s_kv.key[0] = key.as_u64[0];
1237               s_kv.key[1] = key.as_u64[1];
1238               if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 0))
1239                 clib_warning ("in2out key del failed");
1240
1241               key.l_addr = s->out2in.addr;
1242               key.fib_index = s->out2in.fib_index;
1243               s_kv.key[0] = key.as_u64[0];
1244               s_kv.key[1] = key.as_u64[1];
1245               if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 0))
1246                 clib_warning ("out2in key del failed");
1247             }
1248           else
1249             {
1250               /* log NAT event */
1251               snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1252                                                   s->out2in.addr.as_u32,
1253                                                   s->in2out.protocol,
1254                                                   s->in2out.port,
1255                                                   s->out2in.port,
1256                                                   s->in2out.fib_index);
1257
1258               snat_free_outside_address_and_port (sm, thread_index, &s->out2in,
1259                                                   s->outside_address_index);
1260
1261               /* Remove in2out, out2in keys */
1262               kv.key = s->in2out.as_u64;
1263               if (clib_bihash_add_del_8_8 (
1264                     &sm->per_thread_data[thread_index].in2out, &kv, 0))
1265                 clib_warning ("in2out key del failed");
1266               kv.key = s->out2in.as_u64;
1267               if (clib_bihash_add_del_8_8 (
1268                     &sm->per_thread_data[thread_index].out2in, &kv, 0))
1269                 clib_warning ("out2in key del failed");
1270             }
1271         }
1272       else
1273         {
1274           /* Create a new session */
1275           pool_get (tsm->sessions, s);
1276           memset (s, 0, sizeof (*s));
1277
1278           /* Create list elts */
1279           pool_get (tsm->list_pool, elt);
1280           clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1281           elt->value = s - tsm->sessions;
1282           s->per_user_index = elt - tsm->list_pool;
1283           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1284           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1285                               s->per_user_index);
1286         }
1287
1288       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1289       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1290       s->outside_address_index = address_index;
1291       s->out2in.addr.as_u32 = new_addr;
1292       s->out2in.fib_index = sm->outside_fib_index;
1293       s->in2out.addr.as_u32 = old_addr;
1294       s->in2out.fib_index = rx_fib_index;
1295       s->in2out.port = s->out2in.port = ip->protocol;
1296       if (is_sm)
1297         {
1298           u->nstaticsessions++;
1299           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1300         }
1301       else
1302         {
1303           u->nsessions++;
1304         }
1305
1306       /* Add to lookup tables */
1307       key.l_addr.as_u32 = old_addr;
1308       key.r_addr = ip->dst_address;
1309       key.proto = ip->protocol;
1310       key.fib_index = rx_fib_index;
1311       s_kv.key[0] = key.as_u64[0];
1312       s_kv.key[1] = key.as_u64[1];
1313       s_kv.value = s - tsm->sessions;
1314       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1315         clib_warning ("in2out key add failed");
1316
1317       key.l_addr.as_u32 = new_addr;
1318       key.fib_index = sm->outside_fib_index;
1319       s_kv.key[0] = key.as_u64[0];
1320       s_kv.key[1] = key.as_u64[1];
1321       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1322         clib_warning ("out2in key add failed");
1323   }
1324
1325   /* Update IP checksum */
1326   sum = ip->checksum;
1327   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1328   ip->checksum = ip_csum_fold (sum);
1329
1330   /* Accounting */
1331   s->last_heard = now;
1332   s->total_pkts++;
1333   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1334   /* Per-user LRU list maintenance */
1335   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1336   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1337                       s->per_user_index);
1338
1339   /* Hairpinning */
1340   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1341     snat_hairpinning_unknown_proto(sm, b, ip);
1342
1343   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1344     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1345 }
1346
1347 static snat_session_t *
1348 snat_in2out_lb (snat_main_t *sm,
1349                 vlib_buffer_t * b,
1350                 ip4_header_t * ip,
1351                 u32 rx_fib_index,
1352                 u32 thread_index,
1353                 f64 now,
1354                 vlib_main_t * vm)
1355 {
1356   nat_ed_ses_key_t key;
1357   clib_bihash_kv_16_8_t s_kv, s_value;
1358   udp_header_t *udp = ip4_next_header (ip);
1359   tcp_header_t *tcp = (tcp_header_t *) udp;
1360   snat_session_t *s = 0;
1361   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1362   u32 old_addr, new_addr;
1363   u16 new_port, old_port;
1364   ip_csum_t sum;
1365   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1366   snat_session_key_t e_key, l_key;
1367   clib_bihash_kv_8_8_t kv, value;
1368   snat_user_key_t u_key;
1369   snat_user_t *u;
1370   dlist_elt_t *head, *elt;
1371
1372   old_addr = ip->src_address.as_u32;
1373
1374   key.l_addr = ip->src_address;
1375   key.r_addr = ip->dst_address;
1376   key.fib_index = rx_fib_index;
1377   key.proto = ip->protocol;
1378   key.rsvd = 0;
1379   key.l_port = udp->src_port;
1380   s_kv.key[0] = key.as_u64[0];
1381   s_kv.key[1] = key.as_u64[1];
1382
1383   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1384     {
1385       s = pool_elt_at_index (tsm->sessions, s_value.value);
1386     }
1387   else
1388     {
1389       l_key.addr = ip->src_address;
1390       l_key.port = udp->src_port;
1391       l_key.protocol = proto;
1392       l_key.fib_index = rx_fib_index;
1393       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0))
1394         return 0;
1395
1396       u_key.addr = ip->src_address;
1397       u_key.fib_index = rx_fib_index;
1398       kv.key = u_key.as_u64;
1399
1400       /* Ever heard of the "user" = src ip4 address before? */
1401       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1402         {
1403           /* no, make a new one */
1404           pool_get (tsm->users, u);
1405           memset (u, 0, sizeof (*u));
1406           u->addr = ip->src_address;
1407           u->fib_index = rx_fib_index;
1408
1409           pool_get (tsm->list_pool, head);
1410           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1411
1412           clib_dlist_init (tsm->list_pool,
1413                            u->sessions_per_user_list_head_index);
1414
1415           kv.value = u - tsm->users;
1416
1417           /* add user */
1418           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
1419             clib_warning ("user key add failed");
1420         }
1421       else
1422         {
1423           u = pool_elt_at_index (tsm->users, value.value);
1424         }
1425
1426       /* Create a new session */
1427       pool_get (tsm->sessions, s);
1428       memset (s, 0, sizeof (*s));
1429
1430       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1431       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1432       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1433       s->outside_address_index = ~0;
1434       s->in2out = l_key;
1435       s->out2in = e_key;
1436       u->nstaticsessions++;
1437
1438       /* Create list elts */
1439       pool_get (tsm->list_pool, elt);
1440       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1441       elt->value = s - tsm->sessions;
1442       s->per_user_index = elt - tsm->list_pool;
1443       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1444       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1445                           s->per_user_index);
1446
1447       /* Add to lookup tables */
1448       s_kv.value = s - tsm->sessions;
1449       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1450         clib_warning ("in2out-ed key add failed");
1451
1452       key.l_addr = e_key.addr;
1453       key.fib_index = e_key.fib_index;
1454       key.l_port = e_key.port;
1455       s_kv.key[0] = key.as_u64[0];
1456       s_kv.key[1] = key.as_u64[1];
1457       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1458         clib_warning ("out2in-ed key add failed");
1459     }
1460
1461   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1462
1463   /* Update IP checksum */
1464   sum = ip->checksum;
1465   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1466   ip->checksum = ip_csum_fold (sum);
1467
1468   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1469     {
1470       old_port = tcp->src_port;
1471       tcp->src_port = s->out2in.port;
1472       new_port = tcp->src_port;
1473
1474       sum = tcp->checksum;
1475       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1476       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1477       tcp->checksum = ip_csum_fold(sum);
1478     }
1479   else
1480     {
1481       udp->src_port = s->out2in.port;
1482       udp->checksum = 0;
1483     }
1484
1485   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1486     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1487
1488   /* Accounting */
1489   s->last_heard = now;
1490   s->total_pkts++;
1491   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1492   return s;
1493 }
1494
1495 static inline uword
1496 snat_in2out_node_fn_inline (vlib_main_t * vm,
1497                             vlib_node_runtime_t * node,
1498                             vlib_frame_t * frame, int is_slow_path,
1499                             int is_output_feature)
1500 {
1501   u32 n_left_from, * from, * to_next;
1502   snat_in2out_next_t next_index;
1503   u32 pkts_processed = 0;
1504   snat_main_t * sm = &snat_main;
1505   f64 now = vlib_time_now (vm);
1506   u32 stats_node_index;
1507   u32 thread_index = vlib_get_thread_index ();
1508
1509   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1510     snat_in2out_node.index;
1511
1512   from = vlib_frame_vector_args (frame);
1513   n_left_from = frame->n_vectors;
1514   next_index = node->cached_next_index;
1515
1516   while (n_left_from > 0)
1517     {
1518       u32 n_left_to_next;
1519
1520       vlib_get_next_frame (vm, node, next_index,
1521                            to_next, n_left_to_next);
1522
1523       while (n_left_from >= 4 && n_left_to_next >= 2)
1524         {
1525           u32 bi0, bi1;
1526           vlib_buffer_t * b0, * b1;
1527           u32 next0, next1;
1528           u32 sw_if_index0, sw_if_index1;
1529           ip4_header_t * ip0, * ip1;
1530           ip_csum_t sum0, sum1;
1531           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1532           u16 old_port0, new_port0, old_port1, new_port1;
1533           udp_header_t * udp0, * udp1;
1534           tcp_header_t * tcp0, * tcp1;
1535           icmp46_header_t * icmp0, * icmp1;
1536           snat_session_key_t key0, key1;
1537           u32 rx_fib_index0, rx_fib_index1;
1538           u32 proto0, proto1;
1539           snat_session_t * s0 = 0, * s1 = 0;
1540           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1541           u32 iph_offset0 = 0, iph_offset1 = 0;
1542
1543           /* Prefetch next iteration. */
1544           {
1545             vlib_buffer_t * p2, * p3;
1546
1547             p2 = vlib_get_buffer (vm, from[2]);
1548             p3 = vlib_get_buffer (vm, from[3]);
1549
1550             vlib_prefetch_buffer_header (p2, LOAD);
1551             vlib_prefetch_buffer_header (p3, LOAD);
1552
1553             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1554             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1555           }
1556
1557           /* speculatively enqueue b0 and b1 to the current next frame */
1558           to_next[0] = bi0 = from[0];
1559           to_next[1] = bi1 = from[1];
1560           from += 2;
1561           to_next += 2;
1562           n_left_from -= 2;
1563           n_left_to_next -= 2;
1564
1565           b0 = vlib_get_buffer (vm, bi0);
1566           b1 = vlib_get_buffer (vm, bi1);
1567
1568           if (is_output_feature)
1569             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1570
1571           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1572                  iph_offset0);
1573
1574           udp0 = ip4_next_header (ip0);
1575           tcp0 = (tcp_header_t *) udp0;
1576           icmp0 = (icmp46_header_t *) udp0;
1577
1578           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1579           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1580                                    sw_if_index0);
1581
1582           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1583
1584           if (PREDICT_FALSE(ip0->ttl == 1))
1585             {
1586               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1587               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1588                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1589                                            0);
1590               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1591               goto trace00;
1592             }
1593
1594           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1595
1596           /* Next configured feature, probably ip4-lookup */
1597           if (is_slow_path)
1598             {
1599               if (PREDICT_FALSE (proto0 == ~0))
1600                 {
1601                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1602                                              thread_index, now, vm);
1603                   goto trace00;
1604                 }
1605
1606               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1607                 {
1608                   next0 = icmp_in2out_slow_path
1609                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1610                      node, next0, now, thread_index, &s0);
1611                   goto trace00;
1612                 }
1613             }
1614           else
1615             {
1616               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1617                 {
1618                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1619                   goto trace00;
1620                 }
1621             }
1622
1623           key0.addr = ip0->src_address;
1624           key0.port = udp0->src_port;
1625           key0.protocol = proto0;
1626           key0.fib_index = rx_fib_index0;
1627
1628           kv0.key = key0.as_u64;
1629
1630           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1631               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1632             {
1633               if (is_slow_path)
1634                 {
1635                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1636                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1637                     goto trace00;
1638
1639                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1640                                      &s0, node, next0, thread_index);
1641                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1642                     goto trace00;
1643                 }
1644               else
1645                 {
1646                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1647                   goto trace00;
1648                 }
1649             }
1650           else
1651             {
1652               if (PREDICT_FALSE (value0.value == ~0ULL))
1653                 {
1654                   if (is_slow_path)
1655                     {
1656                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1657                                      now, vm);
1658                       goto trace00;
1659                     }
1660                   else
1661                     {
1662                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1663                       goto trace00;
1664                     }
1665                 }
1666               else
1667                 {
1668                   s0 = pool_elt_at_index (
1669                     sm->per_thread_data[thread_index].sessions,
1670                     value0.value);
1671                 }
1672             }
1673
1674           old_addr0 = ip0->src_address.as_u32;
1675           ip0->src_address = s0->out2in.addr;
1676           new_addr0 = ip0->src_address.as_u32;
1677           if (!is_output_feature)
1678             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1679
1680           sum0 = ip0->checksum;
1681           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1682                                  ip4_header_t,
1683                                  src_address /* changed member */);
1684           ip0->checksum = ip_csum_fold (sum0);
1685
1686           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1687             {
1688               old_port0 = tcp0->src_port;
1689               tcp0->src_port = s0->out2in.port;
1690               new_port0 = tcp0->src_port;
1691
1692               sum0 = tcp0->checksum;
1693               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1694                                      ip4_header_t,
1695                                      dst_address /* changed member */);
1696               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1697                                      ip4_header_t /* cheat */,
1698                                      length /* changed member */);
1699               tcp0->checksum = ip_csum_fold(sum0);
1700             }
1701           else
1702             {
1703               old_port0 = udp0->src_port;
1704               udp0->src_port = s0->out2in.port;
1705               udp0->checksum = 0;
1706             }
1707
1708           /* Hairpinning */
1709           if (!is_output_feature)
1710             snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1711
1712           /* Accounting */
1713           s0->last_heard = now;
1714           s0->total_pkts++;
1715           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1716           /* Per-user LRU list maintenance for dynamic translation */
1717           if (!snat_is_session_static (s0))
1718             {
1719               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1720                                  s0->per_user_index);
1721               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1722                                   s0->per_user_list_head_index,
1723                                   s0->per_user_index);
1724             }
1725         trace00:
1726
1727           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1728                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1729             {
1730               snat_in2out_trace_t *t =
1731                  vlib_add_trace (vm, node, b0, sizeof (*t));
1732               t->is_slow_path = is_slow_path;
1733               t->sw_if_index = sw_if_index0;
1734               t->next_index = next0;
1735                   t->session_index = ~0;
1736               if (s0)
1737                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1738             }
1739
1740           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1741
1742           if (is_output_feature)
1743             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1744
1745           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1746                  iph_offset1);
1747
1748           udp1 = ip4_next_header (ip1);
1749           tcp1 = (tcp_header_t *) udp1;
1750           icmp1 = (icmp46_header_t *) udp1;
1751
1752           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1753           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1754                                    sw_if_index1);
1755
1756           if (PREDICT_FALSE(ip1->ttl == 1))
1757             {
1758               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1759               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1760                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1761                                            0);
1762               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1763               goto trace01;
1764             }
1765
1766           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1767
1768           /* Next configured feature, probably ip4-lookup */
1769           if (is_slow_path)
1770             {
1771               if (PREDICT_FALSE (proto1 == ~0))
1772                 {
1773                   snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1774                                              thread_index, now, vm);
1775                   goto trace01;
1776                 }
1777
1778               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1779                 {
1780                   next1 = icmp_in2out_slow_path
1781                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1782                      next1, now, thread_index, &s1);
1783                   goto trace01;
1784                 }
1785             }
1786           else
1787             {
1788               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1789                 {
1790                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1791                   goto trace01;
1792                 }
1793             }
1794
1795           key1.addr = ip1->src_address;
1796           key1.port = udp1->src_port;
1797           key1.protocol = proto1;
1798           key1.fib_index = rx_fib_index1;
1799
1800           kv1.key = key1.as_u64;
1801
1802             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1803                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1804             {
1805               if (is_slow_path)
1806                 {
1807                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1808                       ip1, proto1, rx_fib_index1, thread_index)) && !is_output_feature)
1809                     goto trace01;
1810
1811                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1812                                      &s1, node, next1, thread_index);
1813                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1814                     goto trace01;
1815                 }
1816               else
1817                 {
1818                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1819                   goto trace01;
1820                 }
1821             }
1822           else
1823             {
1824               if (PREDICT_FALSE (value1.value == ~0ULL))
1825                 {
1826                   if (is_slow_path)
1827                     {
1828                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1829                                      now, vm);
1830                       goto trace01;
1831                     }
1832                   else
1833                     {
1834                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1835                       goto trace01;
1836                     }
1837                 }
1838               else
1839                 {
1840                   s1 = pool_elt_at_index (
1841                     sm->per_thread_data[thread_index].sessions,
1842                     value1.value);
1843                 }
1844             }
1845
1846           old_addr1 = ip1->src_address.as_u32;
1847           ip1->src_address = s1->out2in.addr;
1848           new_addr1 = ip1->src_address.as_u32;
1849           if (!is_output_feature)
1850             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1851
1852           sum1 = ip1->checksum;
1853           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1854                                  ip4_header_t,
1855                                  src_address /* changed member */);
1856           ip1->checksum = ip_csum_fold (sum1);
1857
1858           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1859             {
1860               old_port1 = tcp1->src_port;
1861               tcp1->src_port = s1->out2in.port;
1862               new_port1 = tcp1->src_port;
1863
1864               sum1 = tcp1->checksum;
1865               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1866                                      ip4_header_t,
1867                                      dst_address /* changed member */);
1868               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1869                                      ip4_header_t /* cheat */,
1870                                      length /* changed member */);
1871               tcp1->checksum = ip_csum_fold(sum1);
1872             }
1873           else
1874             {
1875               old_port1 = udp1->src_port;
1876               udp1->src_port = s1->out2in.port;
1877               udp1->checksum = 0;
1878             }
1879
1880           /* Hairpinning */
1881           if (!is_output_feature)
1882             snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1883
1884           /* Accounting */
1885           s1->last_heard = now;
1886           s1->total_pkts++;
1887           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1888           /* Per-user LRU list maintenance for dynamic translation */
1889           if (!snat_is_session_static (s1))
1890             {
1891               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1892                                  s1->per_user_index);
1893               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1894                                   s1->per_user_list_head_index,
1895                                   s1->per_user_index);
1896             }
1897         trace01:
1898
1899           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1900                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1901             {
1902               snat_in2out_trace_t *t =
1903                  vlib_add_trace (vm, node, b1, sizeof (*t));
1904               t->sw_if_index = sw_if_index1;
1905               t->next_index = next1;
1906               t->session_index = ~0;
1907               if (s1)
1908                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1909             }
1910
1911           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1912
1913           /* verify speculative enqueues, maybe switch current next frame */
1914           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1915                                            to_next, n_left_to_next,
1916                                            bi0, bi1, next0, next1);
1917         }
1918
1919       while (n_left_from > 0 && n_left_to_next > 0)
1920         {
1921           u32 bi0;
1922           vlib_buffer_t * b0;
1923           u32 next0;
1924           u32 sw_if_index0;
1925           ip4_header_t * ip0;
1926           ip_csum_t sum0;
1927           u32 new_addr0, old_addr0;
1928           u16 old_port0, new_port0;
1929           udp_header_t * udp0;
1930           tcp_header_t * tcp0;
1931           icmp46_header_t * icmp0;
1932           snat_session_key_t key0;
1933           u32 rx_fib_index0;
1934           u32 proto0;
1935           snat_session_t * s0 = 0;
1936           clib_bihash_kv_8_8_t kv0, value0;
1937           u32 iph_offset0 = 0;
1938
1939           /* speculatively enqueue b0 to the current next frame */
1940           bi0 = from[0];
1941           to_next[0] = bi0;
1942           from += 1;
1943           to_next += 1;
1944           n_left_from -= 1;
1945           n_left_to_next -= 1;
1946
1947           b0 = vlib_get_buffer (vm, bi0);
1948           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1949
1950           if (is_output_feature)
1951             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1952
1953           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1954                  iph_offset0);
1955
1956           udp0 = ip4_next_header (ip0);
1957           tcp0 = (tcp_header_t *) udp0;
1958           icmp0 = (icmp46_header_t *) udp0;
1959
1960           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1961           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1962                                    sw_if_index0);
1963
1964           if (PREDICT_FALSE(ip0->ttl == 1))
1965             {
1966               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1967               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1968                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1969                                            0);
1970               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1971               goto trace0;
1972             }
1973
1974           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1975
1976           /* Next configured feature, probably ip4-lookup */
1977           if (is_slow_path)
1978             {
1979               if (PREDICT_FALSE (proto0 == ~0))
1980                 {
1981                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1982                                              thread_index, now, vm);
1983                   goto trace0;
1984                 }
1985
1986               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1987                 {
1988                   next0 = icmp_in2out_slow_path
1989                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1990                      next0, now, thread_index, &s0);
1991                   goto trace0;
1992                 }
1993             }
1994           else
1995             {
1996               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1997                 {
1998                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1999                   goto trace0;
2000                 }
2001             }
2002
2003           key0.addr = ip0->src_address;
2004           key0.port = udp0->src_port;
2005           key0.protocol = proto0;
2006           key0.fib_index = rx_fib_index0;
2007
2008           kv0.key = key0.as_u64;
2009
2010           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2011                                       &kv0, &value0))
2012             {
2013               if (is_slow_path)
2014                 {
2015                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2016                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
2017                     goto trace0;
2018
2019                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2020                                      &s0, node, next0, thread_index);
2021
2022                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2023                     goto trace0;
2024                 }
2025               else
2026                 {
2027                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2028                   goto trace0;
2029                 }
2030             }
2031           else
2032             {
2033               if (PREDICT_FALSE (value0.value == ~0ULL))
2034                 {
2035                   if (is_slow_path)
2036                     {
2037                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0, thread_index,
2038                                      now, vm);
2039                       goto trace0;
2040                     }
2041                   else
2042                     {
2043                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2044                       goto trace0;
2045                     }
2046                 }
2047               else
2048                 {
2049                   s0 = pool_elt_at_index (
2050                     sm->per_thread_data[thread_index].sessions,
2051                     value0.value);
2052                 }
2053             }
2054
2055           old_addr0 = ip0->src_address.as_u32;
2056           ip0->src_address = s0->out2in.addr;
2057           new_addr0 = ip0->src_address.as_u32;
2058           if (!is_output_feature)
2059             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2060
2061           sum0 = ip0->checksum;
2062           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2063                                  ip4_header_t,
2064                                  src_address /* changed member */);
2065           ip0->checksum = ip_csum_fold (sum0);
2066
2067           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2068             {
2069               old_port0 = tcp0->src_port;
2070               tcp0->src_port = s0->out2in.port;
2071               new_port0 = tcp0->src_port;
2072
2073               sum0 = tcp0->checksum;
2074               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2075                                      ip4_header_t,
2076                                      dst_address /* changed member */);
2077               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2078                                      ip4_header_t /* cheat */,
2079                                      length /* changed member */);
2080               tcp0->checksum = ip_csum_fold(sum0);
2081             }
2082           else
2083             {
2084               old_port0 = udp0->src_port;
2085               udp0->src_port = s0->out2in.port;
2086               udp0->checksum = 0;
2087             }
2088
2089           /* Hairpinning */
2090           if (!is_output_feature)
2091             snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2092
2093           /* Accounting */
2094           s0->last_heard = now;
2095           s0->total_pkts++;
2096           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2097           /* Per-user LRU list maintenance for dynamic translation */
2098           if (!snat_is_session_static (s0))
2099             {
2100               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2101                                  s0->per_user_index);
2102               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2103                                   s0->per_user_list_head_index,
2104                                   s0->per_user_index);
2105             }
2106
2107         trace0:
2108           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2109                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2110             {
2111               snat_in2out_trace_t *t =
2112                  vlib_add_trace (vm, node, b0, sizeof (*t));
2113               t->is_slow_path = is_slow_path;
2114               t->sw_if_index = sw_if_index0;
2115               t->next_index = next0;
2116                   t->session_index = ~0;
2117               if (s0)
2118                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2119             }
2120
2121           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2122
2123           /* verify speculative enqueue, maybe switch current next frame */
2124           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2125                                            to_next, n_left_to_next,
2126                                            bi0, next0);
2127         }
2128
2129       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2130     }
2131
2132   vlib_node_increment_counter (vm, stats_node_index,
2133                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2134                                pkts_processed);
2135   return frame->n_vectors;
2136 }
2137
2138 static uword
2139 snat_in2out_fast_path_fn (vlib_main_t * vm,
2140                           vlib_node_runtime_t * node,
2141                           vlib_frame_t * frame)
2142 {
2143   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2144 }
2145
2146 VLIB_REGISTER_NODE (snat_in2out_node) = {
2147   .function = snat_in2out_fast_path_fn,
2148   .name = "nat44-in2out",
2149   .vector_size = sizeof (u32),
2150   .format_trace = format_snat_in2out_trace,
2151   .type = VLIB_NODE_TYPE_INTERNAL,
2152
2153   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2154   .error_strings = snat_in2out_error_strings,
2155
2156   .runtime_data_bytes = sizeof (snat_runtime_t),
2157
2158   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2159
2160   /* edit / add dispositions here */
2161   .next_nodes = {
2162     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2163     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2164     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2165     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2166   },
2167 };
2168
2169 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2170
2171 static uword
2172 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2173                                  vlib_node_runtime_t * node,
2174                                  vlib_frame_t * frame)
2175 {
2176   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2177 }
2178
2179 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2180   .function = snat_in2out_output_fast_path_fn,
2181   .name = "nat44-in2out-output",
2182   .vector_size = sizeof (u32),
2183   .format_trace = format_snat_in2out_trace,
2184   .type = VLIB_NODE_TYPE_INTERNAL,
2185
2186   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2187   .error_strings = snat_in2out_error_strings,
2188
2189   .runtime_data_bytes = sizeof (snat_runtime_t),
2190
2191   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2192
2193   /* edit / add dispositions here */
2194   .next_nodes = {
2195     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2196     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2197     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2198     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2199   },
2200 };
2201
2202 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2203                               snat_in2out_output_fast_path_fn);
2204
2205 static uword
2206 snat_in2out_slow_path_fn (vlib_main_t * vm,
2207                           vlib_node_runtime_t * node,
2208                           vlib_frame_t * frame)
2209 {
2210   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2211 }
2212
2213 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2214   .function = snat_in2out_slow_path_fn,
2215   .name = "nat44-in2out-slowpath",
2216   .vector_size = sizeof (u32),
2217   .format_trace = format_snat_in2out_trace,
2218   .type = VLIB_NODE_TYPE_INTERNAL,
2219
2220   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2221   .error_strings = snat_in2out_error_strings,
2222
2223   .runtime_data_bytes = sizeof (snat_runtime_t),
2224
2225   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2226
2227   /* edit / add dispositions here */
2228   .next_nodes = {
2229     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2230     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2231     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2232     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2233   },
2234 };
2235
2236 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2237                               snat_in2out_slow_path_fn);
2238
2239 static uword
2240 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2241                                  vlib_node_runtime_t * node,
2242                                  vlib_frame_t * frame)
2243 {
2244   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2245 }
2246
2247 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2248   .function = snat_in2out_output_slow_path_fn,
2249   .name = "nat44-in2out-output-slowpath",
2250   .vector_size = sizeof (u32),
2251   .format_trace = format_snat_in2out_trace,
2252   .type = VLIB_NODE_TYPE_INTERNAL,
2253
2254   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2255   .error_strings = snat_in2out_error_strings,
2256
2257   .runtime_data_bytes = sizeof (snat_runtime_t),
2258
2259   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2260
2261   /* edit / add dispositions here */
2262   .next_nodes = {
2263     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2264     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2265     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2266     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2267   },
2268 };
2269
2270 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2271                               snat_in2out_output_slow_path_fn);
2272
2273 /**************************/
2274 /*** deterministic mode ***/
2275 /**************************/
2276 static uword
2277 snat_det_in2out_node_fn (vlib_main_t * vm,
2278                          vlib_node_runtime_t * node,
2279                          vlib_frame_t * frame)
2280 {
2281   u32 n_left_from, * from, * to_next;
2282   snat_in2out_next_t next_index;
2283   u32 pkts_processed = 0;
2284   snat_main_t * sm = &snat_main;
2285   u32 now = (u32) vlib_time_now (vm);
2286   u32 thread_index = vlib_get_thread_index ();
2287
2288   from = vlib_frame_vector_args (frame);
2289   n_left_from = frame->n_vectors;
2290   next_index = node->cached_next_index;
2291
2292   while (n_left_from > 0)
2293     {
2294       u32 n_left_to_next;
2295
2296       vlib_get_next_frame (vm, node, next_index,
2297                            to_next, n_left_to_next);
2298
2299       while (n_left_from >= 4 && n_left_to_next >= 2)
2300         {
2301           u32 bi0, bi1;
2302           vlib_buffer_t * b0, * b1;
2303           u32 next0, next1;
2304           u32 sw_if_index0, sw_if_index1;
2305           ip4_header_t * ip0, * ip1;
2306           ip_csum_t sum0, sum1;
2307           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2308           u16 old_port0, new_port0, lo_port0, i0;
2309           u16 old_port1, new_port1, lo_port1, i1;
2310           udp_header_t * udp0, * udp1;
2311           tcp_header_t * tcp0, * tcp1;
2312           u32 proto0, proto1;
2313           snat_det_out_key_t key0, key1;
2314           snat_det_map_t * dm0, * dm1;
2315           snat_det_session_t * ses0 = 0, * ses1 = 0;
2316           u32 rx_fib_index0, rx_fib_index1;
2317           icmp46_header_t * icmp0, * icmp1;
2318
2319           /* Prefetch next iteration. */
2320           {
2321             vlib_buffer_t * p2, * p3;
2322
2323             p2 = vlib_get_buffer (vm, from[2]);
2324             p3 = vlib_get_buffer (vm, from[3]);
2325
2326             vlib_prefetch_buffer_header (p2, LOAD);
2327             vlib_prefetch_buffer_header (p3, LOAD);
2328
2329             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2330             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2331           }
2332
2333           /* speculatively enqueue b0 and b1 to the current next frame */
2334           to_next[0] = bi0 = from[0];
2335           to_next[1] = bi1 = from[1];
2336           from += 2;
2337           to_next += 2;
2338           n_left_from -= 2;
2339           n_left_to_next -= 2;
2340
2341           b0 = vlib_get_buffer (vm, bi0);
2342           b1 = vlib_get_buffer (vm, bi1);
2343
2344           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2345           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2346
2347           ip0 = vlib_buffer_get_current (b0);
2348           udp0 = ip4_next_header (ip0);
2349           tcp0 = (tcp_header_t *) udp0;
2350
2351           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2352
2353           if (PREDICT_FALSE(ip0->ttl == 1))
2354             {
2355               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2356               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2357                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2358                                            0);
2359               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2360               goto trace0;
2361             }
2362
2363           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2364
2365           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2366             {
2367               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2368               icmp0 = (icmp46_header_t *) udp0;
2369
2370               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2371                                   rx_fib_index0, node, next0, thread_index,
2372                                   &ses0, &dm0);
2373               goto trace0;
2374             }
2375
2376           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2377           if (PREDICT_FALSE(!dm0))
2378             {
2379               clib_warning("no match for internal host %U",
2380                            format_ip4_address, &ip0->src_address);
2381               next0 = SNAT_IN2OUT_NEXT_DROP;
2382               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2383               goto trace0;
2384             }
2385
2386           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2387
2388           key0.ext_host_addr = ip0->dst_address;
2389           key0.ext_host_port = tcp0->dst;
2390
2391           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2392           if (PREDICT_FALSE(!ses0))
2393             {
2394               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2395                 {
2396                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2397                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2398
2399                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2400                     continue;
2401
2402                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2403                   break;
2404                 }
2405               if (PREDICT_FALSE(!ses0))
2406                 {
2407                   /* too many sessions for user, send ICMP error packet */
2408
2409                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2410                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2411                                                ICMP4_destination_unreachable_destination_unreachable_host,
2412                                                0);
2413                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2414                   goto trace0;
2415                 }
2416             }
2417
2418           new_port0 = ses0->out.out_port;
2419
2420           old_addr0.as_u32 = ip0->src_address.as_u32;
2421           ip0->src_address.as_u32 = new_addr0.as_u32;
2422           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2423
2424           sum0 = ip0->checksum;
2425           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2426                                  ip4_header_t,
2427                                  src_address /* changed member */);
2428           ip0->checksum = ip_csum_fold (sum0);
2429
2430           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2431             {
2432               if (tcp0->flags & TCP_FLAG_SYN)
2433                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2434               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2435                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2436               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2437                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2438               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2439                 snat_det_ses_close(dm0, ses0);
2440               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2441                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2442               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2443                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2444
2445               old_port0 = tcp0->src;
2446               tcp0->src = new_port0;
2447
2448               sum0 = tcp0->checksum;
2449               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2450                                      ip4_header_t,
2451                                      dst_address /* changed member */);
2452               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2453                                      ip4_header_t /* cheat */,
2454                                      length /* changed member */);
2455               tcp0->checksum = ip_csum_fold(sum0);
2456             }
2457           else
2458             {
2459               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2460               old_port0 = udp0->src_port;
2461               udp0->src_port = new_port0;
2462               udp0->checksum = 0;
2463             }
2464
2465           switch(ses0->state)
2466             {
2467             case SNAT_SESSION_UDP_ACTIVE:
2468                 ses0->expire = now + sm->udp_timeout;
2469                 break;
2470             case SNAT_SESSION_TCP_SYN_SENT:
2471             case SNAT_SESSION_TCP_FIN_WAIT:
2472             case SNAT_SESSION_TCP_CLOSE_WAIT:
2473             case SNAT_SESSION_TCP_LAST_ACK:
2474                 ses0->expire = now + sm->tcp_transitory_timeout;
2475                 break;
2476             case SNAT_SESSION_TCP_ESTABLISHED:
2477                 ses0->expire = now + sm->tcp_established_timeout;
2478                 break;
2479             }
2480
2481         trace0:
2482           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2483                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2484             {
2485               snat_in2out_trace_t *t =
2486                  vlib_add_trace (vm, node, b0, sizeof (*t));
2487               t->is_slow_path = 0;
2488               t->sw_if_index = sw_if_index0;
2489               t->next_index = next0;
2490               t->session_index = ~0;
2491               if (ses0)
2492                 t->session_index = ses0 - dm0->sessions;
2493             }
2494
2495           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2496
2497           ip1 = vlib_buffer_get_current (b1);
2498           udp1 = ip4_next_header (ip1);
2499           tcp1 = (tcp_header_t *) udp1;
2500
2501           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2502
2503           if (PREDICT_FALSE(ip1->ttl == 1))
2504             {
2505               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2506               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2507                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2508                                            0);
2509               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2510               goto trace1;
2511             }
2512
2513           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2514
2515           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2516             {
2517               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2518               icmp1 = (icmp46_header_t *) udp1;
2519
2520               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2521                                   rx_fib_index1, node, next1, thread_index,
2522                                   &ses1, &dm1);
2523               goto trace1;
2524             }
2525
2526           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2527           if (PREDICT_FALSE(!dm1))
2528             {
2529               clib_warning("no match for internal host %U",
2530                            format_ip4_address, &ip0->src_address);
2531               next1 = SNAT_IN2OUT_NEXT_DROP;
2532               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2533               goto trace1;
2534             }
2535
2536           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2537
2538           key1.ext_host_addr = ip1->dst_address;
2539           key1.ext_host_port = tcp1->dst;
2540
2541           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2542           if (PREDICT_FALSE(!ses1))
2543             {
2544               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2545                 {
2546                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2547                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2548
2549                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2550                     continue;
2551
2552                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2553                   break;
2554                 }
2555               if (PREDICT_FALSE(!ses1))
2556                 {
2557                   /* too many sessions for user, send ICMP error packet */
2558
2559                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2560                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2561                                                ICMP4_destination_unreachable_destination_unreachable_host,
2562                                                0);
2563                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2564                   goto trace1;
2565                 }
2566             }
2567
2568           new_port1 = ses1->out.out_port;
2569
2570           old_addr1.as_u32 = ip1->src_address.as_u32;
2571           ip1->src_address.as_u32 = new_addr1.as_u32;
2572           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2573
2574           sum1 = ip1->checksum;
2575           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2576                                  ip4_header_t,
2577                                  src_address /* changed member */);
2578           ip1->checksum = ip_csum_fold (sum1);
2579
2580           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2581             {
2582               if (tcp1->flags & TCP_FLAG_SYN)
2583                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2584               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2585                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2586               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2587                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2588               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2589                 snat_det_ses_close(dm1, ses1);
2590               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2591                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2592               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2593                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2594
2595               old_port1 = tcp1->src;
2596               tcp1->src = new_port1;
2597
2598               sum1 = tcp1->checksum;
2599               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2600                                      ip4_header_t,
2601                                      dst_address /* changed member */);
2602               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2603                                      ip4_header_t /* cheat */,
2604                                      length /* changed member */);
2605               tcp1->checksum = ip_csum_fold(sum1);
2606             }
2607           else
2608             {
2609               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2610               old_port1 = udp1->src_port;
2611               udp1->src_port = new_port1;
2612               udp1->checksum = 0;
2613             }
2614
2615           switch(ses1->state)
2616             {
2617             case SNAT_SESSION_UDP_ACTIVE:
2618                 ses1->expire = now + sm->udp_timeout;
2619                 break;
2620             case SNAT_SESSION_TCP_SYN_SENT:
2621             case SNAT_SESSION_TCP_FIN_WAIT:
2622             case SNAT_SESSION_TCP_CLOSE_WAIT:
2623             case SNAT_SESSION_TCP_LAST_ACK:
2624                 ses1->expire = now + sm->tcp_transitory_timeout;
2625                 break;
2626             case SNAT_SESSION_TCP_ESTABLISHED:
2627                 ses1->expire = now + sm->tcp_established_timeout;
2628                 break;
2629             }
2630
2631         trace1:
2632           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2633                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2634             {
2635               snat_in2out_trace_t *t =
2636                  vlib_add_trace (vm, node, b1, sizeof (*t));
2637               t->is_slow_path = 0;
2638               t->sw_if_index = sw_if_index1;
2639               t->next_index = next1;
2640               t->session_index = ~0;
2641               if (ses1)
2642                 t->session_index = ses1 - dm1->sessions;
2643             }
2644
2645           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2646
2647           /* verify speculative enqueues, maybe switch current next frame */
2648           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2649                                            to_next, n_left_to_next,
2650                                            bi0, bi1, next0, next1);
2651          }
2652
2653       while (n_left_from > 0 && n_left_to_next > 0)
2654         {
2655           u32 bi0;
2656           vlib_buffer_t * b0;
2657           u32 next0;
2658           u32 sw_if_index0;
2659           ip4_header_t * ip0;
2660           ip_csum_t sum0;
2661           ip4_address_t new_addr0, old_addr0;
2662           u16 old_port0, new_port0, lo_port0, i0;
2663           udp_header_t * udp0;
2664           tcp_header_t * tcp0;
2665           u32 proto0;
2666           snat_det_out_key_t key0;
2667           snat_det_map_t * dm0;
2668           snat_det_session_t * ses0 = 0;
2669           u32 rx_fib_index0;
2670           icmp46_header_t * icmp0;
2671
2672           /* speculatively enqueue b0 to the current next frame */
2673           bi0 = from[0];
2674           to_next[0] = bi0;
2675           from += 1;
2676           to_next += 1;
2677           n_left_from -= 1;
2678           n_left_to_next -= 1;
2679
2680           b0 = vlib_get_buffer (vm, bi0);
2681           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2682
2683           ip0 = vlib_buffer_get_current (b0);
2684           udp0 = ip4_next_header (ip0);
2685           tcp0 = (tcp_header_t *) udp0;
2686
2687           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2688
2689           if (PREDICT_FALSE(ip0->ttl == 1))
2690             {
2691               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2692               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2693                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2694                                            0);
2695               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2696               goto trace00;
2697             }
2698
2699           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2700
2701           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2702             {
2703               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2704               icmp0 = (icmp46_header_t *) udp0;
2705
2706               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2707                                   rx_fib_index0, node, next0, thread_index,
2708                                   &ses0, &dm0);
2709               goto trace00;
2710             }
2711
2712           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2713           if (PREDICT_FALSE(!dm0))
2714             {
2715               clib_warning("no match for internal host %U",
2716                            format_ip4_address, &ip0->src_address);
2717               next0 = SNAT_IN2OUT_NEXT_DROP;
2718               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2719               goto trace00;
2720             }
2721
2722           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2723
2724           key0.ext_host_addr = ip0->dst_address;
2725           key0.ext_host_port = tcp0->dst;
2726
2727           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2728           if (PREDICT_FALSE(!ses0))
2729             {
2730               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2731                 {
2732                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2733                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2734
2735                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2736                     continue;
2737
2738                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2739                   break;
2740                 }
2741               if (PREDICT_FALSE(!ses0))
2742                 {
2743                   /* too many sessions for user, send ICMP error packet */
2744
2745                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2746                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2747                                                ICMP4_destination_unreachable_destination_unreachable_host,
2748                                                0);
2749                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2750                   goto trace00;
2751                 }
2752             }
2753
2754           new_port0 = ses0->out.out_port;
2755
2756           old_addr0.as_u32 = ip0->src_address.as_u32;
2757           ip0->src_address.as_u32 = new_addr0.as_u32;
2758           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2759
2760           sum0 = ip0->checksum;
2761           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2762                                  ip4_header_t,
2763                                  src_address /* changed member */);
2764           ip0->checksum = ip_csum_fold (sum0);
2765
2766           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2767             {
2768               if (tcp0->flags & TCP_FLAG_SYN)
2769                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2770               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2771                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2772               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2773                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2774               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2775                 snat_det_ses_close(dm0, ses0);
2776               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2777                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2778               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2779                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2780
2781               old_port0 = tcp0->src;
2782               tcp0->src = new_port0;
2783
2784               sum0 = tcp0->checksum;
2785               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2786                                      ip4_header_t,
2787                                      dst_address /* changed member */);
2788               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2789                                      ip4_header_t /* cheat */,
2790                                      length /* changed member */);
2791               tcp0->checksum = ip_csum_fold(sum0);
2792             }
2793           else
2794             {
2795               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2796               old_port0 = udp0->src_port;
2797               udp0->src_port = new_port0;
2798               udp0->checksum = 0;
2799             }
2800
2801           switch(ses0->state)
2802             {
2803             case SNAT_SESSION_UDP_ACTIVE:
2804                 ses0->expire = now + sm->udp_timeout;
2805                 break;
2806             case SNAT_SESSION_TCP_SYN_SENT:
2807             case SNAT_SESSION_TCP_FIN_WAIT:
2808             case SNAT_SESSION_TCP_CLOSE_WAIT:
2809             case SNAT_SESSION_TCP_LAST_ACK:
2810                 ses0->expire = now + sm->tcp_transitory_timeout;
2811                 break;
2812             case SNAT_SESSION_TCP_ESTABLISHED:
2813                 ses0->expire = now + sm->tcp_established_timeout;
2814                 break;
2815             }
2816
2817         trace00:
2818           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2819                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2820             {
2821               snat_in2out_trace_t *t =
2822                  vlib_add_trace (vm, node, b0, sizeof (*t));
2823               t->is_slow_path = 0;
2824               t->sw_if_index = sw_if_index0;
2825               t->next_index = next0;
2826               t->session_index = ~0;
2827               if (ses0)
2828                 t->session_index = ses0 - dm0->sessions;
2829             }
2830
2831           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2832
2833           /* verify speculative enqueue, maybe switch current next frame */
2834           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2835                                            to_next, n_left_to_next,
2836                                            bi0, next0);
2837         }
2838
2839       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2840     }
2841
2842   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2843                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2844                                pkts_processed);
2845   return frame->n_vectors;
2846 }
2847
2848 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2849   .function = snat_det_in2out_node_fn,
2850   .name = "nat44-det-in2out",
2851   .vector_size = sizeof (u32),
2852   .format_trace = format_snat_in2out_trace,
2853   .type = VLIB_NODE_TYPE_INTERNAL,
2854
2855   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2856   .error_strings = snat_in2out_error_strings,
2857
2858   .runtime_data_bytes = sizeof (snat_runtime_t),
2859
2860   .n_next_nodes = 3,
2861
2862   /* edit / add dispositions here */
2863   .next_nodes = {
2864     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2865     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2866     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2867   },
2868 };
2869
2870 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2871
2872 /**
2873  * Get address and port values to be used for ICMP packet translation
2874  * and create session if needed
2875  *
2876  * @param[in,out] sm             NAT main
2877  * @param[in,out] node           NAT node runtime
2878  * @param[in] thread_index       thread index
2879  * @param[in,out] b0             buffer containing packet to be translated
2880  * @param[out] p_proto           protocol used for matching
2881  * @param[out] p_value           address and port after NAT translation
2882  * @param[out] p_dont_translate  if packet should not be translated
2883  * @param d                      optional parameter
2884  * @param e                      optional parameter
2885  */
2886 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
2887                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2888                           snat_session_key_t *p_value,
2889                           u8 *p_dont_translate, void *d, void *e)
2890 {
2891   ip4_header_t *ip0;
2892   icmp46_header_t *icmp0;
2893   u32 sw_if_index0;
2894   u32 rx_fib_index0;
2895   u8 protocol;
2896   snat_det_out_key_t key0;
2897   u8 dont_translate = 0;
2898   u32 next0 = ~0;
2899   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2900   ip4_header_t *inner_ip0;
2901   void *l4_header = 0;
2902   icmp46_header_t *inner_icmp0;
2903   snat_det_map_t * dm0 = 0;
2904   ip4_address_t new_addr0;
2905   u16 lo_port0, i0;
2906   snat_det_session_t * ses0 = 0;
2907   ip4_address_t in_addr;
2908   u16 in_port;
2909
2910   ip0 = vlib_buffer_get_current (b0);
2911   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2912   echo0 = (icmp_echo_header_t *)(icmp0+1);
2913   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2914   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2915
2916   if (!icmp_is_error_message (icmp0))
2917     {
2918       protocol = SNAT_PROTOCOL_ICMP;
2919       in_addr = ip0->src_address;
2920       in_port = echo0->identifier;
2921     }
2922   else
2923     {
2924       inner_ip0 = (ip4_header_t *)(echo0+1);
2925       l4_header = ip4_next_header (inner_ip0);
2926       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2927       in_addr = inner_ip0->dst_address;
2928       switch (protocol)
2929         {
2930         case SNAT_PROTOCOL_ICMP:
2931           inner_icmp0 = (icmp46_header_t*)l4_header;
2932           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2933           in_port = inner_echo0->identifier;
2934           break;
2935         case SNAT_PROTOCOL_UDP:
2936         case SNAT_PROTOCOL_TCP:
2937           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2938           break;
2939         default:
2940           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
2941           next0 = SNAT_IN2OUT_NEXT_DROP;
2942           goto out;
2943         }
2944     }
2945
2946   dm0 = snat_det_map_by_user(sm, &in_addr);
2947   if (PREDICT_FALSE(!dm0))
2948     {
2949       clib_warning("no match for internal host %U",
2950                    format_ip4_address, &in_addr);
2951       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2952           IP_PROTOCOL_ICMP, rx_fib_index0)))
2953         {
2954           dont_translate = 1;
2955           goto out;
2956         }
2957       next0 = SNAT_IN2OUT_NEXT_DROP;
2958       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2959       goto out;
2960     }
2961
2962   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
2963
2964   key0.ext_host_addr = ip0->dst_address;
2965   key0.ext_host_port = 0;
2966
2967   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
2968   if (PREDICT_FALSE(!ses0))
2969     {
2970       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2971           IP_PROTOCOL_ICMP, rx_fib_index0)))
2972         {
2973           dont_translate = 1;
2974           goto out;
2975         }
2976       if (icmp0->type != ICMP4_echo_request)
2977         {
2978           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2979           next0 = SNAT_IN2OUT_NEXT_DROP;
2980           goto out;
2981         }
2982       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2983         {
2984           key0.out_port = clib_host_to_net_u16 (lo_port0 +
2985             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
2986
2987           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
2988             continue;
2989
2990           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
2991           break;
2992         }
2993       if (PREDICT_FALSE(!ses0))
2994         {
2995           next0 = SNAT_IN2OUT_NEXT_DROP;
2996           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2997           goto out;
2998         }
2999     }
3000
3001   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3002                     !icmp_is_error_message (icmp0)))
3003     {
3004       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3005       next0 = SNAT_IN2OUT_NEXT_DROP;
3006       goto out;
3007     }
3008
3009   u32 now = (u32) vlib_time_now (sm->vlib_main);
3010
3011   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3012   ses0->expire = now + sm->icmp_timeout;
3013
3014 out:
3015   *p_proto = protocol;
3016   if (ses0)
3017     {
3018       p_value->addr = new_addr0;
3019       p_value->fib_index = sm->outside_fib_index;
3020       p_value->port = ses0->out.out_port;
3021     }
3022   *p_dont_translate = dont_translate;
3023   if (d)
3024     *(snat_det_session_t**)d = ses0;
3025   if (e)
3026     *(snat_det_map_t**)e = dm0;
3027   return next0;
3028 }
3029
3030 /**********************/
3031 /*** worker handoff ***/
3032 /**********************/
3033 static inline uword
3034 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3035                                       vlib_node_runtime_t * node,
3036                                       vlib_frame_t * frame,
3037                                       u8 is_output)
3038 {
3039   snat_main_t *sm = &snat_main;
3040   vlib_thread_main_t *tm = vlib_get_thread_main ();
3041   u32 n_left_from, *from, *to_next = 0;
3042   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3043   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3044     = 0;
3045   vlib_frame_queue_elt_t *hf = 0;
3046   vlib_frame_t *f = 0;
3047   int i;
3048   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3049   u32 next_worker_index = 0;
3050   u32 current_worker_index = ~0;
3051   u32 thread_index = vlib_get_thread_index ();
3052   u32 fq_index;
3053   u32 to_node_index;
3054
3055   ASSERT (vec_len (sm->workers));
3056
3057   if (is_output)
3058     {
3059       fq_index = sm->fq_in2out_output_index;
3060       to_node_index = sm->in2out_output_node_index;
3061     }
3062   else
3063     {
3064       fq_index = sm->fq_in2out_index;
3065       to_node_index = sm->in2out_node_index;
3066     }
3067
3068   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3069     {
3070       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3071
3072       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3073                                sm->first_worker_index + sm->num_workers - 1,
3074                                (vlib_frame_queue_t *) (~0));
3075     }
3076
3077   from = vlib_frame_vector_args (frame);
3078   n_left_from = frame->n_vectors;
3079
3080   while (n_left_from > 0)
3081     {
3082       u32 bi0;
3083       vlib_buffer_t *b0;
3084       u32 sw_if_index0;
3085       u32 rx_fib_index0;
3086       ip4_header_t * ip0;
3087       u8 do_handoff;
3088
3089       bi0 = from[0];
3090       from += 1;
3091       n_left_from -= 1;
3092
3093       b0 = vlib_get_buffer (vm, bi0);
3094
3095       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3096       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3097
3098       ip0 = vlib_buffer_get_current (b0);
3099
3100       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3101
3102       if (PREDICT_FALSE (next_worker_index != thread_index))
3103         {
3104           do_handoff = 1;
3105
3106           if (next_worker_index != current_worker_index)
3107             {
3108               if (hf)
3109                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3110
3111               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3112                                                       next_worker_index,
3113                                                       handoff_queue_elt_by_worker_index);
3114
3115               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3116               to_next_worker = &hf->buffer_index[hf->n_vectors];
3117               current_worker_index = next_worker_index;
3118             }
3119
3120           /* enqueue to correct worker thread */
3121           to_next_worker[0] = bi0;
3122           to_next_worker++;
3123           n_left_to_next_worker--;
3124
3125           if (n_left_to_next_worker == 0)
3126             {
3127               hf->n_vectors = VLIB_FRAME_SIZE;
3128               vlib_put_frame_queue_elt (hf);
3129               current_worker_index = ~0;
3130               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3131               hf = 0;
3132             }
3133         }
3134       else
3135         {
3136           do_handoff = 0;
3137           /* if this is 1st frame */
3138           if (!f)
3139             {
3140               f = vlib_get_frame_to_node (vm, to_node_index);
3141               to_next = vlib_frame_vector_args (f);
3142             }
3143
3144           to_next[0] = bi0;
3145           to_next += 1;
3146           f->n_vectors++;
3147         }
3148
3149       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3150                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3151         {
3152           snat_in2out_worker_handoff_trace_t *t =
3153             vlib_add_trace (vm, node, b0, sizeof (*t));
3154           t->next_worker_index = next_worker_index;
3155           t->do_handoff = do_handoff;
3156         }
3157     }
3158
3159   if (f)
3160     vlib_put_frame_to_node (vm, to_node_index, f);
3161
3162   if (hf)
3163     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3164
3165   /* Ship frames to the worker nodes */
3166   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3167     {
3168       if (handoff_queue_elt_by_worker_index[i])
3169         {
3170           hf = handoff_queue_elt_by_worker_index[i];
3171           /*
3172            * It works better to let the handoff node
3173            * rate-adapt, always ship the handoff queue element.
3174            */
3175           if (1 || hf->n_vectors == hf->last_n_vectors)
3176             {
3177               vlib_put_frame_queue_elt (hf);
3178               handoff_queue_elt_by_worker_index[i] = 0;
3179             }
3180           else
3181             hf->last_n_vectors = hf->n_vectors;
3182         }
3183       congested_handoff_queue_by_worker_index[i] =
3184         (vlib_frame_queue_t *) (~0);
3185     }
3186   hf = 0;
3187   current_worker_index = ~0;
3188   return frame->n_vectors;
3189 }
3190
3191 static uword
3192 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3193                                vlib_node_runtime_t * node,
3194                                vlib_frame_t * frame)
3195 {
3196   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3197 }
3198
3199 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3200   .function = snat_in2out_worker_handoff_fn,
3201   .name = "nat44-in2out-worker-handoff",
3202   .vector_size = sizeof (u32),
3203   .format_trace = format_snat_in2out_worker_handoff_trace,
3204   .type = VLIB_NODE_TYPE_INTERNAL,
3205
3206   .n_next_nodes = 1,
3207
3208   .next_nodes = {
3209     [0] = "error-drop",
3210   },
3211 };
3212
3213 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3214                               snat_in2out_worker_handoff_fn);
3215
3216 static uword
3217 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3218                                       vlib_node_runtime_t * node,
3219                                       vlib_frame_t * frame)
3220 {
3221   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3222 }
3223
3224 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3225   .function = snat_in2out_output_worker_handoff_fn,
3226   .name = "nat44-in2out-output-worker-handoff",
3227   .vector_size = sizeof (u32),
3228   .format_trace = format_snat_in2out_worker_handoff_trace,
3229   .type = VLIB_NODE_TYPE_INTERNAL,
3230
3231   .n_next_nodes = 1,
3232
3233   .next_nodes = {
3234     [0] = "error-drop",
3235   },
3236 };
3237
3238 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3239                               snat_in2out_output_worker_handoff_fn);
3240
3241 static_always_inline int
3242 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3243 {
3244   snat_address_t * ap;
3245   clib_bihash_kv_8_8_t kv, value;
3246   snat_session_key_t m_key;
3247
3248   vec_foreach (ap, sm->addresses)
3249     {
3250       if (ap->addr.as_u32 == dst_addr->as_u32)
3251         return 1;
3252     }
3253
3254   m_key.addr.as_u32 = dst_addr->as_u32;
3255   m_key.fib_index = sm->outside_fib_index;
3256   m_key.port = 0;
3257   m_key.protocol = 0;
3258   kv.key = m_key.as_u64;
3259   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3260     return 1;
3261
3262   return 0;
3263 }
3264
3265 static uword
3266 snat_hairpin_dst_fn (vlib_main_t * vm,
3267                      vlib_node_runtime_t * node,
3268                      vlib_frame_t * frame)
3269 {
3270   u32 n_left_from, * from, * to_next;
3271   snat_in2out_next_t next_index;
3272   u32 pkts_processed = 0;
3273   snat_main_t * sm = &snat_main;
3274
3275   from = vlib_frame_vector_args (frame);
3276   n_left_from = frame->n_vectors;
3277   next_index = node->cached_next_index;
3278
3279   while (n_left_from > 0)
3280     {
3281       u32 n_left_to_next;
3282
3283       vlib_get_next_frame (vm, node, next_index,
3284                            to_next, n_left_to_next);
3285
3286       while (n_left_from > 0 && n_left_to_next > 0)
3287         {
3288           u32 bi0;
3289           vlib_buffer_t * b0;
3290           u32 next0;
3291           ip4_header_t * ip0;
3292           u32 proto0;
3293
3294           /* speculatively enqueue b0 to the current next frame */
3295           bi0 = from[0];
3296           to_next[0] = bi0;
3297           from += 1;
3298           to_next += 1;
3299           n_left_from -= 1;
3300           n_left_to_next -= 1;
3301
3302           b0 = vlib_get_buffer (vm, bi0);
3303           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3304           ip0 = vlib_buffer_get_current (b0);
3305
3306           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3307
3308           vnet_buffer (b0)->snat.flags = 0;
3309           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3310             {
3311               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3312                 {
3313                   udp_header_t * udp0 = ip4_next_header (ip0);
3314                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3315
3316                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3317                 }
3318               else if (proto0 == SNAT_PROTOCOL_ICMP)
3319                 {
3320                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3321
3322                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3323                 }
3324               else
3325                 {
3326                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3327                 }
3328
3329               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3330               clib_warning("is hairpinning");
3331             }
3332
3333           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3334
3335           /* verify speculative enqueue, maybe switch current next frame */
3336           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3337                                            to_next, n_left_to_next,
3338                                            bi0, next0);
3339          }
3340
3341       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3342     }
3343
3344   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3345                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3346                                pkts_processed);
3347   return frame->n_vectors;
3348 }
3349
3350 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3351   .function = snat_hairpin_dst_fn,
3352   .name = "nat44-hairpin-dst",
3353   .vector_size = sizeof (u32),
3354   .type = VLIB_NODE_TYPE_INTERNAL,
3355   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3356   .error_strings = snat_in2out_error_strings,
3357   .n_next_nodes = 2,
3358   .next_nodes = {
3359     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3360     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3361   },
3362 };
3363
3364 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3365                               snat_hairpin_dst_fn);
3366
3367 static uword
3368 snat_hairpin_src_fn (vlib_main_t * vm,
3369                      vlib_node_runtime_t * node,
3370                      vlib_frame_t * frame)
3371 {
3372   u32 n_left_from, * from, * to_next;
3373   snat_in2out_next_t next_index;
3374   u32 pkts_processed = 0;
3375   snat_main_t *sm = &snat_main;
3376
3377   from = vlib_frame_vector_args (frame);
3378   n_left_from = frame->n_vectors;
3379   next_index = node->cached_next_index;
3380
3381   while (n_left_from > 0)
3382     {
3383       u32 n_left_to_next;
3384
3385       vlib_get_next_frame (vm, node, next_index,
3386                            to_next, n_left_to_next);
3387
3388       while (n_left_from > 0 && n_left_to_next > 0)
3389         {
3390           u32 bi0;
3391           vlib_buffer_t * b0;
3392           u32 next0;
3393           snat_interface_t *i;
3394           u32 sw_if_index0;
3395
3396           /* speculatively enqueue b0 to the current next frame */
3397           bi0 = from[0];
3398           to_next[0] = bi0;
3399           from += 1;
3400           to_next += 1;
3401           n_left_from -= 1;
3402           n_left_to_next -= 1;
3403
3404           b0 = vlib_get_buffer (vm, bi0);
3405           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3406           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3407
3408           pool_foreach (i, sm->output_feature_interfaces,
3409           ({
3410             /* Only packets from NAT inside interface */
3411             if ((i->is_inside == 1) && (sw_if_index0 == i->sw_if_index))
3412               {
3413                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3414                                     SNAT_FLAG_HAIRPINNING))
3415                   {
3416                     if (PREDICT_TRUE (sm->num_workers > 1))
3417                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3418                     else
3419                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3420                   }
3421                 break;
3422               }
3423           }));
3424
3425           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3426
3427           /* verify speculative enqueue, maybe switch current next frame */
3428           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3429                                            to_next, n_left_to_next,
3430                                            bi0, next0);
3431          }
3432
3433       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3434     }
3435
3436   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3437                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3438                                pkts_processed);
3439   return frame->n_vectors;
3440 }
3441
3442 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3443   .function = snat_hairpin_src_fn,
3444   .name = "nat44-hairpin-src",
3445   .vector_size = sizeof (u32),
3446   .type = VLIB_NODE_TYPE_INTERNAL,
3447   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3448   .error_strings = snat_in2out_error_strings,
3449   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3450   .next_nodes = {
3451      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3452      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3453      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3454      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3455   },
3456 };
3457
3458 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3459                               snat_hairpin_src_fn);
3460
3461 static uword
3462 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3463                                 vlib_node_runtime_t * node,
3464                                 vlib_frame_t * frame)
3465 {
3466   u32 n_left_from, * from, * to_next;
3467   snat_in2out_next_t next_index;
3468   u32 pkts_processed = 0;
3469   snat_main_t * sm = &snat_main;
3470   u32 stats_node_index;
3471
3472   stats_node_index = snat_in2out_fast_node.index;
3473
3474   from = vlib_frame_vector_args (frame);
3475   n_left_from = frame->n_vectors;
3476   next_index = node->cached_next_index;
3477
3478   while (n_left_from > 0)
3479     {
3480       u32 n_left_to_next;
3481
3482       vlib_get_next_frame (vm, node, next_index,
3483                            to_next, n_left_to_next);
3484
3485       while (n_left_from > 0 && n_left_to_next > 0)
3486         {
3487           u32 bi0;
3488           vlib_buffer_t * b0;
3489           u32 next0;
3490           u32 sw_if_index0;
3491           ip4_header_t * ip0;
3492           ip_csum_t sum0;
3493           u32 new_addr0, old_addr0;
3494           u16 old_port0, new_port0;
3495           udp_header_t * udp0;
3496           tcp_header_t * tcp0;
3497           icmp46_header_t * icmp0;
3498           snat_session_key_t key0, sm0;
3499           u32 proto0;
3500           u32 rx_fib_index0;
3501
3502           /* speculatively enqueue b0 to the current next frame */
3503           bi0 = from[0];
3504           to_next[0] = bi0;
3505           from += 1;
3506           to_next += 1;
3507           n_left_from -= 1;
3508           n_left_to_next -= 1;
3509
3510           b0 = vlib_get_buffer (vm, bi0);
3511           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3512
3513           ip0 = vlib_buffer_get_current (b0);
3514           udp0 = ip4_next_header (ip0);
3515           tcp0 = (tcp_header_t *) udp0;
3516           icmp0 = (icmp46_header_t *) udp0;
3517
3518           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3519           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3520
3521           if (PREDICT_FALSE(ip0->ttl == 1))
3522             {
3523               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3524               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3525                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3526                                            0);
3527               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3528               goto trace0;
3529             }
3530
3531           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3532
3533           if (PREDICT_FALSE (proto0 == ~0))
3534               goto trace0;
3535
3536           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3537             {
3538               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3539                                   rx_fib_index0, node, next0, ~0, 0, 0);
3540               goto trace0;
3541             }
3542
3543           key0.addr = ip0->src_address;
3544           key0.protocol = proto0;
3545           key0.port = udp0->src_port;
3546           key0.fib_index = rx_fib_index0;
3547
3548           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
3549             {
3550               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3551               next0= SNAT_IN2OUT_NEXT_DROP;
3552               goto trace0;
3553             }
3554
3555           new_addr0 = sm0.addr.as_u32;
3556           new_port0 = sm0.port;
3557           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3558           old_addr0 = ip0->src_address.as_u32;
3559           ip0->src_address.as_u32 = new_addr0;
3560
3561           sum0 = ip0->checksum;
3562           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3563                                  ip4_header_t,
3564                                  src_address /* changed member */);
3565           ip0->checksum = ip_csum_fold (sum0);
3566
3567           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3568             {
3569               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3570                 {
3571                   old_port0 = tcp0->src_port;
3572                   tcp0->src_port = new_port0;
3573
3574                   sum0 = tcp0->checksum;
3575                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3576                                          ip4_header_t,
3577                                          dst_address /* changed member */);
3578                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3579                                          ip4_header_t /* cheat */,
3580                                          length /* changed member */);
3581                   tcp0->checksum = ip_csum_fold(sum0);
3582                 }
3583               else
3584                 {
3585                   old_port0 = udp0->src_port;
3586                   udp0->src_port = new_port0;
3587                   udp0->checksum = 0;
3588                 }
3589             }
3590           else
3591             {
3592               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3593                 {
3594                   sum0 = tcp0->checksum;
3595                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3596                                          ip4_header_t,
3597                                          dst_address /* changed member */);
3598                   tcp0->checksum = ip_csum_fold(sum0);
3599                 }
3600             }
3601
3602           /* Hairpinning */
3603           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3604
3605         trace0:
3606           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3607                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3608             {
3609               snat_in2out_trace_t *t =
3610                  vlib_add_trace (vm, node, b0, sizeof (*t));
3611               t->sw_if_index = sw_if_index0;
3612               t->next_index = next0;
3613             }
3614
3615           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3616
3617           /* verify speculative enqueue, maybe switch current next frame */
3618           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3619                                            to_next, n_left_to_next,
3620                                            bi0, next0);
3621         }
3622
3623       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3624     }
3625
3626   vlib_node_increment_counter (vm, stats_node_index,
3627                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3628                                pkts_processed);
3629   return frame->n_vectors;
3630 }
3631
3632
3633 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3634   .function = snat_in2out_fast_static_map_fn,
3635   .name = "nat44-in2out-fast",
3636   .vector_size = sizeof (u32),
3637   .format_trace = format_snat_in2out_fast_trace,
3638   .type = VLIB_NODE_TYPE_INTERNAL,
3639
3640   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3641   .error_strings = snat_in2out_error_strings,
3642
3643   .runtime_data_bytes = sizeof (snat_runtime_t),
3644
3645   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3646
3647   /* edit / add dispositions here */
3648   .next_nodes = {
3649     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3650     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3651     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3652     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3653   },
3654 };
3655
3656 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);