src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91 vlib_node_registration_t snat_in2out_output_node;
  92 vlib_node_registration_t snat_in2out_output_slowpath_node;
  93 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
  94 vlib_node_registration_t snat_hairpin_dst_node;
  95 vlib_node_registration_t snat_hairpin_src_node;
  96
  97
  98 #define foreach_snat_in2out_error                       \
  99 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 100 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 101 _(OUT_OF_PORTS, "Out of ports")                         \
 102 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 103 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 104 _(NO_TRANSLATION, "No translation")
 105
 106 typedef enum {
 107 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 108   foreach_snat_in2out_error
 109 #undef _
 110   SNAT_IN2OUT_N_ERROR,
 111 } snat_in2out_error_t;
 112
 113 static char * snat_in2out_error_strings[] = {
 114 #define _(sym,string) string,
 115   foreach_snat_in2out_error
 116 #undef _
 117 };
 118
 119 typedef enum {
 120   SNAT_IN2OUT_NEXT_LOOKUP,
 121   SNAT_IN2OUT_NEXT_DROP,
 122   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 123   SNAT_IN2OUT_NEXT_SLOW_PATH,
 124   SNAT_IN2OUT_N_NEXT,
 125 } snat_in2out_next_t;
 126
 127 typedef enum {
 128   SNAT_HAIRPIN_SRC_NEXT_DROP,
 129   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 130   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 131   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 132   SNAT_HAIRPIN_SRC_N_NEXT,
 133 } snat_hairpin_next_t;
 134
 135 /**
 136  * @brief Check if packet should be translated
 137  *
 138  * Packets aimed at outside interface and external addresss with active session
 139  * should be translated.
 140  *
 141  * @param sm            NAT main
 142  * @param rt            NAT runtime data
 143  * @param sw_if_index0  index of the inside interface
 144  * @param ip0           IPv4 header
 145  * @param proto0        NAT protocol
 146  * @param rx_fib_index0 RX FIB index
 147  *
 148  * @returns 0 if packet should be translated otherwise 1
 149  */
 150 static inline int
 151 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 152                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 153                          u32 rx_fib_index0)
 154 {
 155   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 156   fib_prefix_t pfx = {
 157     .fp_proto = FIB_PROTOCOL_IP4,
 158     .fp_len = 32,
 159     .fp_addr = {
 160         .ip4.as_u32 = ip0->dst_address.as_u32,
 161     },
 162   };
 163
 164   /* Don't NAT packet aimed at the intfc address */
 165   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 166                                       ip0->dst_address.as_u32)))
 167     return 1;
 168
 169   fei = fib_table_lookup (rx_fib_index0, &pfx);
 170   if (FIB_NODE_INDEX_INVALID != fei)
 171     {
 172       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 173       if (sw_if_index == ~0)
 174         {
 175           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 176           if (FIB_NODE_INDEX_INVALID != fei)
 177             sw_if_index = fib_entry_get_resolving_interface (fei);
 178         }
 179       snat_interface_t *i;
 180       pool_foreach (i, sm->interfaces,
 181       ({
 182         /* NAT packet aimed at outside interface */
 183         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 184           return 0;
 185       }));
 186     }
 187
 188   return 1;
 189 }
 190
 191 static inline int
 192 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 193                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 194                     u32 rx_fib_index0, u32 thread_index)
 195 {
 196   udp_header_t * udp0 = ip4_next_header (ip0);
 197   snat_session_key_t key0, sm0;
 198   clib_bihash_kv_8_8_t kv0, value0;
 199
 200   key0.addr = ip0->dst_address;
 201   key0.port = udp0->dst_port;
 202   key0.protocol = proto0;
 203   key0.fib_index = sm->outside_fib_index;
 204   kv0.key = key0.as_u64;
 205
 206   /* NAT packet aimed at external address if */
 207   /* has active sessions */
 208   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 209                               &value0))
 210     {
 211       /* or is static mappings */
 212       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 213         return 0;
 214     }
 215   else
 216     return 0;
 217
 218   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 219                                  rx_fib_index0);
 220 }
 221
 222 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 223                       ip4_header_t * ip0,
 224                       u32 rx_fib_index0,
 225                       snat_session_key_t * key0,
 226                       snat_session_t ** sessionp,
 227                       vlib_node_runtime_t * node,
 228                       u32 next0,
 229                       u32 thread_index)
 230 {
 231   snat_user_t *u;
 232   snat_user_key_t user_key;
 233   snat_session_t *s;
 234   clib_bihash_kv_8_8_t kv0, value0;
 235   u32 oldest_per_user_translation_list_index;
 236   dlist_elt_t * oldest_per_user_translation_list_elt;
 237   dlist_elt_t * per_user_translation_list_elt;
 238   dlist_elt_t * per_user_list_head_elt;
 239   u32 session_index;
 240   snat_session_key_t key1;
 241   u32 address_index = ~0;
 242   u32 outside_fib_index;
 243   uword * p;
 244   snat_worker_key_t worker_by_out_key;
 245
 246   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 247   if (! p)
 248     {
 249       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 250       return SNAT_IN2OUT_NEXT_DROP;
 251     }
 252   outside_fib_index = p[0];
 253
 254   key1.protocol = key0->protocol;
 255   user_key.addr = ip0->src_address;
 256   user_key.fib_index = rx_fib_index0;
 257   kv0.key = user_key.as_u64;
 258
 259   /* Ever heard of the "user" = src ip4 address before? */
 260   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 261                               &kv0, &value0))
 262     {
 263       /* no, make a new one */
 264       pool_get (sm->per_thread_data[thread_index].users, u);
 265       memset (u, 0, sizeof (*u));
 266       u->addr = ip0->src_address;
 267       u->fib_index = rx_fib_index0;
 268
 269       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 270
 271       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 272         sm->per_thread_data[thread_index].list_pool;
 273
 274       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 275                        u->sessions_per_user_list_head_index);
 276
 277       kv0.value = u - sm->per_thread_data[thread_index].users;
 278
 279       /* add user */
 280       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 281                                &kv0, 1 /* is_add */);
 282     }
 283   else
 284     {
 285       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 286                              value0.value);
 287     }
 288
 289   /* Over quota? Recycle the least recently used dynamic translation */
 290   if (u->nsessions >= sm->max_translations_per_user)
 291     {
 292       /* Remove the oldest dynamic translation */
 293       do {
 294           oldest_per_user_translation_list_index =
 295             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 296                                     u->sessions_per_user_list_head_index);
 297
 298           ASSERT (oldest_per_user_translation_list_index != ~0);
 299
 300           /* add it back to the end of the LRU list */
 301           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 302                               u->sessions_per_user_list_head_index,
 303                               oldest_per_user_translation_list_index);
 304           /* Get the list element */
 305           oldest_per_user_translation_list_elt =
 306             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 307                                oldest_per_user_translation_list_index);
 308
 309           /* Get the session index from the list element */
 310           session_index = oldest_per_user_translation_list_elt->value;
 311
 312           /* Get the session */
 313           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 314                                  session_index);
 315       } while (snat_is_session_static (s));
 316
 317       if (snat_is_unk_proto_session (s))
 318         {
 319           clib_bihash_kv_16_8_t up_kv;
 320           nat_ed_ses_key_t key;
 321
 322           /* Remove from lookup tables */
 323           key.l_addr = s->in2out.addr;
 324           key.r_addr = s->ext_host_addr;
 325           key.fib_index = s->in2out.fib_index;
 326           key.proto = s->in2out.port;
 327           key.rsvd = 0;
 328           key.l_port = 0;
 329           up_kv.key[0] = key.as_u64[0];
 330           up_kv.key[1] = key.as_u64[1];
 331           if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &up_kv, 0))
 332             clib_warning ("in2out key del failed");
 333
 334           key.l_addr = s->out2in.addr;
 335           key.fib_index = s->out2in.fib_index;
 336           up_kv.key[0] = key.as_u64[0];
 337           up_kv.key[1] = key.as_u64[1];
 338           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &up_kv, 0))
 339             clib_warning ("out2in key del failed");
 340         }
 341       else
 342         {
 343           /* Remove in2out, out2in keys */
 344           kv0.key = s->in2out.as_u64;
 345           if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out,
 346                                        &kv0, 0 /* is_add */))
 347               clib_warning ("in2out key delete failed");
 348           kv0.key = s->out2in.as_u64;
 349           if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in,
 350                                        &kv0, 0 /* is_add */))
 351               clib_warning ("out2in key delete failed");
 352
 353           /* log NAT event */
 354           snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 355                                               s->out2in.addr.as_u32,
 356                                               s->in2out.protocol,
 357                                               s->in2out.port,
 358                                               s->out2in.port,
 359                                               s->in2out.fib_index);
 360
 361           snat_free_outside_address_and_port
 362             (sm, thread_index, &s->out2in, s->outside_address_index);
 363         }
 364       s->outside_address_index = ~0;
 365
 366       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, thread_index,
 367                                                &key1, &address_index))
 368         {
 369           ASSERT(0);
 370
 371           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 372           return SNAT_IN2OUT_NEXT_DROP;
 373         }
 374       s->outside_address_index = address_index;
 375     }
 376   else
 377     {
 378       u8 static_mapping = 1;
 379
 380       /* First try to match static mapping by local address and port */
 381       if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
 382         {
 383           static_mapping = 0;
 384           /* Try to create dynamic translation */
 385           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0,
 386                                                    thread_index, &key1,
 387                                                    &address_index))
 388             {
 389               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 390               return SNAT_IN2OUT_NEXT_DROP;
 391             }
 392         }
 393
 394       /* Create a new session */
 395       pool_get (sm->per_thread_data[thread_index].sessions, s);
 396       memset (s, 0, sizeof (*s));
 397
 398       s->outside_address_index = address_index;
 399
 400       if (static_mapping)
 401         {
 402           u->nstaticsessions++;
 403           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 404         }
 405       else
 406         {
 407           u->nsessions++;
 408         }
 409
 410       /* Create list elts */
 411       pool_get (sm->per_thread_data[thread_index].list_pool,
 412                 per_user_translation_list_elt);
 413       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 414                        per_user_translation_list_elt -
 415                        sm->per_thread_data[thread_index].list_pool);
 416
 417       per_user_translation_list_elt->value =
 418         s - sm->per_thread_data[thread_index].sessions;
 419       s->per_user_index = per_user_translation_list_elt -
 420                           sm->per_thread_data[thread_index].list_pool;
 421       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 422
 423       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 424                           s->per_user_list_head_index,
 425                           per_user_translation_list_elt -
 426                           sm->per_thread_data[thread_index].list_pool);
 427    }
 428
 429   s->in2out = *key0;
 430   s->out2in = key1;
 431   s->out2in.protocol = key0->protocol;
 432   s->out2in.fib_index = outside_fib_index;
 433   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 434   *sessionp = s;
 435
 436   /* Add to translation hashes */
 437   kv0.key = s->in2out.as_u64;
 438   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 439   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 440                                1 /* is_add */))
 441       clib_warning ("in2out key add failed");
 442
 443   kv0.key = s->out2in.as_u64;
 444   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 445
 446   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 447                                1 /* is_add */))
 448       clib_warning ("out2in key add failed");
 449
 450   /* Add to translated packets worker lookup */
 451   worker_by_out_key.addr = s->out2in.addr;
 452   worker_by_out_key.port = s->out2in.port;
 453   worker_by_out_key.fib_index = s->out2in.fib_index;
 454   kv0.key = worker_by_out_key.as_u64;
 455   kv0.value = thread_index;
 456   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 457
 458   /* log NAT event */
 459   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 460                                       s->out2in.addr.as_u32,
 461                                       s->in2out.protocol,
 462                                       s->in2out.port,
 463                                       s->out2in.port,
 464                                       s->in2out.fib_index);
 465   return next0;
 466 }
 467
 468 static_always_inline
 469 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 470                                  snat_session_key_t *p_key0)
 471 {
 472   icmp46_header_t *icmp0;
 473   snat_session_key_t key0;
 474   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 475   ip4_header_t *inner_ip0 = 0;
 476   void *l4_header = 0;
 477   icmp46_header_t *inner_icmp0;
 478
 479   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 480   echo0 = (icmp_echo_header_t *)(icmp0+1);
 481
 482   if (!icmp_is_error_message (icmp0))
 483     {
 484       key0.protocol = SNAT_PROTOCOL_ICMP;
 485       key0.addr = ip0->src_address;
 486       key0.port = echo0->identifier;
 487     }
 488   else
 489     {
 490       inner_ip0 = (ip4_header_t *)(echo0+1);
 491       l4_header = ip4_next_header (inner_ip0);
 492       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 493       key0.addr = inner_ip0->dst_address;
 494       switch (key0.protocol)
 495         {
 496         case SNAT_PROTOCOL_ICMP:
 497           inner_icmp0 = (icmp46_header_t*)l4_header;
 498           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 499           key0.port = inner_echo0->identifier;
 500           break;
 501         case SNAT_PROTOCOL_UDP:
 502         case SNAT_PROTOCOL_TCP:
 503           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 504           break;
 505         default:
 506           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 507         }
 508     }
 509   *p_key0 = key0;
 510   return -1; /* success */
 511 }
 512
 513 /**
 514  * Get address and port values to be used for ICMP packet translation
 515  * and create session if needed
 516  *
 517  * @param[in,out] sm             NAT main
 518  * @param[in,out] node           NAT node runtime
 519  * @param[in] thread_index       thread index
 520  * @param[in,out] b0             buffer containing packet to be translated
 521  * @param[out] p_proto           protocol used for matching
 522  * @param[out] p_value           address and port after NAT translation
 523  * @param[out] p_dont_translate  if packet should not be translated
 524  * @param d                      optional parameter
 525  * @param e                      optional parameter
 526  */
 527 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 528                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 529                            snat_session_key_t *p_value,
 530                            u8 *p_dont_translate, void *d, void *e)
 531 {
 532   ip4_header_t *ip0;
 533   icmp46_header_t *icmp0;
 534   u32 sw_if_index0;
 535   u32 rx_fib_index0;
 536   snat_session_key_t key0;
 537   snat_session_t *s0 = 0;
 538   u8 dont_translate = 0;
 539   clib_bihash_kv_8_8_t kv0, value0;
 540   u32 next0 = ~0;
 541   int err;
 542   u32 iph_offset0 = 0;
 543
 544   if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
 545     {
 546       iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 547     }
 548   ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
 549   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 550   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 551   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 552
 553   err = icmp_get_key (ip0, &key0);
 554   if (err != -1)
 555     {
 556       b0->error = node->errors[err];
 557       next0 = SNAT_IN2OUT_NEXT_DROP;
 558       goto out;
 559     }
 560   key0.fib_index = rx_fib_index0;
 561
 562   kv0.key = key0.as_u64;
 563
 564   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 565                               &value0))
 566     {
 567       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 568           IP_PROTOCOL_ICMP, rx_fib_index0, thread_index) &&
 569           vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
 570         {
 571           dont_translate = 1;
 572           goto out;
 573         }
 574
 575       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 576         {
 577           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 578           next0 = SNAT_IN2OUT_NEXT_DROP;
 579           goto out;
 580         }
 581
 582       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 583                          &s0, node, next0, thread_index);
 584
 585       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 586         goto out;
 587     }
 588   else
 589     {
 590       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 591                         icmp0->type != ICMP4_echo_reply &&
 592                         !icmp_is_error_message (icmp0)))
 593         {
 594           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 595           next0 = SNAT_IN2OUT_NEXT_DROP;
 596           goto out;
 597         }
 598
 599       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 600                               value0.value);
 601     }
 602
 603 out:
 604   *p_proto = key0.protocol;
 605   if (s0)
 606     *p_value = s0->out2in;
 607   *p_dont_translate = dont_translate;
 608   if (d)
 609     *(snat_session_t**)d = s0;
 610   return next0;
 611 }
 612
 613 /**
 614  * Get address and port values to be used for ICMP packet translation
 615  *
 616  * @param[in] sm                 NAT main
 617  * @param[in,out] node           NAT node runtime
 618  * @param[in] thread_index       thread index
 619  * @param[in,out] b0             buffer containing packet to be translated
 620  * @param[out] p_proto           protocol used for matching
 621  * @param[out] p_value           address and port after NAT translation
 622  * @param[out] p_dont_translate  if packet should not be translated
 623  * @param d                      optional parameter
 624  * @param e                      optional parameter
 625  */
 626 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 627                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 628                            snat_session_key_t *p_value,
 629                            u8 *p_dont_translate, void *d, void *e)
 630 {
 631   ip4_header_t *ip0;
 632   icmp46_header_t *icmp0;
 633   u32 sw_if_index0;
 634   u32 rx_fib_index0;
 635   snat_session_key_t key0;
 636   snat_session_key_t sm0;
 637   u8 dont_translate = 0;
 638   u8 is_addr_only;
 639   u32 next0 = ~0;
 640   int err;
 641
 642   ip0 = vlib_buffer_get_current (b0);
 643   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 644   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 645   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 646
 647   err = icmp_get_key (ip0, &key0);
 648   if (err != -1)
 649     {
 650       b0->error = node->errors[err];
 651       next0 = SNAT_IN2OUT_NEXT_DROP;
 652       goto out2;
 653     }
 654   key0.fib_index = rx_fib_index0;
 655
 656   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
 657     {
 658       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 659           IP_PROTOCOL_ICMP, rx_fib_index0)))
 660         {
 661           dont_translate = 1;
 662           goto out;
 663         }
 664
 665       if (icmp_is_error_message (icmp0))
 666         {
 667           next0 = SNAT_IN2OUT_NEXT_DROP;
 668           goto out;
 669         }
 670
 671       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 672       next0 = SNAT_IN2OUT_NEXT_DROP;
 673       goto out;
 674     }
 675
 676   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 677                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 678                     !icmp_is_error_message (icmp0)))
 679     {
 680       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 681       next0 = SNAT_IN2OUT_NEXT_DROP;
 682       goto out;
 683     }
 684
 685 out:
 686   *p_value = sm0;
 687 out2:
 688   *p_proto = key0.protocol;
 689   *p_dont_translate = dont_translate;
 690   return next0;
 691 }
 692
 693 static inline u32 icmp_in2out (snat_main_t *sm,
 694                                vlib_buffer_t * b0,
 695                                ip4_header_t * ip0,
 696                                icmp46_header_t * icmp0,
 697                                u32 sw_if_index0,
 698                                u32 rx_fib_index0,
 699                                vlib_node_runtime_t * node,
 700                                u32 next0,
 701                                u32 thread_index,
 702                                void *d,
 703                                void *e)
 704 {
 705   snat_session_key_t sm0;
 706   u8 protocol;
 707   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 708   ip4_header_t *inner_ip0;
 709   void *l4_header = 0;
 710   icmp46_header_t *inner_icmp0;
 711   u8 dont_translate;
 712   u32 new_addr0, old_addr0;
 713   u16 old_id0, new_id0;
 714   ip_csum_t sum0;
 715   u16 checksum0;
 716   u32 next0_tmp;
 717
 718   echo0 = (icmp_echo_header_t *)(icmp0+1);
 719
 720   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 721                                        &protocol, &sm0, &dont_translate, d, e);
 722   if (next0_tmp != ~0)
 723     next0 = next0_tmp;
 724   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 725     goto out;
 726
 727   sum0 = ip_incremental_checksum (0, icmp0,
 728                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 729   checksum0 = ~ip_csum_fold (sum0);
 730   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 731     {
 732       next0 = SNAT_IN2OUT_NEXT_DROP;
 733       goto out;
 734     }
 735
 736   old_addr0 = ip0->src_address.as_u32;
 737   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 738   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 739     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 740
 741   sum0 = ip0->checksum;
 742   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 743                          src_address /* changed member */);
 744   ip0->checksum = ip_csum_fold (sum0);
 745
 746   if (!icmp_is_error_message (icmp0))
 747     {
 748       new_id0 = sm0.port;
 749       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 750         {
 751           old_id0 = echo0->identifier;
 752           new_id0 = sm0.port;
 753           echo0->identifier = new_id0;
 754
 755           sum0 = icmp0->checksum;
 756           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 757                                  identifier);
 758           icmp0->checksum = ip_csum_fold (sum0);
 759         }
 760     }
 761   else
 762     {
 763       inner_ip0 = (ip4_header_t *)(echo0+1);
 764       l4_header = ip4_next_header (inner_ip0);
 765
 766       if (!ip4_header_checksum_is_valid (inner_ip0))
 767         {
 768           next0 = SNAT_IN2OUT_NEXT_DROP;
 769           goto out;
 770         }
 771
 772       old_addr0 = inner_ip0->dst_address.as_u32;
 773       inner_ip0->dst_address = sm0.addr;
 774       new_addr0 = inner_ip0->dst_address.as_u32;
 775
 776       sum0 = icmp0->checksum;
 777       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 778                              dst_address /* changed member */);
 779       icmp0->checksum = ip_csum_fold (sum0);
 780
 781       switch (protocol)
 782         {
 783           case SNAT_PROTOCOL_ICMP:
 784             inner_icmp0 = (icmp46_header_t*)l4_header;
 785             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 786
 787             old_id0 = inner_echo0->identifier;
 788             new_id0 = sm0.port;
 789             inner_echo0->identifier = new_id0;
 790
 791             sum0 = icmp0->checksum;
 792             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 793                                    identifier);
 794             icmp0->checksum = ip_csum_fold (sum0);
 795             break;
 796           case SNAT_PROTOCOL_UDP:
 797           case SNAT_PROTOCOL_TCP:
 798             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 799             new_id0 = sm0.port;
 800             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 801
 802             sum0 = icmp0->checksum;
 803             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 804                                    dst_port);
 805             icmp0->checksum = ip_csum_fold (sum0);
 806             break;
 807           default:
 808             ASSERT(0);
 809         }
 810     }
 811
 812 out:
 813   return next0;
 814 }
 815
 816 /**
 817  * @brief Hairpinning
 818  *
 819  * Hairpinning allows two endpoints on the internal side of the NAT to
 820  * communicate even if they only use each other's external IP addresses
 821  * and ports.
 822  *
 823  * @param sm     NAT main.
 824  * @param b0     Vlib buffer.
 825  * @param ip0    IP header.
 826  * @param udp0   UDP header.
 827  * @param tcp0   TCP header.
 828  * @param proto0 NAT protocol.
 829  */
 830 static inline void
 831 snat_hairpinning (snat_main_t *sm,
 832                   vlib_buffer_t * b0,
 833                   ip4_header_t * ip0,
 834                   udp_header_t * udp0,
 835                   tcp_header_t * tcp0,
 836                   u32 proto0)
 837 {
 838   snat_session_key_t key0, sm0;
 839   snat_worker_key_t k0;
 840   snat_session_t * s0;
 841   clib_bihash_kv_8_8_t kv0, value0;
 842   ip_csum_t sum0;
 843   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 844   u16 new_dst_port0, old_dst_port0;
 845
 846   key0.addr = ip0->dst_address;
 847   key0.port = udp0->dst_port;
 848   key0.protocol = proto0;
 849   key0.fib_index = sm->outside_fib_index;
 850   kv0.key = key0.as_u64;
 851
 852   if (sm->num_workers > 1)
 853     {
 854       k0.addr = ip0->dst_address;
 855       k0.port = udp0->dst_port;
 856       k0.fib_index = sm->outside_fib_index;
 857       kv0.key = k0.as_u64;
 858       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 859         return;
 860       else
 861         ti = value0.value;
 862     }
 863   else
 864     ti = sm->num_workers;
 865
 866   /* Check if destination is in active sessions */
 867   if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 868     {
 869       /* or static mappings */
 870       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 871         {
 872           new_dst_addr0 = sm0.addr.as_u32;
 873           new_dst_port0 = sm0.port;
 874           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 875         }
 876     }
 877   else
 878     {
 879       si = value0.value;
 880
 881       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 882       new_dst_addr0 = s0->in2out.addr.as_u32;
 883       new_dst_port0 = s0->in2out.port;
 884       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 885     }
 886
 887   /* Destination is behind the same NAT, use internal address and port */
 888   if (new_dst_addr0)
 889     {
 890       old_dst_addr0 = ip0->dst_address.as_u32;
 891       ip0->dst_address.as_u32 = new_dst_addr0;
 892       sum0 = ip0->checksum;
 893       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 894                              ip4_header_t, dst_address);
 895       ip0->checksum = ip_csum_fold (sum0);
 896
 897       old_dst_port0 = tcp0->dst;
 898       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 899         {
 900           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 901             {
 902               tcp0->dst = new_dst_port0;
 903               sum0 = tcp0->checksum;
 904               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 905                                      ip4_header_t, dst_address);
 906               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 907                                      ip4_header_t /* cheat */, length);
 908               tcp0->checksum = ip_csum_fold(sum0);
 909             }
 910           else
 911             {
 912               udp0->dst_port = new_dst_port0;
 913               udp0->checksum = 0;
 914             }
 915         }
 916       else
 917         {
 918           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 919             {
 920               sum0 = tcp0->checksum;
 921               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 922                                      ip4_header_t, dst_address);
 923               tcp0->checksum = ip_csum_fold(sum0);
 924             }
 925         }
 926     }
 927 }
 928
 929 static inline void
 930 snat_icmp_hairpinning (snat_main_t *sm,
 931                        vlib_buffer_t * b0,
 932                        ip4_header_t * ip0,
 933                        icmp46_header_t * icmp0)
 934 {
 935   snat_session_key_t key0, sm0;
 936   clib_bihash_kv_8_8_t kv0, value0;
 937   snat_worker_key_t k0;
 938   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 939   ip_csum_t sum0;
 940   snat_session_t *s0;
 941
 942   if (!icmp_is_error_message (icmp0))
 943     {
 944       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 945       u16 icmp_id0 = echo0->identifier;
 946       key0.addr = ip0->dst_address;
 947       key0.port = icmp_id0;
 948       key0.protocol = SNAT_PROTOCOL_ICMP;
 949       key0.fib_index = sm->outside_fib_index;
 950       kv0.key = key0.as_u64;
 951
 952       if (sm->num_workers > 1)
 953         {
 954           k0.addr = ip0->dst_address;
 955           k0.port = icmp_id0;
 956           k0.fib_index = sm->outside_fib_index;
 957           kv0.key = k0.as_u64;
 958           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 959             return;
 960           else
 961             ti = value0.value;
 962         }
 963       else
 964         ti = sm->num_workers;
 965
 966       /* Check if destination is in active sessions */
 967       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 968                                   &value0))
 969         {
 970           /* or static mappings */
 971           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 972             {
 973               new_dst_addr0 = sm0.addr.as_u32;
 974               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 975             }
 976         }
 977       else
 978         {
 979           si = value0.value;
 980
 981           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 982           new_dst_addr0 = s0->in2out.addr.as_u32;
 983           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 984           echo0->identifier = s0->in2out.port;
 985           sum0 = icmp0->checksum;
 986           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 987                                  icmp_echo_header_t, identifier);
 988           icmp0->checksum = ip_csum_fold (sum0);
 989         }
 990
 991       /* Destination is behind the same NAT, use internal address and port */
 992       if (new_dst_addr0)
 993         {
 994           old_dst_addr0 = ip0->dst_address.as_u32;
 995           ip0->dst_address.as_u32 = new_dst_addr0;
 996           sum0 = ip0->checksum;
 997           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 998                                  ip4_header_t, dst_address);
 999           ip0->checksum = ip_csum_fold (sum0);
1000         }
1001     }
1002
1003 }
1004
1005 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1006                                          vlib_buffer_t * b0,
1007                                          ip4_header_t * ip0,
1008                                          icmp46_header_t * icmp0,
1009                                          u32 sw_if_index0,
1010                                          u32 rx_fib_index0,
1011                                          vlib_node_runtime_t * node,
1012                                          u32 next0,
1013                                          f64 now,
1014                                          u32 thread_index,
1015                                          snat_session_t ** p_s0)
1016 {
1017   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1018                       next0, thread_index, p_s0, 0);
1019   snat_session_t * s0 = *p_s0;
1020   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1021     {
1022       /* Hairpinning */
1023       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1024         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1025       /* Accounting */
1026       s0->last_heard = now;
1027       s0->total_pkts++;
1028       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1029       /* Per-user LRU list maintenance for dynamic translations */
1030       if (!snat_is_session_static (s0))
1031         {
1032           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1033                              s0->per_user_index);
1034           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1035                               s0->per_user_list_head_index,
1036                               s0->per_user_index);
1037         }
1038     }
1039   return next0;
1040 }
1041 static inline void
1042 snat_hairpinning_unknown_proto (snat_main_t *sm,
1043                                 vlib_buffer_t * b,
1044                                 ip4_header_t * ip)
1045 {
1046   u32 old_addr, new_addr = 0, ti = 0;
1047   clib_bihash_kv_8_8_t kv, value;
1048   clib_bihash_kv_16_8_t s_kv, s_value;
1049   nat_ed_ses_key_t key;
1050   snat_session_key_t m_key;
1051   snat_worker_key_t w_key;
1052   snat_static_mapping_t *m;
1053   ip_csum_t sum;
1054   snat_session_t *s;
1055
1056   old_addr = ip->dst_address.as_u32;
1057   key.l_addr.as_u32 = ip->dst_address.as_u32;
1058   key.r_addr.as_u32 = ip->src_address.as_u32;
1059   key.fib_index = sm->outside_fib_index;
1060   key.proto = ip->protocol;
1061   key.rsvd = 0;
1062   key.l_port = 0;
1063   s_kv.key[0] = key.as_u64[0];
1064   s_kv.key[1] = key.as_u64[1];
1065   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1066     {
1067       m_key.addr = ip->dst_address;
1068       m_key.fib_index = sm->outside_fib_index;
1069       m_key.port = 0;
1070       m_key.protocol = 0;
1071       kv.key = m_key.as_u64;
1072       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1073         return;
1074
1075       m = pool_elt_at_index (sm->static_mappings, value.value);
1076       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1077         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1078       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1079     }
1080   else
1081     {
1082       if (sm->num_workers > 1)
1083         {
1084           w_key.addr = ip->dst_address;
1085           w_key.port = 0;
1086           w_key.fib_index = sm->outside_fib_index;
1087           kv.key = w_key.as_u64;
1088           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv, &value))
1089             return;
1090           else
1091             ti = value.value;
1092         }
1093       else
1094         ti = sm->num_workers;
1095
1096       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1097       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1098         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1099       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1100     }
1101   sum = ip->checksum;
1102   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1103   ip->checksum = ip_csum_fold (sum);
1104 }
1105
1106 static void
1107 snat_in2out_unknown_proto (snat_main_t *sm,
1108                            vlib_buffer_t * b,
1109                            ip4_header_t * ip,
1110                            u32 rx_fib_index,
1111                            u32 thread_index,
1112                            f64 now,
1113                            vlib_main_t * vm)
1114 {
1115   clib_bihash_kv_8_8_t kv, value;
1116   clib_bihash_kv_16_8_t s_kv, s_value;
1117   snat_static_mapping_t *m;
1118   snat_session_key_t m_key;
1119   u32 old_addr, new_addr = 0;
1120   ip_csum_t sum;
1121   snat_user_key_t u_key;
1122   snat_user_t *u;
1123   dlist_elt_t *head, *elt, *oldest;
1124   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1125   u32 elt_index, head_index, ses_index, oldest_index;
1126   snat_session_t * s;
1127   nat_ed_ses_key_t key;
1128   u32 address_index = ~0;
1129   int i;
1130   u8 is_sm = 0;
1131
1132   old_addr = ip->src_address.as_u32;
1133
1134   key.l_addr = ip->src_address;
1135   key.r_addr = ip->dst_address;
1136   key.fib_index = rx_fib_index;
1137   key.proto = ip->protocol;
1138   key.rsvd = 0;
1139   key.l_port = 0;
1140   s_kv.key[0] = key.as_u64[0];
1141   s_kv.key[1] = key.as_u64[1];
1142
1143   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1144     {
1145       s = pool_elt_at_index (tsm->sessions, s_value.value);
1146       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1147     }
1148   else
1149     {
1150       u_key.addr = ip->src_address;
1151       u_key.fib_index = rx_fib_index;
1152       kv.key = u_key.as_u64;
1153
1154       /* Ever heard of the "user" = src ip4 address before? */
1155       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1156         {
1157           /* no, make a new one */
1158           pool_get (tsm->users, u);
1159           memset (u, 0, sizeof (*u));
1160           u->addr = ip->src_address;
1161           u->fib_index = rx_fib_index;
1162
1163           pool_get (tsm->list_pool, head);
1164           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1165
1166           clib_dlist_init (tsm->list_pool,
1167                            u->sessions_per_user_list_head_index);
1168
1169           kv.value = u - tsm->users;
1170
1171           /* add user */
1172           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
1173         }
1174       else
1175         {
1176           u = pool_elt_at_index (tsm->users, value.value);
1177         }
1178
1179       m_key.addr = ip->src_address;
1180       m_key.port = 0;
1181       m_key.protocol = 0;
1182       m_key.fib_index = rx_fib_index;
1183       kv.key = m_key.as_u64;
1184
1185       /* Try to find static mapping first */
1186       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1187         {
1188           m = pool_elt_at_index (sm->static_mappings, value.value);
1189           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1190           is_sm = 1;
1191           goto create_ses;
1192         }
1193       /* Fallback to 3-tuple key */
1194       else
1195         {
1196           /* Choose same out address as for TCP/UDP session to same destination */
1197           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1198             {
1199               head_index = u->sessions_per_user_list_head_index;
1200               head = pool_elt_at_index (tsm->list_pool, head_index);
1201               elt_index = head->next;
1202               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1203               ses_index = elt->value;
1204               while (ses_index != ~0)
1205                 {
1206                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1207                   elt_index = elt->next;
1208                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1209                   ses_index = elt->value;
1210
1211                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1212                     {
1213                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1214                       address_index = s->outside_address_index;
1215
1216                       key.fib_index = sm->outside_fib_index;
1217                       key.l_addr.as_u32 = new_addr;
1218                       s_kv.key[0] = key.as_u64[0];
1219                       s_kv.key[1] = key.as_u64[1];
1220                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1221                         break;
1222
1223                       goto create_ses;
1224                     }
1225                 }
1226             }
1227           key.fib_index = sm->outside_fib_index;
1228           for (i = 0; i < vec_len (sm->addresses); i++)
1229             {
1230               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1231               s_kv.key[0] = key.as_u64[0];
1232               s_kv.key[1] = key.as_u64[1];
1233               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1234                 {
1235                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1236                   address_index = i;
1237                   goto create_ses;
1238                 }
1239             }
1240           return;
1241         }
1242
1243 create_ses:
1244       /* Over quota? Recycle the least recently used dynamic translation */
1245       if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1246         {
1247           /* Remove the oldest dynamic translation */
1248           do {
1249               oldest_index = clib_dlist_remove_head (
1250                 tsm->list_pool, u->sessions_per_user_list_head_index);
1251
1252               ASSERT (oldest_index != ~0);
1253
1254               /* add it back to the end of the LRU list */
1255               clib_dlist_addtail (tsm->list_pool,
1256                                   u->sessions_per_user_list_head_index,
1257                                   oldest_index);
1258               /* Get the list element */
1259               oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1260
1261               /* Get the session index from the list element */
1262               ses_index = oldest->value;
1263
1264               /* Get the session */
1265               s = pool_elt_at_index (tsm->sessions, ses_index);
1266           } while (snat_is_session_static (s));
1267
1268           if (snat_is_unk_proto_session (s))
1269             {
1270               /* Remove from lookup tables */
1271               key.l_addr = s->in2out.addr;
1272               key.r_addr = s->ext_host_addr;
1273               key.fib_index = s->in2out.fib_index;
1274               key.proto = s->in2out.port;
1275               s_kv.key[0] = key.as_u64[0];
1276               s_kv.key[1] = key.as_u64[1];
1277               if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 0))
1278                 clib_warning ("in2out key del failed");
1279
1280               key.l_addr = s->out2in.addr;
1281               key.fib_index = s->out2in.fib_index;
1282               s_kv.key[0] = key.as_u64[0];
1283               s_kv.key[1] = key.as_u64[1];
1284               if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 0))
1285                 clib_warning ("out2in key del failed");
1286             }
1287           else
1288             {
1289               /* log NAT event */
1290               snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1291                                                   s->out2in.addr.as_u32,
1292                                                   s->in2out.protocol,
1293                                                   s->in2out.port,
1294                                                   s->out2in.port,
1295                                                   s->in2out.fib_index);
1296
1297               snat_free_outside_address_and_port (sm, thread_index, &s->out2in,
1298                                                   s->outside_address_index);
1299
1300               /* Remove in2out, out2in keys */
1301               kv.key = s->in2out.as_u64;
1302               if (clib_bihash_add_del_8_8 (
1303                     &sm->per_thread_data[thread_index].in2out, &kv, 0))
1304                 clib_warning ("in2out key del failed");
1305               kv.key = s->out2in.as_u64;
1306               if (clib_bihash_add_del_8_8 (
1307                     &sm->per_thread_data[thread_index].out2in, &kv, 0))
1308                 clib_warning ("out2in key del failed");
1309             }
1310         }
1311       else
1312         {
1313           /* Create a new session */
1314           pool_get (tsm->sessions, s);
1315           memset (s, 0, sizeof (*s));
1316
1317           /* Create list elts */
1318           pool_get (tsm->list_pool, elt);
1319           clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1320           elt->value = s - tsm->sessions;
1321           s->per_user_index = elt - tsm->list_pool;
1322           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1323           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1324                               s->per_user_index);
1325         }
1326
1327       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1328       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1329       s->outside_address_index = address_index;
1330       s->out2in.addr.as_u32 = new_addr;
1331       s->out2in.fib_index = sm->outside_fib_index;
1332       s->in2out.addr.as_u32 = old_addr;
1333       s->in2out.fib_index = rx_fib_index;
1334       s->in2out.port = s->out2in.port = ip->protocol;
1335       if (is_sm)
1336         {
1337           u->nstaticsessions++;
1338           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1339         }
1340       else
1341         {
1342           u->nsessions++;
1343         }
1344
1345       /* Add to lookup tables */
1346       key.l_addr.as_u32 = old_addr;
1347       key.r_addr = ip->dst_address;
1348       key.proto = ip->protocol;
1349       key.fib_index = rx_fib_index;
1350       s_kv.key[0] = key.as_u64[0];
1351       s_kv.key[1] = key.as_u64[1];
1352       s_kv.value = s - tsm->sessions;
1353       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1354         clib_warning ("in2out key add failed");
1355
1356       key.l_addr.as_u32 = new_addr;
1357       key.fib_index = sm->outside_fib_index;
1358       s_kv.key[0] = key.as_u64[0];
1359       s_kv.key[1] = key.as_u64[1];
1360       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1361         clib_warning ("out2in key add failed");
1362   }
1363
1364   /* Update IP checksum */
1365   sum = ip->checksum;
1366   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1367   ip->checksum = ip_csum_fold (sum);
1368
1369   /* Accounting */
1370   s->last_heard = now;
1371   s->total_pkts++;
1372   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1373   /* Per-user LRU list maintenance */
1374   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1375   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1376                       s->per_user_index);
1377
1378   /* Hairpinning */
1379   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1380     snat_hairpinning_unknown_proto(sm, b, ip);
1381
1382   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1383     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1384 }
1385
1386 static snat_session_t *
1387 snat_in2out_lb (snat_main_t *sm,
1388                 vlib_buffer_t * b,
1389                 ip4_header_t * ip,
1390                 u32 rx_fib_index,
1391                 u32 thread_index,
1392                 f64 now,
1393                 vlib_main_t * vm)
1394 {
1395   nat_ed_ses_key_t key;
1396   clib_bihash_kv_16_8_t s_kv, s_value;
1397   udp_header_t *udp = ip4_next_header (ip);
1398   tcp_header_t *tcp = (tcp_header_t *) udp;
1399   snat_session_t *s = 0;
1400   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1401   u32 old_addr, new_addr;
1402   u16 new_port, old_port;
1403   ip_csum_t sum;
1404   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1405   snat_session_key_t e_key, l_key;
1406   clib_bihash_kv_8_8_t kv, value;
1407   snat_user_key_t u_key;
1408   snat_user_t *u;
1409   dlist_elt_t *head, *elt;
1410
1411   old_addr = ip->src_address.as_u32;
1412
1413   key.l_addr = ip->src_address;
1414   key.r_addr = ip->dst_address;
1415   key.fib_index = rx_fib_index;
1416   key.proto = ip->protocol;
1417   key.rsvd = 0;
1418   key.l_port = udp->src_port;
1419   s_kv.key[0] = key.as_u64[0];
1420   s_kv.key[1] = key.as_u64[1];
1421
1422   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1423     {
1424       s = pool_elt_at_index (tsm->sessions, s_value.value);
1425     }
1426   else
1427     {
1428       l_key.addr = ip->src_address;
1429       l_key.port = udp->src_port;
1430       l_key.protocol = proto;
1431       l_key.fib_index = rx_fib_index;
1432       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0))
1433         return 0;
1434
1435       u_key.addr = ip->src_address;
1436       u_key.fib_index = rx_fib_index;
1437       kv.key = u_key.as_u64;
1438
1439       /* Ever heard of the "user" = src ip4 address before? */
1440       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1441         {
1442           /* no, make a new one */
1443           pool_get (tsm->users, u);
1444           memset (u, 0, sizeof (*u));
1445           u->addr = ip->src_address;
1446           u->fib_index = rx_fib_index;
1447
1448           pool_get (tsm->list_pool, head);
1449           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1450
1451           clib_dlist_init (tsm->list_pool,
1452                            u->sessions_per_user_list_head_index);
1453
1454           kv.value = u - tsm->users;
1455
1456           /* add user */
1457           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
1458             clib_warning ("user key add failed");
1459         }
1460       else
1461         {
1462           u = pool_elt_at_index (tsm->users, value.value);
1463         }
1464
1465       /* Create a new session */
1466       pool_get (tsm->sessions, s);
1467       memset (s, 0, sizeof (*s));
1468
1469       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1470       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1471       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1472       s->outside_address_index = ~0;
1473       s->in2out = l_key;
1474       s->out2in = e_key;
1475       u->nstaticsessions++;
1476
1477       /* Create list elts */
1478       pool_get (tsm->list_pool, elt);
1479       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1480       elt->value = s - tsm->sessions;
1481       s->per_user_index = elt - tsm->list_pool;
1482       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1483       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1484                           s->per_user_index);
1485
1486       /* Add to lookup tables */
1487       s_kv.value = s - tsm->sessions;
1488       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1489         clib_warning ("in2out-ed key add failed");
1490
1491       key.l_addr = e_key.addr;
1492       key.fib_index = e_key.fib_index;
1493       key.l_port = e_key.port;
1494       s_kv.key[0] = key.as_u64[0];
1495       s_kv.key[1] = key.as_u64[1];
1496       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1497         clib_warning ("out2in-ed key add failed");
1498     }
1499
1500   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1501
1502   /* Update IP checksum */
1503   sum = ip->checksum;
1504   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1505   ip->checksum = ip_csum_fold (sum);
1506
1507   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1508     {
1509       old_port = tcp->src_port;
1510       tcp->src_port = s->out2in.port;
1511       new_port = tcp->src_port;
1512
1513       sum = tcp->checksum;
1514       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1515       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1516       tcp->checksum = ip_csum_fold(sum);
1517     }
1518   else
1519     {
1520       udp->src_port = s->out2in.port;
1521       udp->checksum = 0;
1522     }
1523
1524   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1525     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1526
1527   /* Accounting */
1528   s->last_heard = now;
1529   s->total_pkts++;
1530   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1531   return s;
1532 }
1533
1534 static inline uword
1535 snat_in2out_node_fn_inline (vlib_main_t * vm,
1536                             vlib_node_runtime_t * node,
1537                             vlib_frame_t * frame, int is_slow_path,
1538                             int is_output_feature)
1539 {
1540   u32 n_left_from, * from, * to_next;
1541   snat_in2out_next_t next_index;
1542   u32 pkts_processed = 0;
1543   snat_main_t * sm = &snat_main;
1544   f64 now = vlib_time_now (vm);
1545   u32 stats_node_index;
1546   u32 thread_index = vlib_get_thread_index ();
1547
1548   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1549     snat_in2out_node.index;
1550
1551   from = vlib_frame_vector_args (frame);
1552   n_left_from = frame->n_vectors;
1553   next_index = node->cached_next_index;
1554
1555   while (n_left_from > 0)
1556     {
1557       u32 n_left_to_next;
1558
1559       vlib_get_next_frame (vm, node, next_index,
1560                            to_next, n_left_to_next);
1561
1562       while (n_left_from >= 4 && n_left_to_next >= 2)
1563         {
1564           u32 bi0, bi1;
1565           vlib_buffer_t * b0, * b1;
1566           u32 next0, next1;
1567           u32 sw_if_index0, sw_if_index1;
1568           ip4_header_t * ip0, * ip1;
1569           ip_csum_t sum0, sum1;
1570           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1571           u16 old_port0, new_port0, old_port1, new_port1;
1572           udp_header_t * udp0, * udp1;
1573           tcp_header_t * tcp0, * tcp1;
1574           icmp46_header_t * icmp0, * icmp1;
1575           snat_session_key_t key0, key1;
1576           u32 rx_fib_index0, rx_fib_index1;
1577           u32 proto0, proto1;
1578           snat_session_t * s0 = 0, * s1 = 0;
1579           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1580           u32 iph_offset0 = 0, iph_offset1 = 0;
1581
1582           /* Prefetch next iteration. */
1583           {
1584             vlib_buffer_t * p2, * p3;
1585
1586             p2 = vlib_get_buffer (vm, from[2]);
1587             p3 = vlib_get_buffer (vm, from[3]);
1588
1589             vlib_prefetch_buffer_header (p2, LOAD);
1590             vlib_prefetch_buffer_header (p3, LOAD);
1591
1592             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1593             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1594           }
1595
1596           /* speculatively enqueue b0 and b1 to the current next frame */
1597           to_next[0] = bi0 = from[0];
1598           to_next[1] = bi1 = from[1];
1599           from += 2;
1600           to_next += 2;
1601           n_left_from -= 2;
1602           n_left_to_next -= 2;
1603
1604           b0 = vlib_get_buffer (vm, bi0);
1605           b1 = vlib_get_buffer (vm, bi1);
1606
1607           if (is_output_feature)
1608             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1609
1610           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1611                  iph_offset0);
1612
1613           udp0 = ip4_next_header (ip0);
1614           tcp0 = (tcp_header_t *) udp0;
1615           icmp0 = (icmp46_header_t *) udp0;
1616
1617           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1618           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1619                                    sw_if_index0);
1620
1621           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1622
1623           if (PREDICT_FALSE(ip0->ttl == 1))
1624             {
1625               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1626               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1627                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1628                                            0);
1629               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1630               goto trace00;
1631             }
1632
1633           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1634
1635           /* Next configured feature, probably ip4-lookup */
1636           if (is_slow_path)
1637             {
1638               if (PREDICT_FALSE (proto0 == ~0))
1639                 {
1640                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1641                                              thread_index, now, vm);
1642                   goto trace00;
1643                 }
1644
1645               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1646                 {
1647                   next0 = icmp_in2out_slow_path
1648                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1649                      node, next0, now, thread_index, &s0);
1650                   goto trace00;
1651                 }
1652             }
1653           else
1654             {
1655               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1656                 {
1657                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1658                   goto trace00;
1659                 }
1660             }
1661
1662           key0.addr = ip0->src_address;
1663           key0.port = udp0->src_port;
1664           key0.protocol = proto0;
1665           key0.fib_index = rx_fib_index0;
1666
1667           kv0.key = key0.as_u64;
1668
1669           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1670               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1671             {
1672               if (is_slow_path)
1673                 {
1674                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1675                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1676                     goto trace00;
1677
1678                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1679                                      &s0, node, next0, thread_index);
1680                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1681                     goto trace00;
1682                 }
1683               else
1684                 {
1685                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1686                   goto trace00;
1687                 }
1688             }
1689           else
1690             {
1691               if (PREDICT_FALSE (value0.value == ~0ULL))
1692                 {
1693                   if (is_slow_path)
1694                     {
1695                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1696                                      now, vm);
1697                       goto trace00;
1698                     }
1699                   else
1700                     {
1701                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1702                       goto trace00;
1703                     }
1704                 }
1705               else
1706                 {
1707                   s0 = pool_elt_at_index (
1708                     sm->per_thread_data[thread_index].sessions,
1709                     value0.value);
1710                 }
1711             }
1712
1713           old_addr0 = ip0->src_address.as_u32;
1714           ip0->src_address = s0->out2in.addr;
1715           new_addr0 = ip0->src_address.as_u32;
1716           if (!is_output_feature)
1717             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1718
1719           sum0 = ip0->checksum;
1720           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1721                                  ip4_header_t,
1722                                  src_address /* changed member */);
1723           ip0->checksum = ip_csum_fold (sum0);
1724
1725           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1726             {
1727               old_port0 = tcp0->src_port;
1728               tcp0->src_port = s0->out2in.port;
1729               new_port0 = tcp0->src_port;
1730
1731               sum0 = tcp0->checksum;
1732               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1733                                      ip4_header_t,
1734                                      dst_address /* changed member */);
1735               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1736                                      ip4_header_t /* cheat */,
1737                                      length /* changed member */);
1738               tcp0->checksum = ip_csum_fold(sum0);
1739             }
1740           else
1741             {
1742               old_port0 = udp0->src_port;
1743               udp0->src_port = s0->out2in.port;
1744               udp0->checksum = 0;
1745             }
1746
1747           /* Hairpinning */
1748           if (!is_output_feature)
1749             snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1750
1751           /* Accounting */
1752           s0->last_heard = now;
1753           s0->total_pkts++;
1754           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1755           /* Per-user LRU list maintenance for dynamic translation */
1756           if (!snat_is_session_static (s0))
1757             {
1758               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1759                                  s0->per_user_index);
1760               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1761                                   s0->per_user_list_head_index,
1762                                   s0->per_user_index);
1763             }
1764         trace00:
1765
1766           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1767                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1768             {
1769               snat_in2out_trace_t *t =
1770                  vlib_add_trace (vm, node, b0, sizeof (*t));
1771               t->is_slow_path = is_slow_path;
1772               t->sw_if_index = sw_if_index0;
1773               t->next_index = next0;
1774                   t->session_index = ~0;
1775               if (s0)
1776                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1777             }
1778
1779           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1780
1781           if (is_output_feature)
1782             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1783
1784           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1785                  iph_offset1);
1786
1787           udp1 = ip4_next_header (ip1);
1788           tcp1 = (tcp_header_t *) udp1;
1789           icmp1 = (icmp46_header_t *) udp1;
1790
1791           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1792           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1793                                    sw_if_index1);
1794
1795           if (PREDICT_FALSE(ip1->ttl == 1))
1796             {
1797               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1798               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1799                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1800                                            0);
1801               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1802               goto trace01;
1803             }
1804
1805           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1806
1807           /* Next configured feature, probably ip4-lookup */
1808           if (is_slow_path)
1809             {
1810               if (PREDICT_FALSE (proto1 == ~0))
1811                 {
1812                   snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1813                                              thread_index, now, vm);
1814                   goto trace01;
1815                 }
1816
1817               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1818                 {
1819                   next1 = icmp_in2out_slow_path
1820                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1821                      next1, now, thread_index, &s1);
1822                   goto trace01;
1823                 }
1824             }
1825           else
1826             {
1827               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1828                 {
1829                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1830                   goto trace01;
1831                 }
1832             }
1833
1834           key1.addr = ip1->src_address;
1835           key1.port = udp1->src_port;
1836           key1.protocol = proto1;
1837           key1.fib_index = rx_fib_index1;
1838
1839           kv1.key = key1.as_u64;
1840
1841             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1842                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1843             {
1844               if (is_slow_path)
1845                 {
1846                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1847                       ip1, proto1, rx_fib_index1, thread_index)) && !is_output_feature)
1848                     goto trace01;
1849
1850                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1851                                      &s1, node, next1, thread_index);
1852                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1853                     goto trace01;
1854                 }
1855               else
1856                 {
1857                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1858                   goto trace01;
1859                 }
1860             }
1861           else
1862             {
1863               if (PREDICT_FALSE (value1.value == ~0ULL))
1864                 {
1865                   if (is_slow_path)
1866                     {
1867                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1868                                      now, vm);
1869                       goto trace01;
1870                     }
1871                   else
1872                     {
1873                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1874                       goto trace01;
1875                     }
1876                 }
1877               else
1878                 {
1879                   s1 = pool_elt_at_index (
1880                     sm->per_thread_data[thread_index].sessions,
1881                     value1.value);
1882                 }
1883             }
1884
1885           old_addr1 = ip1->src_address.as_u32;
1886           ip1->src_address = s1->out2in.addr;
1887           new_addr1 = ip1->src_address.as_u32;
1888           if (!is_output_feature)
1889             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1890
1891           sum1 = ip1->checksum;
1892           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1893                                  ip4_header_t,
1894                                  src_address /* changed member */);
1895           ip1->checksum = ip_csum_fold (sum1);
1896
1897           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1898             {
1899               old_port1 = tcp1->src_port;
1900               tcp1->src_port = s1->out2in.port;
1901               new_port1 = tcp1->src_port;
1902
1903               sum1 = tcp1->checksum;
1904               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1905                                      ip4_header_t,
1906                                      dst_address /* changed member */);
1907               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1908                                      ip4_header_t /* cheat */,
1909                                      length /* changed member */);
1910               tcp1->checksum = ip_csum_fold(sum1);
1911             }
1912           else
1913             {
1914               old_port1 = udp1->src_port;
1915               udp1->src_port = s1->out2in.port;
1916               udp1->checksum = 0;
1917             }
1918
1919           /* Hairpinning */
1920           if (!is_output_feature)
1921             snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1922
1923           /* Accounting */
1924           s1->last_heard = now;
1925           s1->total_pkts++;
1926           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1927           /* Per-user LRU list maintenance for dynamic translation */
1928           if (!snat_is_session_static (s1))
1929             {
1930               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1931                                  s1->per_user_index);
1932               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1933                                   s1->per_user_list_head_index,
1934                                   s1->per_user_index);
1935             }
1936         trace01:
1937
1938           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1939                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1940             {
1941               snat_in2out_trace_t *t =
1942                  vlib_add_trace (vm, node, b1, sizeof (*t));
1943               t->sw_if_index = sw_if_index1;
1944               t->next_index = next1;
1945               t->session_index = ~0;
1946               if (s1)
1947                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1948             }
1949
1950           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1951
1952           /* verify speculative enqueues, maybe switch current next frame */
1953           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1954                                            to_next, n_left_to_next,
1955                                            bi0, bi1, next0, next1);
1956         }
1957
1958       while (n_left_from > 0 && n_left_to_next > 0)
1959         {
1960           u32 bi0;
1961           vlib_buffer_t * b0;
1962           u32 next0;
1963           u32 sw_if_index0;
1964           ip4_header_t * ip0;
1965           ip_csum_t sum0;
1966           u32 new_addr0, old_addr0;
1967           u16 old_port0, new_port0;
1968           udp_header_t * udp0;
1969           tcp_header_t * tcp0;
1970           icmp46_header_t * icmp0;
1971           snat_session_key_t key0;
1972           u32 rx_fib_index0;
1973           u32 proto0;
1974           snat_session_t * s0 = 0;
1975           clib_bihash_kv_8_8_t kv0, value0;
1976           u32 iph_offset0 = 0;
1977
1978           /* speculatively enqueue b0 to the current next frame */
1979           bi0 = from[0];
1980           to_next[0] = bi0;
1981           from += 1;
1982           to_next += 1;
1983           n_left_from -= 1;
1984           n_left_to_next -= 1;
1985
1986           b0 = vlib_get_buffer (vm, bi0);
1987           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1988
1989           if (is_output_feature)
1990             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1991
1992           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1993                  iph_offset0);
1994
1995           udp0 = ip4_next_header (ip0);
1996           tcp0 = (tcp_header_t *) udp0;
1997           icmp0 = (icmp46_header_t *) udp0;
1998
1999           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2000           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
2001                                    sw_if_index0);
2002
2003           if (PREDICT_FALSE(ip0->ttl == 1))
2004             {
2005               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2006               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2007                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2008                                            0);
2009               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2010               goto trace0;
2011             }
2012
2013           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2014
2015           /* Next configured feature, probably ip4-lookup */
2016           if (is_slow_path)
2017             {
2018               if (PREDICT_FALSE (proto0 == ~0))
2019                 {
2020                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2021                                              thread_index, now, vm);
2022                   goto trace0;
2023                 }
2024
2025               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2026                 {
2027                   next0 = icmp_in2out_slow_path
2028                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2029                      next0, now, thread_index, &s0);
2030                   goto trace0;
2031                 }
2032             }
2033           else
2034             {
2035               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2036                 {
2037                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2038                   goto trace0;
2039                 }
2040             }
2041
2042           key0.addr = ip0->src_address;
2043           key0.port = udp0->src_port;
2044           key0.protocol = proto0;
2045           key0.fib_index = rx_fib_index0;
2046
2047           kv0.key = key0.as_u64;
2048
2049           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2050                                       &kv0, &value0))
2051             {
2052               if (is_slow_path)
2053                 {
2054                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2055                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
2056                     goto trace0;
2057
2058                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2059                                      &s0, node, next0, thread_index);
2060
2061                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2062                     goto trace0;
2063                 }
2064               else
2065                 {
2066                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2067                   goto trace0;
2068                 }
2069             }
2070           else
2071             {
2072               if (PREDICT_FALSE (value0.value == ~0ULL))
2073                 {
2074                   if (is_slow_path)
2075                     {
2076                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0, thread_index,
2077                                      now, vm);
2078                       goto trace0;
2079                     }
2080                   else
2081                     {
2082                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2083                       goto trace0;
2084                     }
2085                 }
2086               else
2087                 {
2088                   s0 = pool_elt_at_index (
2089                     sm->per_thread_data[thread_index].sessions,
2090                     value0.value);
2091                 }
2092             }
2093
2094           old_addr0 = ip0->src_address.as_u32;
2095           ip0->src_address = s0->out2in.addr;
2096           new_addr0 = ip0->src_address.as_u32;
2097           if (!is_output_feature)
2098             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2099
2100           sum0 = ip0->checksum;
2101           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2102                                  ip4_header_t,
2103                                  src_address /* changed member */);
2104           ip0->checksum = ip_csum_fold (sum0);
2105
2106           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2107             {
2108               old_port0 = tcp0->src_port;
2109               tcp0->src_port = s0->out2in.port;
2110               new_port0 = tcp0->src_port;
2111
2112               sum0 = tcp0->checksum;
2113               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2114                                      ip4_header_t,
2115                                      dst_address /* changed member */);
2116               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2117                                      ip4_header_t /* cheat */,
2118                                      length /* changed member */);
2119               tcp0->checksum = ip_csum_fold(sum0);
2120             }
2121           else
2122             {
2123               old_port0 = udp0->src_port;
2124               udp0->src_port = s0->out2in.port;
2125               udp0->checksum = 0;
2126             }
2127
2128           /* Hairpinning */
2129           if (!is_output_feature)
2130             snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2131
2132           /* Accounting */
2133           s0->last_heard = now;
2134           s0->total_pkts++;
2135           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2136           /* Per-user LRU list maintenance for dynamic translation */
2137           if (!snat_is_session_static (s0))
2138             {
2139               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2140                                  s0->per_user_index);
2141               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2142                                   s0->per_user_list_head_index,
2143                                   s0->per_user_index);
2144             }
2145
2146         trace0:
2147           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2148                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2149             {
2150               snat_in2out_trace_t *t =
2151                  vlib_add_trace (vm, node, b0, sizeof (*t));
2152               t->is_slow_path = is_slow_path;
2153               t->sw_if_index = sw_if_index0;
2154               t->next_index = next0;
2155                   t->session_index = ~0;
2156               if (s0)
2157                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2158             }
2159
2160           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2161
2162           /* verify speculative enqueue, maybe switch current next frame */
2163           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2164                                            to_next, n_left_to_next,
2165                                            bi0, next0);
2166         }
2167
2168       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2169     }
2170
2171   vlib_node_increment_counter (vm, stats_node_index,
2172                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2173                                pkts_processed);
2174   return frame->n_vectors;
2175 }
2176
2177 static uword
2178 snat_in2out_fast_path_fn (vlib_main_t * vm,
2179                           vlib_node_runtime_t * node,
2180                           vlib_frame_t * frame)
2181 {
2182   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2183 }
2184
2185 VLIB_REGISTER_NODE (snat_in2out_node) = {
2186   .function = snat_in2out_fast_path_fn,
2187   .name = "nat44-in2out",
2188   .vector_size = sizeof (u32),
2189   .format_trace = format_snat_in2out_trace,
2190   .type = VLIB_NODE_TYPE_INTERNAL,
2191
2192   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2193   .error_strings = snat_in2out_error_strings,
2194
2195   .runtime_data_bytes = sizeof (snat_runtime_t),
2196
2197   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2198
2199   /* edit / add dispositions here */
2200   .next_nodes = {
2201     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2202     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2203     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2204     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2205   },
2206 };
2207
2208 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2209
2210 static uword
2211 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2212                                  vlib_node_runtime_t * node,
2213                                  vlib_frame_t * frame)
2214 {
2215   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2216 }
2217
2218 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2219   .function = snat_in2out_output_fast_path_fn,
2220   .name = "nat44-in2out-output",
2221   .vector_size = sizeof (u32),
2222   .format_trace = format_snat_in2out_trace,
2223   .type = VLIB_NODE_TYPE_INTERNAL,
2224
2225   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2226   .error_strings = snat_in2out_error_strings,
2227
2228   .runtime_data_bytes = sizeof (snat_runtime_t),
2229
2230   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2231
2232   /* edit / add dispositions here */
2233   .next_nodes = {
2234     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2235     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2236     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2237     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2238   },
2239 };
2240
2241 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2242                               snat_in2out_output_fast_path_fn);
2243
2244 static uword
2245 snat_in2out_slow_path_fn (vlib_main_t * vm,
2246                           vlib_node_runtime_t * node,
2247                           vlib_frame_t * frame)
2248 {
2249   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2250 }
2251
2252 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2253   .function = snat_in2out_slow_path_fn,
2254   .name = "nat44-in2out-slowpath",
2255   .vector_size = sizeof (u32),
2256   .format_trace = format_snat_in2out_trace,
2257   .type = VLIB_NODE_TYPE_INTERNAL,
2258
2259   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2260   .error_strings = snat_in2out_error_strings,
2261
2262   .runtime_data_bytes = sizeof (snat_runtime_t),
2263
2264   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2265
2266   /* edit / add dispositions here */
2267   .next_nodes = {
2268     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2269     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2270     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2271     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2272   },
2273 };
2274
2275 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2276                               snat_in2out_slow_path_fn);
2277
2278 static uword
2279 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2280                                  vlib_node_runtime_t * node,
2281                                  vlib_frame_t * frame)
2282 {
2283   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2284 }
2285
2286 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2287   .function = snat_in2out_output_slow_path_fn,
2288   .name = "nat44-in2out-output-slowpath",
2289   .vector_size = sizeof (u32),
2290   .format_trace = format_snat_in2out_trace,
2291   .type = VLIB_NODE_TYPE_INTERNAL,
2292
2293   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2294   .error_strings = snat_in2out_error_strings,
2295
2296   .runtime_data_bytes = sizeof (snat_runtime_t),
2297
2298   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2299
2300   /* edit / add dispositions here */
2301   .next_nodes = {
2302     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2303     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2304     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2305     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2306   },
2307 };
2308
2309 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2310                               snat_in2out_output_slow_path_fn);
2311
2312 /**************************/
2313 /*** deterministic mode ***/
2314 /**************************/
2315 static uword
2316 snat_det_in2out_node_fn (vlib_main_t * vm,
2317                          vlib_node_runtime_t * node,
2318                          vlib_frame_t * frame)
2319 {
2320   u32 n_left_from, * from, * to_next;
2321   snat_in2out_next_t next_index;
2322   u32 pkts_processed = 0;
2323   snat_main_t * sm = &snat_main;
2324   u32 now = (u32) vlib_time_now (vm);
2325   u32 thread_index = vlib_get_thread_index ();
2326
2327   from = vlib_frame_vector_args (frame);
2328   n_left_from = frame->n_vectors;
2329   next_index = node->cached_next_index;
2330
2331   while (n_left_from > 0)
2332     {
2333       u32 n_left_to_next;
2334
2335       vlib_get_next_frame (vm, node, next_index,
2336                            to_next, n_left_to_next);
2337
2338       while (n_left_from >= 4 && n_left_to_next >= 2)
2339         {
2340           u32 bi0, bi1;
2341           vlib_buffer_t * b0, * b1;
2342           u32 next0, next1;
2343           u32 sw_if_index0, sw_if_index1;
2344           ip4_header_t * ip0, * ip1;
2345           ip_csum_t sum0, sum1;
2346           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2347           u16 old_port0, new_port0, lo_port0, i0;
2348           u16 old_port1, new_port1, lo_port1, i1;
2349           udp_header_t * udp0, * udp1;
2350           tcp_header_t * tcp0, * tcp1;
2351           u32 proto0, proto1;
2352           snat_det_out_key_t key0, key1;
2353           snat_det_map_t * dm0, * dm1;
2354           snat_det_session_t * ses0 = 0, * ses1 = 0;
2355           u32 rx_fib_index0, rx_fib_index1;
2356           icmp46_header_t * icmp0, * icmp1;
2357
2358           /* Prefetch next iteration. */
2359           {
2360             vlib_buffer_t * p2, * p3;
2361
2362             p2 = vlib_get_buffer (vm, from[2]);
2363             p3 = vlib_get_buffer (vm, from[3]);
2364
2365             vlib_prefetch_buffer_header (p2, LOAD);
2366             vlib_prefetch_buffer_header (p3, LOAD);
2367
2368             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2369             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2370           }
2371
2372           /* speculatively enqueue b0 and b1 to the current next frame */
2373           to_next[0] = bi0 = from[0];
2374           to_next[1] = bi1 = from[1];
2375           from += 2;
2376           to_next += 2;
2377           n_left_from -= 2;
2378           n_left_to_next -= 2;
2379
2380           b0 = vlib_get_buffer (vm, bi0);
2381           b1 = vlib_get_buffer (vm, bi1);
2382
2383           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2384           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2385
2386           ip0 = vlib_buffer_get_current (b0);
2387           udp0 = ip4_next_header (ip0);
2388           tcp0 = (tcp_header_t *) udp0;
2389
2390           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2391
2392           if (PREDICT_FALSE(ip0->ttl == 1))
2393             {
2394               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2395               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2396                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2397                                            0);
2398               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2399               goto trace0;
2400             }
2401
2402           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2403
2404           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2405             {
2406               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2407               icmp0 = (icmp46_header_t *) udp0;
2408
2409               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2410                                   rx_fib_index0, node, next0, thread_index,
2411                                   &ses0, &dm0);
2412               goto trace0;
2413             }
2414
2415           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2416           if (PREDICT_FALSE(!dm0))
2417             {
2418               clib_warning("no match for internal host %U",
2419                            format_ip4_address, &ip0->src_address);
2420               next0 = SNAT_IN2OUT_NEXT_DROP;
2421               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2422               goto trace0;
2423             }
2424
2425           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2426
2427           key0.ext_host_addr = ip0->dst_address;
2428           key0.ext_host_port = tcp0->dst;
2429
2430           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2431           if (PREDICT_FALSE(!ses0))
2432             {
2433               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2434                 {
2435                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2436                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2437
2438                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2439                     continue;
2440
2441                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2442                   break;
2443                 }
2444               if (PREDICT_FALSE(!ses0))
2445                 {
2446                   /* too many sessions for user, send ICMP error packet */
2447
2448                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2449                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2450                                                ICMP4_destination_unreachable_destination_unreachable_host,
2451                                                0);
2452                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2453                   goto trace0;
2454                 }
2455             }
2456
2457           new_port0 = ses0->out.out_port;
2458
2459           old_addr0.as_u32 = ip0->src_address.as_u32;
2460           ip0->src_address.as_u32 = new_addr0.as_u32;
2461           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2462
2463           sum0 = ip0->checksum;
2464           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2465                                  ip4_header_t,
2466                                  src_address /* changed member */);
2467           ip0->checksum = ip_csum_fold (sum0);
2468
2469           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2470             {
2471               if (tcp0->flags & TCP_FLAG_SYN)
2472                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2473               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2474                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2475               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2476                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2477               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2478                 snat_det_ses_close(dm0, ses0);
2479               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2480                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2481               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2482                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2483
2484               old_port0 = tcp0->src;
2485               tcp0->src = new_port0;
2486
2487               sum0 = tcp0->checksum;
2488               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2489                                      ip4_header_t,
2490                                      dst_address /* changed member */);
2491               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2492                                      ip4_header_t /* cheat */,
2493                                      length /* changed member */);
2494               tcp0->checksum = ip_csum_fold(sum0);
2495             }
2496           else
2497             {
2498               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2499               old_port0 = udp0->src_port;
2500               udp0->src_port = new_port0;
2501               udp0->checksum = 0;
2502             }
2503
2504           switch(ses0->state)
2505             {
2506             case SNAT_SESSION_UDP_ACTIVE:
2507                 ses0->expire = now + sm->udp_timeout;
2508                 break;
2509             case SNAT_SESSION_TCP_SYN_SENT:
2510             case SNAT_SESSION_TCP_FIN_WAIT:
2511             case SNAT_SESSION_TCP_CLOSE_WAIT:
2512             case SNAT_SESSION_TCP_LAST_ACK:
2513                 ses0->expire = now + sm->tcp_transitory_timeout;
2514                 break;
2515             case SNAT_SESSION_TCP_ESTABLISHED:
2516                 ses0->expire = now + sm->tcp_established_timeout;
2517                 break;
2518             }
2519
2520         trace0:
2521           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2522                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2523             {
2524               snat_in2out_trace_t *t =
2525                  vlib_add_trace (vm, node, b0, sizeof (*t));
2526               t->is_slow_path = 0;
2527               t->sw_if_index = sw_if_index0;
2528               t->next_index = next0;
2529               t->session_index = ~0;
2530               if (ses0)
2531                 t->session_index = ses0 - dm0->sessions;
2532             }
2533
2534           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2535
2536           ip1 = vlib_buffer_get_current (b1);
2537           udp1 = ip4_next_header (ip1);
2538           tcp1 = (tcp_header_t *) udp1;
2539
2540           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2541
2542           if (PREDICT_FALSE(ip1->ttl == 1))
2543             {
2544               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2545               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2546                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2547                                            0);
2548               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2549               goto trace1;
2550             }
2551
2552           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2553
2554           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2555             {
2556               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2557               icmp1 = (icmp46_header_t *) udp1;
2558
2559               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2560                                   rx_fib_index1, node, next1, thread_index,
2561                                   &ses1, &dm1);
2562               goto trace1;
2563             }
2564
2565           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2566           if (PREDICT_FALSE(!dm1))
2567             {
2568               clib_warning("no match for internal host %U",
2569                            format_ip4_address, &ip0->src_address);
2570               next1 = SNAT_IN2OUT_NEXT_DROP;
2571               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2572               goto trace1;
2573             }
2574
2575           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2576
2577           key1.ext_host_addr = ip1->dst_address;
2578           key1.ext_host_port = tcp1->dst;
2579
2580           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2581           if (PREDICT_FALSE(!ses1))
2582             {
2583               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2584                 {
2585                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2586                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2587
2588                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2589                     continue;
2590
2591                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2592                   break;
2593                 }
2594               if (PREDICT_FALSE(!ses1))
2595                 {
2596                   /* too many sessions for user, send ICMP error packet */
2597
2598                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2599                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2600                                                ICMP4_destination_unreachable_destination_unreachable_host,
2601                                                0);
2602                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2603                   goto trace1;
2604                 }
2605             }
2606
2607           new_port1 = ses1->out.out_port;
2608
2609           old_addr1.as_u32 = ip1->src_address.as_u32;
2610           ip1->src_address.as_u32 = new_addr1.as_u32;
2611           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2612
2613           sum1 = ip1->checksum;
2614           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2615                                  ip4_header_t,
2616                                  src_address /* changed member */);
2617           ip1->checksum = ip_csum_fold (sum1);
2618
2619           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2620             {
2621               if (tcp1->flags & TCP_FLAG_SYN)
2622                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2623               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2624                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2625               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2626                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2627               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2628                 snat_det_ses_close(dm1, ses1);
2629               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2630                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2631               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2632                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2633
2634               old_port1 = tcp1->src;
2635               tcp1->src = new_port1;
2636
2637               sum1 = tcp1->checksum;
2638               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2639                                      ip4_header_t,
2640                                      dst_address /* changed member */);
2641               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2642                                      ip4_header_t /* cheat */,
2643                                      length /* changed member */);
2644               tcp1->checksum = ip_csum_fold(sum1);
2645             }
2646           else
2647             {
2648               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2649               old_port1 = udp1->src_port;
2650               udp1->src_port = new_port1;
2651               udp1->checksum = 0;
2652             }
2653
2654           switch(ses1->state)
2655             {
2656             case SNAT_SESSION_UDP_ACTIVE:
2657                 ses1->expire = now + sm->udp_timeout;
2658                 break;
2659             case SNAT_SESSION_TCP_SYN_SENT:
2660             case SNAT_SESSION_TCP_FIN_WAIT:
2661             case SNAT_SESSION_TCP_CLOSE_WAIT:
2662             case SNAT_SESSION_TCP_LAST_ACK:
2663                 ses1->expire = now + sm->tcp_transitory_timeout;
2664                 break;
2665             case SNAT_SESSION_TCP_ESTABLISHED:
2666                 ses1->expire = now + sm->tcp_established_timeout;
2667                 break;
2668             }
2669
2670         trace1:
2671           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2672                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2673             {
2674               snat_in2out_trace_t *t =
2675                  vlib_add_trace (vm, node, b1, sizeof (*t));
2676               t->is_slow_path = 0;
2677               t->sw_if_index = sw_if_index1;
2678               t->next_index = next1;
2679               t->session_index = ~0;
2680               if (ses1)
2681                 t->session_index = ses1 - dm1->sessions;
2682             }
2683
2684           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2685
2686           /* verify speculative enqueues, maybe switch current next frame */
2687           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2688                                            to_next, n_left_to_next,
2689                                            bi0, bi1, next0, next1);
2690          }
2691
2692       while (n_left_from > 0 && n_left_to_next > 0)
2693         {
2694           u32 bi0;
2695           vlib_buffer_t * b0;
2696           u32 next0;
2697           u32 sw_if_index0;
2698           ip4_header_t * ip0;
2699           ip_csum_t sum0;
2700           ip4_address_t new_addr0, old_addr0;
2701           u16 old_port0, new_port0, lo_port0, i0;
2702           udp_header_t * udp0;
2703           tcp_header_t * tcp0;
2704           u32 proto0;
2705           snat_det_out_key_t key0;
2706           snat_det_map_t * dm0;
2707           snat_det_session_t * ses0 = 0;
2708           u32 rx_fib_index0;
2709           icmp46_header_t * icmp0;
2710
2711           /* speculatively enqueue b0 to the current next frame */
2712           bi0 = from[0];
2713           to_next[0] = bi0;
2714           from += 1;
2715           to_next += 1;
2716           n_left_from -= 1;
2717           n_left_to_next -= 1;
2718
2719           b0 = vlib_get_buffer (vm, bi0);
2720           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2721
2722           ip0 = vlib_buffer_get_current (b0);
2723           udp0 = ip4_next_header (ip0);
2724           tcp0 = (tcp_header_t *) udp0;
2725
2726           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2727
2728           if (PREDICT_FALSE(ip0->ttl == 1))
2729             {
2730               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2731               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2732                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2733                                            0);
2734               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2735               goto trace00;
2736             }
2737
2738           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2739
2740           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2741             {
2742               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2743               icmp0 = (icmp46_header_t *) udp0;
2744
2745               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2746                                   rx_fib_index0, node, next0, thread_index,
2747                                   &ses0, &dm0);
2748               goto trace00;
2749             }
2750
2751           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2752           if (PREDICT_FALSE(!dm0))
2753             {
2754               clib_warning("no match for internal host %U",
2755                            format_ip4_address, &ip0->src_address);
2756               next0 = SNAT_IN2OUT_NEXT_DROP;
2757               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2758               goto trace00;
2759             }
2760
2761           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2762
2763           key0.ext_host_addr = ip0->dst_address;
2764           key0.ext_host_port = tcp0->dst;
2765
2766           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2767           if (PREDICT_FALSE(!ses0))
2768             {
2769               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2770                 {
2771                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2772                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2773
2774                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2775                     continue;
2776
2777                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2778                   break;
2779                 }
2780               if (PREDICT_FALSE(!ses0))
2781                 {
2782                   /* too many sessions for user, send ICMP error packet */
2783
2784                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2785                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2786                                                ICMP4_destination_unreachable_destination_unreachable_host,
2787                                                0);
2788                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2789                   goto trace00;
2790                 }
2791             }
2792
2793           new_port0 = ses0->out.out_port;
2794
2795           old_addr0.as_u32 = ip0->src_address.as_u32;
2796           ip0->src_address.as_u32 = new_addr0.as_u32;
2797           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2798
2799           sum0 = ip0->checksum;
2800           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2801                                  ip4_header_t,
2802                                  src_address /* changed member */);
2803           ip0->checksum = ip_csum_fold (sum0);
2804
2805           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2806             {
2807               if (tcp0->flags & TCP_FLAG_SYN)
2808                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2809               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2810                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2811               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2812                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2813               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2814                 snat_det_ses_close(dm0, ses0);
2815               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2816                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2817               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2818                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2819
2820               old_port0 = tcp0->src;
2821               tcp0->src = new_port0;
2822
2823               sum0 = tcp0->checksum;
2824               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2825                                      ip4_header_t,
2826                                      dst_address /* changed member */);
2827               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2828                                      ip4_header_t /* cheat */,
2829                                      length /* changed member */);
2830               tcp0->checksum = ip_csum_fold(sum0);
2831             }
2832           else
2833             {
2834               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2835               old_port0 = udp0->src_port;
2836               udp0->src_port = new_port0;
2837               udp0->checksum = 0;
2838             }
2839
2840           switch(ses0->state)
2841             {
2842             case SNAT_SESSION_UDP_ACTIVE:
2843                 ses0->expire = now + sm->udp_timeout;
2844                 break;
2845             case SNAT_SESSION_TCP_SYN_SENT:
2846             case SNAT_SESSION_TCP_FIN_WAIT:
2847             case SNAT_SESSION_TCP_CLOSE_WAIT:
2848             case SNAT_SESSION_TCP_LAST_ACK:
2849                 ses0->expire = now + sm->tcp_transitory_timeout;
2850                 break;
2851             case SNAT_SESSION_TCP_ESTABLISHED:
2852                 ses0->expire = now + sm->tcp_established_timeout;
2853                 break;
2854             }
2855
2856         trace00:
2857           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2858                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2859             {
2860               snat_in2out_trace_t *t =
2861                  vlib_add_trace (vm, node, b0, sizeof (*t));
2862               t->is_slow_path = 0;
2863               t->sw_if_index = sw_if_index0;
2864               t->next_index = next0;
2865               t->session_index = ~0;
2866               if (ses0)
2867                 t->session_index = ses0 - dm0->sessions;
2868             }
2869
2870           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2871
2872           /* verify speculative enqueue, maybe switch current next frame */
2873           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2874                                            to_next, n_left_to_next,
2875                                            bi0, next0);
2876         }
2877
2878       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2879     }
2880
2881   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2882                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2883                                pkts_processed);
2884   return frame->n_vectors;
2885 }
2886
2887 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2888   .function = snat_det_in2out_node_fn,
2889   .name = "nat44-det-in2out",
2890   .vector_size = sizeof (u32),
2891   .format_trace = format_snat_in2out_trace,
2892   .type = VLIB_NODE_TYPE_INTERNAL,
2893
2894   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2895   .error_strings = snat_in2out_error_strings,
2896
2897   .runtime_data_bytes = sizeof (snat_runtime_t),
2898
2899   .n_next_nodes = 3,
2900
2901   /* edit / add dispositions here */
2902   .next_nodes = {
2903     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2904     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2905     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2906   },
2907 };
2908
2909 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2910
2911 /**
2912  * Get address and port values to be used for ICMP packet translation
2913  * and create session if needed
2914  *
2915  * @param[in,out] sm             NAT main
2916  * @param[in,out] node           NAT node runtime
2917  * @param[in] thread_index       thread index
2918  * @param[in,out] b0             buffer containing packet to be translated
2919  * @param[out] p_proto           protocol used for matching
2920  * @param[out] p_value           address and port after NAT translation
2921  * @param[out] p_dont_translate  if packet should not be translated
2922  * @param d                      optional parameter
2923  * @param e                      optional parameter
2924  */
2925 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
2926                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2927                           snat_session_key_t *p_value,
2928                           u8 *p_dont_translate, void *d, void *e)
2929 {
2930   ip4_header_t *ip0;
2931   icmp46_header_t *icmp0;
2932   u32 sw_if_index0;
2933   u32 rx_fib_index0;
2934   u8 protocol;
2935   snat_det_out_key_t key0;
2936   u8 dont_translate = 0;
2937   u32 next0 = ~0;
2938   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2939   ip4_header_t *inner_ip0;
2940   void *l4_header = 0;
2941   icmp46_header_t *inner_icmp0;
2942   snat_det_map_t * dm0 = 0;
2943   ip4_address_t new_addr0;
2944   u16 lo_port0, i0;
2945   snat_det_session_t * ses0 = 0;
2946   ip4_address_t in_addr;
2947   u16 in_port;
2948
2949   ip0 = vlib_buffer_get_current (b0);
2950   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2951   echo0 = (icmp_echo_header_t *)(icmp0+1);
2952   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2953   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2954
2955   if (!icmp_is_error_message (icmp0))
2956     {
2957       protocol = SNAT_PROTOCOL_ICMP;
2958       in_addr = ip0->src_address;
2959       in_port = echo0->identifier;
2960     }
2961   else
2962     {
2963       inner_ip0 = (ip4_header_t *)(echo0+1);
2964       l4_header = ip4_next_header (inner_ip0);
2965       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2966       in_addr = inner_ip0->dst_address;
2967       switch (protocol)
2968         {
2969         case SNAT_PROTOCOL_ICMP:
2970           inner_icmp0 = (icmp46_header_t*)l4_header;
2971           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2972           in_port = inner_echo0->identifier;
2973           break;
2974         case SNAT_PROTOCOL_UDP:
2975         case SNAT_PROTOCOL_TCP:
2976           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2977           break;
2978         default:
2979           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
2980           next0 = SNAT_IN2OUT_NEXT_DROP;
2981           goto out;
2982         }
2983     }
2984
2985   dm0 = snat_det_map_by_user(sm, &in_addr);
2986   if (PREDICT_FALSE(!dm0))
2987     {
2988       clib_warning("no match for internal host %U",
2989                    format_ip4_address, &in_addr);
2990       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2991           IP_PROTOCOL_ICMP, rx_fib_index0)))
2992         {
2993           dont_translate = 1;
2994           goto out;
2995         }
2996       next0 = SNAT_IN2OUT_NEXT_DROP;
2997       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2998       goto out;
2999     }
3000
3001   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3002
3003   key0.ext_host_addr = ip0->dst_address;
3004   key0.ext_host_port = 0;
3005
3006   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3007   if (PREDICT_FALSE(!ses0))
3008     {
3009       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3010           IP_PROTOCOL_ICMP, rx_fib_index0)))
3011         {
3012           dont_translate = 1;
3013           goto out;
3014         }
3015       if (icmp0->type != ICMP4_echo_request)
3016         {
3017           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3018           next0 = SNAT_IN2OUT_NEXT_DROP;
3019           goto out;
3020         }
3021       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3022         {
3023           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3024             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3025
3026           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3027             continue;
3028
3029           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3030           break;
3031         }
3032       if (PREDICT_FALSE(!ses0))
3033         {
3034           next0 = SNAT_IN2OUT_NEXT_DROP;
3035           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3036           goto out;
3037         }
3038     }
3039
3040   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3041                     !icmp_is_error_message (icmp0)))
3042     {
3043       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3044       next0 = SNAT_IN2OUT_NEXT_DROP;
3045       goto out;
3046     }
3047
3048   u32 now = (u32) vlib_time_now (sm->vlib_main);
3049
3050   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3051   ses0->expire = now + sm->icmp_timeout;
3052
3053 out:
3054   *p_proto = protocol;
3055   if (ses0)
3056     {
3057       p_value->addr = new_addr0;
3058       p_value->fib_index = sm->outside_fib_index;
3059       p_value->port = ses0->out.out_port;
3060     }
3061   *p_dont_translate = dont_translate;
3062   if (d)
3063     *(snat_det_session_t**)d = ses0;
3064   if (e)
3065     *(snat_det_map_t**)e = dm0;
3066   return next0;
3067 }
3068
3069 /**********************/
3070 /*** worker handoff ***/
3071 /**********************/
3072 static inline uword
3073 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3074                                       vlib_node_runtime_t * node,
3075                                       vlib_frame_t * frame,
3076                                       u8 is_output)
3077 {
3078   snat_main_t *sm = &snat_main;
3079   vlib_thread_main_t *tm = vlib_get_thread_main ();
3080   u32 n_left_from, *from, *to_next = 0;
3081   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3082   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3083     = 0;
3084   vlib_frame_queue_elt_t *hf = 0;
3085   vlib_frame_t *f = 0;
3086   int i;
3087   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3088   u32 next_worker_index = 0;
3089   u32 current_worker_index = ~0;
3090   u32 thread_index = vlib_get_thread_index ();
3091   u32 fq_index;
3092   u32 to_node_index;
3093
3094   ASSERT (vec_len (sm->workers));
3095
3096   if (is_output)
3097     {
3098       fq_index = sm->fq_in2out_output_index;
3099       to_node_index = sm->in2out_output_node_index;
3100     }
3101   else
3102     {
3103       fq_index = sm->fq_in2out_index;
3104       to_node_index = sm->in2out_node_index;
3105     }
3106
3107   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3108     {
3109       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3110
3111       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3112                                sm->first_worker_index + sm->num_workers - 1,
3113                                (vlib_frame_queue_t *) (~0));
3114     }
3115
3116   from = vlib_frame_vector_args (frame);
3117   n_left_from = frame->n_vectors;
3118
3119   while (n_left_from > 0)
3120     {
3121       u32 bi0;
3122       vlib_buffer_t *b0;
3123       u32 sw_if_index0;
3124       u32 rx_fib_index0;
3125       ip4_header_t * ip0;
3126       u8 do_handoff;
3127
3128       bi0 = from[0];
3129       from += 1;
3130       n_left_from -= 1;
3131
3132       b0 = vlib_get_buffer (vm, bi0);
3133
3134       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3135       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3136
3137       ip0 = vlib_buffer_get_current (b0);
3138
3139       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3140
3141       if (PREDICT_FALSE (next_worker_index != thread_index))
3142         {
3143           do_handoff = 1;
3144
3145           if (next_worker_index != current_worker_index)
3146             {
3147               if (hf)
3148                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3149
3150               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3151                                                       next_worker_index,
3152                                                       handoff_queue_elt_by_worker_index);
3153
3154               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3155               to_next_worker = &hf->buffer_index[hf->n_vectors];
3156               current_worker_index = next_worker_index;
3157             }
3158
3159           /* enqueue to correct worker thread */
3160           to_next_worker[0] = bi0;
3161           to_next_worker++;
3162           n_left_to_next_worker--;
3163
3164           if (n_left_to_next_worker == 0)
3165             {
3166               hf->n_vectors = VLIB_FRAME_SIZE;
3167               vlib_put_frame_queue_elt (hf);
3168               current_worker_index = ~0;
3169               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3170               hf = 0;
3171             }
3172         }
3173       else
3174         {
3175           do_handoff = 0;
3176           /* if this is 1st frame */
3177           if (!f)
3178             {
3179               f = vlib_get_frame_to_node (vm, to_node_index);
3180               to_next = vlib_frame_vector_args (f);
3181             }
3182
3183           to_next[0] = bi0;
3184           to_next += 1;
3185           f->n_vectors++;
3186         }
3187
3188       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3189                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3190         {
3191           snat_in2out_worker_handoff_trace_t *t =
3192             vlib_add_trace (vm, node, b0, sizeof (*t));
3193           t->next_worker_index = next_worker_index;
3194           t->do_handoff = do_handoff;
3195         }
3196     }
3197
3198   if (f)
3199     vlib_put_frame_to_node (vm, to_node_index, f);
3200
3201   if (hf)
3202     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3203
3204   /* Ship frames to the worker nodes */
3205   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3206     {
3207       if (handoff_queue_elt_by_worker_index[i])
3208         {
3209           hf = handoff_queue_elt_by_worker_index[i];
3210           /*
3211            * It works better to let the handoff node
3212            * rate-adapt, always ship the handoff queue element.
3213            */
3214           if (1 || hf->n_vectors == hf->last_n_vectors)
3215             {
3216               vlib_put_frame_queue_elt (hf);
3217               handoff_queue_elt_by_worker_index[i] = 0;
3218             }
3219           else
3220             hf->last_n_vectors = hf->n_vectors;
3221         }
3222       congested_handoff_queue_by_worker_index[i] =
3223         (vlib_frame_queue_t *) (~0);
3224     }
3225   hf = 0;
3226   current_worker_index = ~0;
3227   return frame->n_vectors;
3228 }
3229
3230 static uword
3231 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3232                                vlib_node_runtime_t * node,
3233                                vlib_frame_t * frame)
3234 {
3235   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3236 }
3237
3238 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3239   .function = snat_in2out_worker_handoff_fn,
3240   .name = "nat44-in2out-worker-handoff",
3241   .vector_size = sizeof (u32),
3242   .format_trace = format_snat_in2out_worker_handoff_trace,
3243   .type = VLIB_NODE_TYPE_INTERNAL,
3244
3245   .n_next_nodes = 1,
3246
3247   .next_nodes = {
3248     [0] = "error-drop",
3249   },
3250 };
3251
3252 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3253                               snat_in2out_worker_handoff_fn);
3254
3255 static uword
3256 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3257                                       vlib_node_runtime_t * node,
3258                                       vlib_frame_t * frame)
3259 {
3260   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3261 }
3262
3263 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3264   .function = snat_in2out_output_worker_handoff_fn,
3265   .name = "nat44-in2out-output-worker-handoff",
3266   .vector_size = sizeof (u32),
3267   .format_trace = format_snat_in2out_worker_handoff_trace,
3268   .type = VLIB_NODE_TYPE_INTERNAL,
3269
3270   .n_next_nodes = 1,
3271
3272   .next_nodes = {
3273     [0] = "error-drop",
3274   },
3275 };
3276
3277 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3278                               snat_in2out_output_worker_handoff_fn);
3279
3280 static_always_inline int
3281 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3282 {
3283   snat_address_t * ap;
3284   clib_bihash_kv_8_8_t kv, value;
3285   snat_session_key_t m_key;
3286
3287   vec_foreach (ap, sm->addresses)
3288     {
3289       if (ap->addr.as_u32 == dst_addr->as_u32)
3290         return 1;
3291     }
3292
3293   m_key.addr.as_u32 = dst_addr->as_u32;
3294   m_key.fib_index = sm->outside_fib_index;
3295   m_key.port = 0;
3296   m_key.protocol = 0;
3297   kv.key = m_key.as_u64;
3298   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3299     return 1;
3300
3301   return 0;
3302 }
3303
3304 static uword
3305 snat_hairpin_dst_fn (vlib_main_t * vm,
3306                      vlib_node_runtime_t * node,
3307                      vlib_frame_t * frame)
3308 {
3309   u32 n_left_from, * from, * to_next;
3310   snat_in2out_next_t next_index;
3311   u32 pkts_processed = 0;
3312   snat_main_t * sm = &snat_main;
3313
3314   from = vlib_frame_vector_args (frame);
3315   n_left_from = frame->n_vectors;
3316   next_index = node->cached_next_index;
3317
3318   while (n_left_from > 0)
3319     {
3320       u32 n_left_to_next;
3321
3322       vlib_get_next_frame (vm, node, next_index,
3323                            to_next, n_left_to_next);
3324
3325       while (n_left_from > 0 && n_left_to_next > 0)
3326         {
3327           u32 bi0;
3328           vlib_buffer_t * b0;
3329           u32 next0;
3330           ip4_header_t * ip0;
3331           u32 proto0;
3332
3333           /* speculatively enqueue b0 to the current next frame */
3334           bi0 = from[0];
3335           to_next[0] = bi0;
3336           from += 1;
3337           to_next += 1;
3338           n_left_from -= 1;
3339           n_left_to_next -= 1;
3340
3341           b0 = vlib_get_buffer (vm, bi0);
3342           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3343           ip0 = vlib_buffer_get_current (b0);
3344
3345           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3346
3347           vnet_buffer (b0)->snat.flags = 0;
3348           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3349             {
3350               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3351                 {
3352                   udp_header_t * udp0 = ip4_next_header (ip0);
3353                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3354
3355                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3356                 }
3357               else if (proto0 == SNAT_PROTOCOL_ICMP)
3358                 {
3359                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3360
3361                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3362                 }
3363               else
3364                 {
3365                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3366                 }
3367
3368               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3369               clib_warning("is hairpinning");
3370             }
3371
3372           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3373
3374           /* verify speculative enqueue, maybe switch current next frame */
3375           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3376                                            to_next, n_left_to_next,
3377                                            bi0, next0);
3378          }
3379
3380       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3381     }
3382
3383   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3384                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3385                                pkts_processed);
3386   return frame->n_vectors;
3387 }
3388
3389 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3390   .function = snat_hairpin_dst_fn,
3391   .name = "nat44-hairpin-dst",
3392   .vector_size = sizeof (u32),
3393   .type = VLIB_NODE_TYPE_INTERNAL,
3394   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3395   .error_strings = snat_in2out_error_strings,
3396   .n_next_nodes = 2,
3397   .next_nodes = {
3398     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3399     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3400   },
3401 };
3402
3403 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3404                               snat_hairpin_dst_fn);
3405
3406 static uword
3407 snat_hairpin_src_fn (vlib_main_t * vm,
3408                      vlib_node_runtime_t * node,
3409                      vlib_frame_t * frame)
3410 {
3411   u32 n_left_from, * from, * to_next;
3412   snat_in2out_next_t next_index;
3413   u32 pkts_processed = 0;
3414   snat_main_t *sm = &snat_main;
3415
3416   from = vlib_frame_vector_args (frame);
3417   n_left_from = frame->n_vectors;
3418   next_index = node->cached_next_index;
3419
3420   while (n_left_from > 0)
3421     {
3422       u32 n_left_to_next;
3423
3424       vlib_get_next_frame (vm, node, next_index,
3425                            to_next, n_left_to_next);
3426
3427       while (n_left_from > 0 && n_left_to_next > 0)
3428         {
3429           u32 bi0;
3430           vlib_buffer_t * b0;
3431           u32 next0;
3432           snat_interface_t *i;
3433           u32 sw_if_index0;
3434
3435           /* speculatively enqueue b0 to the current next frame */
3436           bi0 = from[0];
3437           to_next[0] = bi0;
3438           from += 1;
3439           to_next += 1;
3440           n_left_from -= 1;
3441           n_left_to_next -= 1;
3442
3443           b0 = vlib_get_buffer (vm, bi0);
3444           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3445           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3446
3447           pool_foreach (i, sm->output_feature_interfaces,
3448           ({
3449             /* Only packets from NAT inside interface */
3450             if ((i->is_inside == 1) && (sw_if_index0 == i->sw_if_index))
3451               {
3452                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3453                                     SNAT_FLAG_HAIRPINNING))
3454                   {
3455                     if (PREDICT_TRUE (sm->num_workers > 1))
3456                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3457                     else
3458                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3459                   }
3460                 break;
3461               }
3462           }));
3463
3464           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3465
3466           /* verify speculative enqueue, maybe switch current next frame */
3467           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3468                                            to_next, n_left_to_next,
3469                                            bi0, next0);
3470          }
3471
3472       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3473     }
3474
3475   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3476                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3477                                pkts_processed);
3478   return frame->n_vectors;
3479 }
3480
3481 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3482   .function = snat_hairpin_src_fn,
3483   .name = "nat44-hairpin-src",
3484   .vector_size = sizeof (u32),
3485   .type = VLIB_NODE_TYPE_INTERNAL,
3486   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3487   .error_strings = snat_in2out_error_strings,
3488   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3489   .next_nodes = {
3490      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3491      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3492      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3493      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3494   },
3495 };
3496
3497 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3498                               snat_hairpin_src_fn);
3499
3500 static uword
3501 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3502                                 vlib_node_runtime_t * node,
3503                                 vlib_frame_t * frame)
3504 {
3505   u32 n_left_from, * from, * to_next;
3506   snat_in2out_next_t next_index;
3507   u32 pkts_processed = 0;
3508   snat_main_t * sm = &snat_main;
3509   u32 stats_node_index;
3510
3511   stats_node_index = snat_in2out_fast_node.index;
3512
3513   from = vlib_frame_vector_args (frame);
3514   n_left_from = frame->n_vectors;
3515   next_index = node->cached_next_index;
3516
3517   while (n_left_from > 0)
3518     {
3519       u32 n_left_to_next;
3520
3521       vlib_get_next_frame (vm, node, next_index,
3522                            to_next, n_left_to_next);
3523
3524       while (n_left_from > 0 && n_left_to_next > 0)
3525         {
3526           u32 bi0;
3527           vlib_buffer_t * b0;
3528           u32 next0;
3529           u32 sw_if_index0;
3530           ip4_header_t * ip0;
3531           ip_csum_t sum0;
3532           u32 new_addr0, old_addr0;
3533           u16 old_port0, new_port0;
3534           udp_header_t * udp0;
3535           tcp_header_t * tcp0;
3536           icmp46_header_t * icmp0;
3537           snat_session_key_t key0, sm0;
3538           u32 proto0;
3539           u32 rx_fib_index0;
3540
3541           /* speculatively enqueue b0 to the current next frame */
3542           bi0 = from[0];
3543           to_next[0] = bi0;
3544           from += 1;
3545           to_next += 1;
3546           n_left_from -= 1;
3547           n_left_to_next -= 1;
3548
3549           b0 = vlib_get_buffer (vm, bi0);
3550           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3551
3552           ip0 = vlib_buffer_get_current (b0);
3553           udp0 = ip4_next_header (ip0);
3554           tcp0 = (tcp_header_t *) udp0;
3555           icmp0 = (icmp46_header_t *) udp0;
3556
3557           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3558           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3559
3560           if (PREDICT_FALSE(ip0->ttl == 1))
3561             {
3562               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3563               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3564                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3565                                            0);
3566               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3567               goto trace0;
3568             }
3569
3570           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3571
3572           if (PREDICT_FALSE (proto0 == ~0))
3573               goto trace0;
3574
3575           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3576             {
3577               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3578                                   rx_fib_index0, node, next0, ~0, 0, 0);
3579               goto trace0;
3580             }
3581
3582           key0.addr = ip0->src_address;
3583           key0.protocol = proto0;
3584           key0.port = udp0->src_port;
3585           key0.fib_index = rx_fib_index0;
3586
3587           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
3588             {
3589               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3590               next0= SNAT_IN2OUT_NEXT_DROP;
3591               goto trace0;
3592             }
3593
3594           new_addr0 = sm0.addr.as_u32;
3595           new_port0 = sm0.port;
3596           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3597           old_addr0 = ip0->src_address.as_u32;
3598           ip0->src_address.as_u32 = new_addr0;
3599
3600           sum0 = ip0->checksum;
3601           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3602                                  ip4_header_t,
3603                                  src_address /* changed member */);
3604           ip0->checksum = ip_csum_fold (sum0);
3605
3606           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3607             {
3608               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3609                 {
3610                   old_port0 = tcp0->src_port;
3611                   tcp0->src_port = new_port0;
3612
3613                   sum0 = tcp0->checksum;
3614                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3615                                          ip4_header_t,
3616                                          dst_address /* changed member */);
3617                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3618                                          ip4_header_t /* cheat */,
3619                                          length /* changed member */);
3620                   tcp0->checksum = ip_csum_fold(sum0);
3621                 }
3622               else
3623                 {
3624                   old_port0 = udp0->src_port;
3625                   udp0->src_port = new_port0;
3626                   udp0->checksum = 0;
3627                 }
3628             }
3629           else
3630             {
3631               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3632                 {
3633                   sum0 = tcp0->checksum;
3634                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3635                                          ip4_header_t,
3636                                          dst_address /* changed member */);
3637                   tcp0->checksum = ip_csum_fold(sum0);
3638                 }
3639             }
3640
3641           /* Hairpinning */
3642           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3643
3644         trace0:
3645           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3646                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3647             {
3648               snat_in2out_trace_t *t =
3649                  vlib_add_trace (vm, node, b0, sizeof (*t));
3650               t->sw_if_index = sw_if_index0;
3651               t->next_index = next0;
3652             }
3653
3654           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3655
3656           /* verify speculative enqueue, maybe switch current next frame */
3657           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3658                                            to_next, n_left_to_next,
3659                                            bi0, next0);
3660         }
3661
3662       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3663     }
3664
3665   vlib_node_increment_counter (vm, stats_node_index,
3666                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3667                                pkts_processed);
3668   return frame->n_vectors;
3669 }
3670
3671
3672 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3673   .function = snat_in2out_fast_static_map_fn,
3674   .name = "nat44-in2out-fast",
3675   .vector_size = sizeof (u32),
3676   .format_trace = format_snat_in2out_fast_trace,
3677   .type = VLIB_NODE_TYPE_INTERNAL,
3678
3679   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3680   .error_strings = snat_in2out_error_strings,
3681
3682   .runtime_data_bytes = sizeof (snat_runtime_t),
3683
3684   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3685
3686   /* edit / add dispositions here */
3687   .next_nodes = {
3688     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3689     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3690     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3691     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3692   },
3693 };
3694
3695 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);