src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91 vlib_node_registration_t snat_in2out_output_node;
  92 vlib_node_registration_t snat_in2out_output_slowpath_node;
  93 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
  94 vlib_node_registration_t snat_hairpin_dst_node;
  95 vlib_node_registration_t snat_hairpin_src_node;
  96
  97
  98 #define foreach_snat_in2out_error                       \
  99 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 100 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 101 _(OUT_OF_PORTS, "Out of ports")                         \
 102 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 103 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 104 _(NO_TRANSLATION, "No translation")
 105
 106 typedef enum {
 107 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 108   foreach_snat_in2out_error
 109 #undef _
 110   SNAT_IN2OUT_N_ERROR,
 111 } snat_in2out_error_t;
 112
 113 static char * snat_in2out_error_strings[] = {
 114 #define _(sym,string) string,
 115   foreach_snat_in2out_error
 116 #undef _
 117 };
 118
 119 typedef enum {
 120   SNAT_IN2OUT_NEXT_LOOKUP,
 121   SNAT_IN2OUT_NEXT_DROP,
 122   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 123   SNAT_IN2OUT_NEXT_SLOW_PATH,
 124   SNAT_IN2OUT_N_NEXT,
 125 } snat_in2out_next_t;
 126
 127 typedef enum {
 128   SNAT_HAIRPIN_SRC_NEXT_DROP,
 129   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 130   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 131   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 132   SNAT_HAIRPIN_SRC_N_NEXT,
 133 } snat_hairpin_next_t;
 134
 135 /**
 136  * @brief Check if packet should be translated
 137  *
 138  * Packets aimed at outside interface and external addresss with active session
 139  * should be translated.
 140  *
 141  * @param sm            NAT main
 142  * @param rt            NAT runtime data
 143  * @param sw_if_index0  index of the inside interface
 144  * @param ip0           IPv4 header
 145  * @param proto0        NAT protocol
 146  * @param rx_fib_index0 RX FIB index
 147  *
 148  * @returns 0 if packet should be translated otherwise 1
 149  */
 150 static inline int
 151 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 152                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 153                          u32 rx_fib_index0)
 154 {
 155   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 156   fib_prefix_t pfx = {
 157     .fp_proto = FIB_PROTOCOL_IP4,
 158     .fp_len = 32,
 159     .fp_addr = {
 160         .ip4.as_u32 = ip0->dst_address.as_u32,
 161     },
 162   };
 163
 164   /* Don't NAT packet aimed at the intfc address */
 165   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 166                                       ip0->dst_address.as_u32)))
 167     return 1;
 168
 169   fei = fib_table_lookup (rx_fib_index0, &pfx);
 170   if (FIB_NODE_INDEX_INVALID != fei)
 171     {
 172       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 173       if (sw_if_index == ~0)
 174         {
 175           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 176           if (FIB_NODE_INDEX_INVALID != fei)
 177             sw_if_index = fib_entry_get_resolving_interface (fei);
 178         }
 179       snat_interface_t *i;
 180       pool_foreach (i, sm->interfaces,
 181       ({
 182         /* NAT packet aimed at outside interface */
 183         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 184           return 0;
 185       }));
 186     }
 187
 188   return 1;
 189 }
 190
 191 static inline int
 192 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 193                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 194                     u32 rx_fib_index0)
 195 {
 196   udp_header_t * udp0 = ip4_next_header (ip0);
 197   snat_session_key_t key0, sm0;
 198   clib_bihash_kv_8_8_t kv0, value0;
 199
 200   key0.addr = ip0->dst_address;
 201   key0.port = udp0->dst_port;
 202   key0.protocol = proto0;
 203   key0.fib_index = sm->outside_fib_index;
 204   kv0.key = key0.as_u64;
 205
 206   /* NAT packet aimed at external address if */
 207   /* has active sessions */
 208   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 209     {
 210       /* or is static mappings */
 211       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 212         return 0;
 213     }
 214   else
 215     return 0;
 216
 217   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 218                                  rx_fib_index0);
 219 }
 220
 221 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 222                       ip4_header_t * ip0,
 223                       u32 rx_fib_index0,
 224                       snat_session_key_t * key0,
 225                       snat_session_t ** sessionp,
 226                       vlib_node_runtime_t * node,
 227                       u32 next0,
 228                       u32 thread_index)
 229 {
 230   snat_user_t *u;
 231   snat_user_key_t user_key;
 232   snat_session_t *s;
 233   clib_bihash_kv_8_8_t kv0, value0;
 234   u32 oldest_per_user_translation_list_index;
 235   dlist_elt_t * oldest_per_user_translation_list_elt;
 236   dlist_elt_t * per_user_translation_list_elt;
 237   dlist_elt_t * per_user_list_head_elt;
 238   u32 session_index;
 239   snat_session_key_t key1;
 240   u32 address_index = ~0;
 241   u32 outside_fib_index;
 242   uword * p;
 243   snat_worker_key_t worker_by_out_key;
 244
 245   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 246   if (! p)
 247     {
 248       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 249       return SNAT_IN2OUT_NEXT_DROP;
 250     }
 251   outside_fib_index = p[0];
 252
 253   key1.protocol = key0->protocol;
 254   user_key.addr = ip0->src_address;
 255   user_key.fib_index = rx_fib_index0;
 256   kv0.key = user_key.as_u64;
 257
 258   /* Ever heard of the "user" = src ip4 address before? */
 259   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 260     {
 261       /* no, make a new one */
 262       pool_get (sm->per_thread_data[thread_index].users, u);
 263       memset (u, 0, sizeof (*u));
 264       u->addr = ip0->src_address;
 265       u->fib_index = rx_fib_index0;
 266
 267       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 268
 269       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 270         sm->per_thread_data[thread_index].list_pool;
 271
 272       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 273                        u->sessions_per_user_list_head_index);
 274
 275       kv0.value = u - sm->per_thread_data[thread_index].users;
 276
 277       /* add user */
 278       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 279     }
 280   else
 281     {
 282       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 283                              value0.value);
 284     }
 285
 286   /* Over quota? Recycle the least recently used dynamic translation */
 287   if (u->nsessions >= sm->max_translations_per_user)
 288     {
 289       /* Remove the oldest dynamic translation */
 290       do {
 291           oldest_per_user_translation_list_index =
 292             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 293                                     u->sessions_per_user_list_head_index);
 294
 295           ASSERT (oldest_per_user_translation_list_index != ~0);
 296
 297           /* add it back to the end of the LRU list */
 298           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 299                               u->sessions_per_user_list_head_index,
 300                               oldest_per_user_translation_list_index);
 301           /* Get the list element */
 302           oldest_per_user_translation_list_elt =
 303             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 304                                oldest_per_user_translation_list_index);
 305
 306           /* Get the session index from the list element */
 307           session_index = oldest_per_user_translation_list_elt->value;
 308
 309           /* Get the session */
 310           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 311                                  session_index);
 312       } while (snat_is_session_static (s));
 313
 314       if (snat_is_unk_proto_session (s))
 315         {
 316           clib_bihash_kv_16_8_t up_kv;
 317           nat_ed_ses_key_t key;
 318
 319           /* Remove from lookup tables */
 320           key.l_addr = s->in2out.addr;
 321           key.r_addr = s->ext_host_addr;
 322           key.fib_index = s->in2out.fib_index;
 323           key.proto = s->in2out.port;
 324           key.rsvd = 0;
 325           key.l_port = 0;
 326           up_kv.key[0] = key.as_u64[0];
 327           up_kv.key[1] = key.as_u64[1];
 328           if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &up_kv, 0))
 329             clib_warning ("in2out key del failed");
 330
 331           key.l_addr = s->out2in.addr;
 332           key.fib_index = s->out2in.fib_index;
 333           up_kv.key[0] = key.as_u64[0];
 334           up_kv.key[1] = key.as_u64[1];
 335           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &up_kv, 0))
 336             clib_warning ("out2in key del failed");
 337         }
 338       else
 339         {
 340           /* Remove in2out, out2in keys */
 341           kv0.key = s->in2out.as_u64;
 342           if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 343               clib_warning ("in2out key delete failed");
 344           kv0.key = s->out2in.as_u64;
 345           if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 346               clib_warning ("out2in key delete failed");
 347
 348           /* log NAT event */
 349           snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 350                                               s->out2in.addr.as_u32,
 351                                               s->in2out.protocol,
 352                                               s->in2out.port,
 353                                               s->out2in.port,
 354                                               s->in2out.fib_index);
 355
 356           snat_free_outside_address_and_port
 357             (sm, thread_index, &s->out2in, s->outside_address_index);
 358         }
 359       s->outside_address_index = ~0;
 360
 361       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, thread_index,
 362                                                &key1, &address_index))
 363         {
 364           ASSERT(0);
 365
 366           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 367           return SNAT_IN2OUT_NEXT_DROP;
 368         }
 369       s->outside_address_index = address_index;
 370     }
 371   else
 372     {
 373       u8 static_mapping = 1;
 374
 375       /* First try to match static mapping by local address and port */
 376       if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
 377         {
 378           static_mapping = 0;
 379           /* Try to create dynamic translation */
 380           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0,
 381                                                    thread_index, &key1,
 382                                                    &address_index))
 383             {
 384               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 385               return SNAT_IN2OUT_NEXT_DROP;
 386             }
 387         }
 388
 389       /* Create a new session */
 390       pool_get (sm->per_thread_data[thread_index].sessions, s);
 391       memset (s, 0, sizeof (*s));
 392
 393       s->outside_address_index = address_index;
 394
 395       if (static_mapping)
 396         {
 397           u->nstaticsessions++;
 398           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 399         }
 400       else
 401         {
 402           u->nsessions++;
 403         }
 404
 405       /* Create list elts */
 406       pool_get (sm->per_thread_data[thread_index].list_pool,
 407                 per_user_translation_list_elt);
 408       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 409                        per_user_translation_list_elt -
 410                        sm->per_thread_data[thread_index].list_pool);
 411
 412       per_user_translation_list_elt->value =
 413         s - sm->per_thread_data[thread_index].sessions;
 414       s->per_user_index = per_user_translation_list_elt -
 415                           sm->per_thread_data[thread_index].list_pool;
 416       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 417
 418       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 419                           s->per_user_list_head_index,
 420                           per_user_translation_list_elt -
 421                           sm->per_thread_data[thread_index].list_pool);
 422    }
 423
 424   s->in2out = *key0;
 425   s->out2in = key1;
 426   s->out2in.protocol = key0->protocol;
 427   s->out2in.fib_index = outside_fib_index;
 428   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 429   *sessionp = s;
 430
 431   /* Add to translation hashes */
 432   kv0.key = s->in2out.as_u64;
 433   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 434   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 435       clib_warning ("in2out key add failed");
 436
 437   kv0.key = s->out2in.as_u64;
 438   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 439
 440   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 441       clib_warning ("out2in key add failed");
 442
 443   /* Add to translated packets worker lookup */
 444   worker_by_out_key.addr = s->out2in.addr;
 445   worker_by_out_key.port = s->out2in.port;
 446   worker_by_out_key.fib_index = s->out2in.fib_index;
 447   kv0.key = worker_by_out_key.as_u64;
 448   kv0.value = thread_index;
 449   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 450
 451   /* log NAT event */
 452   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 453                                       s->out2in.addr.as_u32,
 454                                       s->in2out.protocol,
 455                                       s->in2out.port,
 456                                       s->out2in.port,
 457                                       s->in2out.fib_index);
 458   return next0;
 459 }
 460
 461 static_always_inline
 462 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 463                                  snat_session_key_t *p_key0)
 464 {
 465   icmp46_header_t *icmp0;
 466   snat_session_key_t key0;
 467   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 468   ip4_header_t *inner_ip0 = 0;
 469   void *l4_header = 0;
 470   icmp46_header_t *inner_icmp0;
 471
 472   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 473   echo0 = (icmp_echo_header_t *)(icmp0+1);
 474
 475   if (!icmp_is_error_message (icmp0))
 476     {
 477       key0.protocol = SNAT_PROTOCOL_ICMP;
 478       key0.addr = ip0->src_address;
 479       key0.port = echo0->identifier;
 480     }
 481   else
 482     {
 483       inner_ip0 = (ip4_header_t *)(echo0+1);
 484       l4_header = ip4_next_header (inner_ip0);
 485       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 486       key0.addr = inner_ip0->dst_address;
 487       switch (key0.protocol)
 488         {
 489         case SNAT_PROTOCOL_ICMP:
 490           inner_icmp0 = (icmp46_header_t*)l4_header;
 491           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 492           key0.port = inner_echo0->identifier;
 493           break;
 494         case SNAT_PROTOCOL_UDP:
 495         case SNAT_PROTOCOL_TCP:
 496           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 497           break;
 498         default:
 499           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 500         }
 501     }
 502   *p_key0 = key0;
 503   return -1; /* success */
 504 }
 505
 506 /**
 507  * Get address and port values to be used for ICMP packet translation
 508  * and create session if needed
 509  *
 510  * @param[in,out] sm             NAT main
 511  * @param[in,out] node           NAT node runtime
 512  * @param[in] thread_index       thread index
 513  * @param[in,out] b0             buffer containing packet to be translated
 514  * @param[out] p_proto           protocol used for matching
 515  * @param[out] p_value           address and port after NAT translation
 516  * @param[out] p_dont_translate  if packet should not be translated
 517  * @param d                      optional parameter
 518  * @param e                      optional parameter
 519  */
 520 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 521                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 522                            snat_session_key_t *p_value,
 523                            u8 *p_dont_translate, void *d, void *e)
 524 {
 525   ip4_header_t *ip0;
 526   icmp46_header_t *icmp0;
 527   u32 sw_if_index0;
 528   u32 rx_fib_index0;
 529   snat_session_key_t key0;
 530   snat_session_t *s0 = 0;
 531   u8 dont_translate = 0;
 532   clib_bihash_kv_8_8_t kv0, value0;
 533   u32 next0 = ~0;
 534   int err;
 535   u32 iph_offset0 = 0;
 536
 537   if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
 538     {
 539       iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 540     }
 541   ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
 542   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 543   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 544   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 545
 546   err = icmp_get_key (ip0, &key0);
 547   if (err != -1)
 548     {
 549       b0->error = node->errors[err];
 550       next0 = SNAT_IN2OUT_NEXT_DROP;
 551       goto out;
 552     }
 553   key0.fib_index = rx_fib_index0;
 554
 555   kv0.key = key0.as_u64;
 556
 557   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 558     {
 559       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 560           IP_PROTOCOL_ICMP, rx_fib_index0) &&
 561           vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
 562         {
 563           dont_translate = 1;
 564           goto out;
 565         }
 566
 567       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 568         {
 569           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 570           next0 = SNAT_IN2OUT_NEXT_DROP;
 571           goto out;
 572         }
 573
 574       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 575                          &s0, node, next0, thread_index);
 576
 577       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 578         goto out;
 579     }
 580   else
 581     {
 582       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 583                         icmp0->type != ICMP4_echo_reply &&
 584                         !icmp_is_error_message (icmp0)))
 585         {
 586           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 587           next0 = SNAT_IN2OUT_NEXT_DROP;
 588           goto out;
 589         }
 590
 591       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 592                               value0.value);
 593     }
 594
 595 out:
 596   *p_proto = key0.protocol;
 597   if (s0)
 598     *p_value = s0->out2in;
 599   *p_dont_translate = dont_translate;
 600   if (d)
 601     *(snat_session_t**)d = s0;
 602   return next0;
 603 }
 604
 605 /**
 606  * Get address and port values to be used for ICMP packet translation
 607  *
 608  * @param[in] sm                 NAT main
 609  * @param[in,out] node           NAT node runtime
 610  * @param[in] thread_index       thread index
 611  * @param[in,out] b0             buffer containing packet to be translated
 612  * @param[out] p_proto           protocol used for matching
 613  * @param[out] p_value           address and port after NAT translation
 614  * @param[out] p_dont_translate  if packet should not be translated
 615  * @param d                      optional parameter
 616  * @param e                      optional parameter
 617  */
 618 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 619                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 620                            snat_session_key_t *p_value,
 621                            u8 *p_dont_translate, void *d, void *e)
 622 {
 623   ip4_header_t *ip0;
 624   icmp46_header_t *icmp0;
 625   u32 sw_if_index0;
 626   u32 rx_fib_index0;
 627   snat_session_key_t key0;
 628   snat_session_key_t sm0;
 629   u8 dont_translate = 0;
 630   u8 is_addr_only;
 631   u32 next0 = ~0;
 632   int err;
 633
 634   ip0 = vlib_buffer_get_current (b0);
 635   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 636   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 637   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 638
 639   err = icmp_get_key (ip0, &key0);
 640   if (err != -1)
 641     {
 642       b0->error = node->errors[err];
 643       next0 = SNAT_IN2OUT_NEXT_DROP;
 644       goto out2;
 645     }
 646   key0.fib_index = rx_fib_index0;
 647
 648   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
 649     {
 650       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 651           IP_PROTOCOL_ICMP, rx_fib_index0)))
 652         {
 653           dont_translate = 1;
 654           goto out;
 655         }
 656
 657       if (icmp_is_error_message (icmp0))
 658         {
 659           next0 = SNAT_IN2OUT_NEXT_DROP;
 660           goto out;
 661         }
 662
 663       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 664       next0 = SNAT_IN2OUT_NEXT_DROP;
 665       goto out;
 666     }
 667
 668   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 669                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 670                     !icmp_is_error_message (icmp0)))
 671     {
 672       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 673       next0 = SNAT_IN2OUT_NEXT_DROP;
 674       goto out;
 675     }
 676
 677 out:
 678   *p_value = sm0;
 679 out2:
 680   *p_proto = key0.protocol;
 681   *p_dont_translate = dont_translate;
 682   return next0;
 683 }
 684
 685 static inline u32 icmp_in2out (snat_main_t *sm,
 686                                vlib_buffer_t * b0,
 687                                ip4_header_t * ip0,
 688                                icmp46_header_t * icmp0,
 689                                u32 sw_if_index0,
 690                                u32 rx_fib_index0,
 691                                vlib_node_runtime_t * node,
 692                                u32 next0,
 693                                u32 thread_index,
 694                                void *d,
 695                                void *e)
 696 {
 697   snat_session_key_t sm0;
 698   u8 protocol;
 699   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 700   ip4_header_t *inner_ip0;
 701   void *l4_header = 0;
 702   icmp46_header_t *inner_icmp0;
 703   u8 dont_translate;
 704   u32 new_addr0, old_addr0;
 705   u16 old_id0, new_id0;
 706   ip_csum_t sum0;
 707   u16 checksum0;
 708   u32 next0_tmp;
 709
 710   echo0 = (icmp_echo_header_t *)(icmp0+1);
 711
 712   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 713                                        &protocol, &sm0, &dont_translate, d, e);
 714   if (next0_tmp != ~0)
 715     next0 = next0_tmp;
 716   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 717     goto out;
 718
 719   sum0 = ip_incremental_checksum (0, icmp0,
 720                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 721   checksum0 = ~ip_csum_fold (sum0);
 722   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 723     {
 724       next0 = SNAT_IN2OUT_NEXT_DROP;
 725       goto out;
 726     }
 727
 728   old_addr0 = ip0->src_address.as_u32;
 729   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 730   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 731     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 732
 733   sum0 = ip0->checksum;
 734   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 735                          src_address /* changed member */);
 736   ip0->checksum = ip_csum_fold (sum0);
 737
 738   if (!icmp_is_error_message (icmp0))
 739     {
 740       new_id0 = sm0.port;
 741       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 742         {
 743           old_id0 = echo0->identifier;
 744           new_id0 = sm0.port;
 745           echo0->identifier = new_id0;
 746
 747           sum0 = icmp0->checksum;
 748           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 749                                  identifier);
 750           icmp0->checksum = ip_csum_fold (sum0);
 751         }
 752     }
 753   else
 754     {
 755       inner_ip0 = (ip4_header_t *)(echo0+1);
 756       l4_header = ip4_next_header (inner_ip0);
 757
 758       if (!ip4_header_checksum_is_valid (inner_ip0))
 759         {
 760           next0 = SNAT_IN2OUT_NEXT_DROP;
 761           goto out;
 762         }
 763
 764       old_addr0 = inner_ip0->dst_address.as_u32;
 765       inner_ip0->dst_address = sm0.addr;
 766       new_addr0 = inner_ip0->dst_address.as_u32;
 767
 768       sum0 = icmp0->checksum;
 769       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 770                              dst_address /* changed member */);
 771       icmp0->checksum = ip_csum_fold (sum0);
 772
 773       switch (protocol)
 774         {
 775           case SNAT_PROTOCOL_ICMP:
 776             inner_icmp0 = (icmp46_header_t*)l4_header;
 777             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 778
 779             old_id0 = inner_echo0->identifier;
 780             new_id0 = sm0.port;
 781             inner_echo0->identifier = new_id0;
 782
 783             sum0 = icmp0->checksum;
 784             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 785                                    identifier);
 786             icmp0->checksum = ip_csum_fold (sum0);
 787             break;
 788           case SNAT_PROTOCOL_UDP:
 789           case SNAT_PROTOCOL_TCP:
 790             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 791             new_id0 = sm0.port;
 792             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 793
 794             sum0 = icmp0->checksum;
 795             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 796                                    dst_port);
 797             icmp0->checksum = ip_csum_fold (sum0);
 798             break;
 799           default:
 800             ASSERT(0);
 801         }
 802     }
 803
 804 out:
 805   return next0;
 806 }
 807
 808 /**
 809  * @brief Hairpinning
 810  *
 811  * Hairpinning allows two endpoints on the internal side of the NAT to
 812  * communicate even if they only use each other's external IP addresses
 813  * and ports.
 814  *
 815  * @param sm     NAT main.
 816  * @param b0     Vlib buffer.
 817  * @param ip0    IP header.
 818  * @param udp0   UDP header.
 819  * @param tcp0   TCP header.
 820  * @param proto0 NAT protocol.
 821  */
 822 static inline void
 823 snat_hairpinning (snat_main_t *sm,
 824                   vlib_buffer_t * b0,
 825                   ip4_header_t * ip0,
 826                   udp_header_t * udp0,
 827                   tcp_header_t * tcp0,
 828                   u32 proto0)
 829 {
 830   snat_session_key_t key0, sm0;
 831   snat_worker_key_t k0;
 832   snat_session_t * s0;
 833   clib_bihash_kv_8_8_t kv0, value0;
 834   ip_csum_t sum0;
 835   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 836   u16 new_dst_port0, old_dst_port0;
 837
 838   key0.addr = ip0->dst_address;
 839   key0.port = udp0->dst_port;
 840   key0.protocol = proto0;
 841   key0.fib_index = sm->outside_fib_index;
 842   kv0.key = key0.as_u64;
 843
 844   /* Check if destination is in active sessions */
 845   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 846     {
 847       /* or static mappings */
 848       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 849         {
 850           new_dst_addr0 = sm0.addr.as_u32;
 851           new_dst_port0 = sm0.port;
 852           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 853         }
 854     }
 855   else
 856     {
 857       si = value0.value;
 858       if (sm->num_workers > 1)
 859         {
 860           k0.addr = ip0->dst_address;
 861           k0.port = udp0->dst_port;
 862           k0.fib_index = sm->outside_fib_index;
 863           kv0.key = k0.as_u64;
 864           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 865             ASSERT(0);
 866           else
 867             ti = value0.value;
 868         }
 869       else
 870         ti = sm->num_workers;
 871
 872       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 873       new_dst_addr0 = s0->in2out.addr.as_u32;
 874       new_dst_port0 = s0->in2out.port;
 875       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 876     }
 877
 878   /* Destination is behind the same NAT, use internal address and port */
 879   if (new_dst_addr0)
 880     {
 881       old_dst_addr0 = ip0->dst_address.as_u32;
 882       ip0->dst_address.as_u32 = new_dst_addr0;
 883       sum0 = ip0->checksum;
 884       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 885                              ip4_header_t, dst_address);
 886       ip0->checksum = ip_csum_fold (sum0);
 887
 888       old_dst_port0 = tcp0->dst;
 889       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 890         {
 891           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 892             {
 893               tcp0->dst = new_dst_port0;
 894               sum0 = tcp0->checksum;
 895               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 896                                      ip4_header_t, dst_address);
 897               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 898                                      ip4_header_t /* cheat */, length);
 899               tcp0->checksum = ip_csum_fold(sum0);
 900             }
 901           else
 902             {
 903               udp0->dst_port = new_dst_port0;
 904               udp0->checksum = 0;
 905             }
 906         }
 907       else
 908         {
 909           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 910             {
 911               sum0 = tcp0->checksum;
 912               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 913                                      ip4_header_t, dst_address);
 914               tcp0->checksum = ip_csum_fold(sum0);
 915             }
 916         }
 917     }
 918 }
 919
 920 static inline void
 921 snat_icmp_hairpinning (snat_main_t *sm,
 922                        vlib_buffer_t * b0,
 923                        ip4_header_t * ip0,
 924                        icmp46_header_t * icmp0)
 925 {
 926   snat_session_key_t key0, sm0;
 927   clib_bihash_kv_8_8_t kv0, value0;
 928   snat_worker_key_t k0;
 929   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 930   ip_csum_t sum0;
 931   snat_session_t *s0;
 932
 933   if (!icmp_is_error_message (icmp0))
 934     {
 935       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 936       u16 icmp_id0 = echo0->identifier;
 937       key0.addr = ip0->dst_address;
 938       key0.port = icmp_id0;
 939       key0.protocol = SNAT_PROTOCOL_ICMP;
 940       key0.fib_index = sm->outside_fib_index;
 941       kv0.key = key0.as_u64;
 942
 943       /* Check if destination is in active sessions */
 944       if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 945         {
 946           /* or static mappings */
 947           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 948             {
 949               new_dst_addr0 = sm0.addr.as_u32;
 950               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 951             }
 952         }
 953       else
 954         {
 955           si = value0.value;
 956           if (sm->num_workers > 1)
 957             {
 958               k0.addr = ip0->dst_address;
 959               k0.port = icmp_id0;
 960               k0.fib_index = sm->outside_fib_index;
 961               kv0.key = k0.as_u64;
 962               if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 963                 ASSERT(0);
 964               else
 965                 ti = value0.value;
 966             }
 967           else
 968             ti = sm->num_workers;
 969
 970           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 971           new_dst_addr0 = s0->in2out.addr.as_u32;
 972           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 973           echo0->identifier = s0->in2out.port;
 974           sum0 = icmp0->checksum;
 975           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 976                                  icmp_echo_header_t, identifier);
 977           icmp0->checksum = ip_csum_fold (sum0);
 978         }
 979
 980       /* Destination is behind the same NAT, use internal address and port */
 981       if (new_dst_addr0)
 982         {
 983           old_dst_addr0 = ip0->dst_address.as_u32;
 984           ip0->dst_address.as_u32 = new_dst_addr0;
 985           sum0 = ip0->checksum;
 986           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 987                                  ip4_header_t, dst_address);
 988           ip0->checksum = ip_csum_fold (sum0);
 989         }
 990     }
 991
 992 }
 993
 994 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 995                                          vlib_buffer_t * b0,
 996                                          ip4_header_t * ip0,
 997                                          icmp46_header_t * icmp0,
 998                                          u32 sw_if_index0,
 999                                          u32 rx_fib_index0,
1000                                          vlib_node_runtime_t * node,
1001                                          u32 next0,
1002                                          f64 now,
1003                                          u32 thread_index,
1004                                          snat_session_t ** p_s0)
1005 {
1006   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1007                       next0, thread_index, p_s0, 0);
1008   snat_session_t * s0 = *p_s0;
1009   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1010     {
1011       /* Hairpinning */
1012       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1013         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1014       /* Accounting */
1015       s0->last_heard = now;
1016       s0->total_pkts++;
1017       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1018       /* Per-user LRU list maintenance for dynamic translations */
1019       if (!snat_is_session_static (s0))
1020         {
1021           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1022                              s0->per_user_index);
1023           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1024                               s0->per_user_list_head_index,
1025                               s0->per_user_index);
1026         }
1027     }
1028   return next0;
1029 }
1030 static inline void
1031 snat_hairpinning_unknown_proto (snat_main_t *sm,
1032                                 vlib_buffer_t * b,
1033                                 ip4_header_t * ip)
1034 {
1035   u32 old_addr, new_addr = 0, ti = 0;
1036   clib_bihash_kv_8_8_t kv, value;
1037   clib_bihash_kv_16_8_t s_kv, s_value;
1038   nat_ed_ses_key_t key;
1039   snat_session_key_t m_key;
1040   snat_worker_key_t w_key;
1041   snat_static_mapping_t *m;
1042   ip_csum_t sum;
1043   snat_session_t *s;
1044
1045   old_addr = ip->dst_address.as_u32;
1046   key.l_addr.as_u32 = ip->dst_address.as_u32;
1047   key.r_addr.as_u32 = ip->src_address.as_u32;
1048   key.fib_index = sm->outside_fib_index;
1049   key.proto = ip->protocol;
1050   key.rsvd = 0;
1051   key.l_port = 0;
1052   s_kv.key[0] = key.as_u64[0];
1053   s_kv.key[1] = key.as_u64[1];
1054   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1055     {
1056       m_key.addr = ip->dst_address;
1057       m_key.fib_index = sm->outside_fib_index;
1058       m_key.port = 0;
1059       m_key.protocol = 0;
1060       kv.key = m_key.as_u64;
1061       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1062         return;
1063
1064       m = pool_elt_at_index (sm->static_mappings, value.value);
1065       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1066         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1067       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1068     }
1069   else
1070     {
1071       if (sm->num_workers > 1)
1072         {
1073           w_key.addr = ip->dst_address;
1074           w_key.port = 0;
1075           w_key.fib_index = sm->outside_fib_index;
1076           kv.key = w_key.as_u64;
1077           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv, &value))
1078             return;
1079           else
1080             ti = value.value;
1081         }
1082       else
1083         ti = sm->num_workers;
1084
1085       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1086       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1087         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1088       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1089     }
1090   sum = ip->checksum;
1091   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1092   ip->checksum = ip_csum_fold (sum);
1093 }
1094
1095 static void
1096 snat_in2out_unknown_proto (snat_main_t *sm,
1097                            vlib_buffer_t * b,
1098                            ip4_header_t * ip,
1099                            u32 rx_fib_index,
1100                            u32 thread_index,
1101                            f64 now,
1102                            vlib_main_t * vm)
1103 {
1104   clib_bihash_kv_8_8_t kv, value;
1105   clib_bihash_kv_16_8_t s_kv, s_value;
1106   snat_static_mapping_t *m;
1107   snat_session_key_t m_key;
1108   u32 old_addr, new_addr = 0;
1109   ip_csum_t sum;
1110   snat_user_key_t u_key;
1111   snat_user_t *u;
1112   dlist_elt_t *head, *elt, *oldest;
1113   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1114   u32 elt_index, head_index, ses_index, oldest_index;
1115   snat_session_t * s;
1116   nat_ed_ses_key_t key;
1117   u32 address_index = ~0;
1118   int i;
1119   u8 is_sm = 0;
1120
1121   old_addr = ip->src_address.as_u32;
1122
1123   key.l_addr = ip->src_address;
1124   key.r_addr = ip->dst_address;
1125   key.fib_index = rx_fib_index;
1126   key.proto = ip->protocol;
1127   key.rsvd = 0;
1128   key.l_port = 0;
1129   s_kv.key[0] = key.as_u64[0];
1130   s_kv.key[1] = key.as_u64[1];
1131
1132   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1133     {
1134       s = pool_elt_at_index (tsm->sessions, s_value.value);
1135       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1136     }
1137   else
1138     {
1139       u_key.addr = ip->src_address;
1140       u_key.fib_index = rx_fib_index;
1141       kv.key = u_key.as_u64;
1142
1143       /* Ever heard of the "user" = src ip4 address before? */
1144       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1145         {
1146           /* no, make a new one */
1147           pool_get (tsm->users, u);
1148           memset (u, 0, sizeof (*u));
1149           u->addr = ip->src_address;
1150           u->fib_index = rx_fib_index;
1151
1152           pool_get (tsm->list_pool, head);
1153           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1154
1155           clib_dlist_init (tsm->list_pool,
1156                            u->sessions_per_user_list_head_index);
1157
1158           kv.value = u - tsm->users;
1159
1160           /* add user */
1161           clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
1162         }
1163       else
1164         {
1165           u = pool_elt_at_index (tsm->users, value.value);
1166         }
1167
1168       m_key.addr = ip->src_address;
1169       m_key.port = 0;
1170       m_key.protocol = 0;
1171       m_key.fib_index = rx_fib_index;
1172       kv.key = m_key.as_u64;
1173
1174       /* Try to find static mapping first */
1175       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1176         {
1177           m = pool_elt_at_index (sm->static_mappings, value.value);
1178           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1179           is_sm = 1;
1180           goto create_ses;
1181         }
1182       /* Fallback to 3-tuple key */
1183       else
1184         {
1185           /* Choose same out address as for TCP/UDP session to same destination */
1186           if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1187             {
1188               head_index = u->sessions_per_user_list_head_index;
1189               head = pool_elt_at_index (tsm->list_pool, head_index);
1190               elt_index = head->next;
1191               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1192               ses_index = elt->value;
1193               while (ses_index != ~0)
1194                 {
1195                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1196                   elt_index = elt->next;
1197                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1198                   ses_index = elt->value;
1199
1200                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1201                     {
1202                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1203                       address_index = s->outside_address_index;
1204
1205                       key.fib_index = sm->outside_fib_index;
1206                       key.l_addr.as_u32 = new_addr;
1207                       s_kv.key[0] = key.as_u64[0];
1208                       s_kv.key[1] = key.as_u64[1];
1209                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1210                         break;
1211
1212                       goto create_ses;
1213                     }
1214                 }
1215             }
1216           key.fib_index = sm->outside_fib_index;
1217           for (i = 0; i < vec_len (sm->addresses); i++)
1218             {
1219               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1220               s_kv.key[0] = key.as_u64[0];
1221               s_kv.key[1] = key.as_u64[1];
1222               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1223                 {
1224                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1225                   address_index = i;
1226                   goto create_ses;
1227                 }
1228             }
1229           return;
1230         }
1231
1232 create_ses:
1233       /* Over quota? Recycle the least recently used dynamic translation */
1234       if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1235         {
1236           /* Remove the oldest dynamic translation */
1237           do {
1238               oldest_index = clib_dlist_remove_head (
1239                 tsm->list_pool, u->sessions_per_user_list_head_index);
1240
1241               ASSERT (oldest_index != ~0);
1242
1243               /* add it back to the end of the LRU list */
1244               clib_dlist_addtail (tsm->list_pool,
1245                                   u->sessions_per_user_list_head_index,
1246                                   oldest_index);
1247               /* Get the list element */
1248               oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1249
1250               /* Get the session index from the list element */
1251               ses_index = oldest->value;
1252
1253               /* Get the session */
1254               s = pool_elt_at_index (tsm->sessions, ses_index);
1255           } while (snat_is_session_static (s));
1256
1257           if (snat_is_unk_proto_session (s))
1258             {
1259               /* Remove from lookup tables */
1260               key.l_addr = s->in2out.addr;
1261               key.r_addr = s->ext_host_addr;
1262               key.fib_index = s->in2out.fib_index;
1263               key.proto = s->in2out.port;
1264               s_kv.key[0] = key.as_u64[0];
1265               s_kv.key[1] = key.as_u64[1];
1266               if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 0))
1267                 clib_warning ("in2out key del failed");
1268
1269               key.l_addr = s->out2in.addr;
1270               key.fib_index = s->out2in.fib_index;
1271               s_kv.key[0] = key.as_u64[0];
1272               s_kv.key[1] = key.as_u64[1];
1273               if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 0))
1274                 clib_warning ("out2in key del failed");
1275             }
1276           else
1277             {
1278               /* log NAT event */
1279               snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1280                                                   s->out2in.addr.as_u32,
1281                                                   s->in2out.protocol,
1282                                                   s->in2out.port,
1283                                                   s->out2in.port,
1284                                                   s->in2out.fib_index);
1285
1286               snat_free_outside_address_and_port (sm, thread_index, &s->out2in,
1287                                                   s->outside_address_index);
1288
1289               /* Remove in2out, out2in keys */
1290               kv.key = s->in2out.as_u64;
1291               if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
1292                 clib_warning ("in2out key del failed");
1293               kv.key = s->out2in.as_u64;
1294               if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
1295                 clib_warning ("out2in key del failed");
1296             }
1297         }
1298       else
1299         {
1300           /* Create a new session */
1301           pool_get (tsm->sessions, s);
1302           memset (s, 0, sizeof (*s));
1303
1304           /* Create list elts */
1305           pool_get (tsm->list_pool, elt);
1306           clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1307           elt->value = s - tsm->sessions;
1308           s->per_user_index = elt - tsm->list_pool;
1309           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1310           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1311                               s->per_user_index);
1312         }
1313
1314       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1315       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1316       s->outside_address_index = address_index;
1317       s->out2in.addr.as_u32 = new_addr;
1318       s->out2in.fib_index = sm->outside_fib_index;
1319       s->in2out.addr.as_u32 = old_addr;
1320       s->in2out.fib_index = rx_fib_index;
1321       s->in2out.port = s->out2in.port = ip->protocol;
1322       if (is_sm)
1323         {
1324           u->nstaticsessions++;
1325           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1326         }
1327       else
1328         {
1329           u->nsessions++;
1330         }
1331
1332       /* Add to lookup tables */
1333       key.l_addr.as_u32 = old_addr;
1334       key.r_addr = ip->dst_address;
1335       key.proto = ip->protocol;
1336       key.fib_index = rx_fib_index;
1337       s_kv.key[0] = key.as_u64[0];
1338       s_kv.key[1] = key.as_u64[1];
1339       s_kv.value = s - tsm->sessions;
1340       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1341         clib_warning ("in2out key add failed");
1342
1343       key.l_addr.as_u32 = new_addr;
1344       key.fib_index = sm->outside_fib_index;
1345       s_kv.key[0] = key.as_u64[0];
1346       s_kv.key[1] = key.as_u64[1];
1347       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1348         clib_warning ("out2in key add failed");
1349   }
1350
1351   /* Update IP checksum */
1352   sum = ip->checksum;
1353   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1354   ip->checksum = ip_csum_fold (sum);
1355
1356   /* Accounting */
1357   s->last_heard = now;
1358   s->total_pkts++;
1359   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1360   /* Per-user LRU list maintenance */
1361   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1362   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1363                       s->per_user_index);
1364
1365   /* Hairpinning */
1366   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1367     snat_hairpinning_unknown_proto(sm, b, ip);
1368
1369   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1370     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1371 }
1372
1373 static snat_session_t *
1374 snat_in2out_lb (snat_main_t *sm,
1375                 vlib_buffer_t * b,
1376                 ip4_header_t * ip,
1377                 u32 rx_fib_index,
1378                 u32 thread_index,
1379                 f64 now,
1380                 vlib_main_t * vm)
1381 {
1382   nat_ed_ses_key_t key;
1383   clib_bihash_kv_16_8_t s_kv, s_value;
1384   udp_header_t *udp = ip4_next_header (ip);
1385   tcp_header_t *tcp = (tcp_header_t *) udp;
1386   snat_session_t *s = 0;
1387   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1388   u32 old_addr, new_addr;
1389   u16 new_port, old_port;
1390   ip_csum_t sum;
1391   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1392   snat_session_key_t e_key, l_key;
1393   clib_bihash_kv_8_8_t kv, value;
1394   snat_user_key_t u_key;
1395   snat_user_t *u;
1396   dlist_elt_t *head, *elt;
1397
1398   old_addr = ip->src_address.as_u32;
1399
1400   key.l_addr = ip->src_address;
1401   key.r_addr = ip->dst_address;
1402   key.fib_index = rx_fib_index;
1403   key.proto = ip->protocol;
1404   key.rsvd = 0;
1405   key.l_port = udp->src_port;
1406   s_kv.key[0] = key.as_u64[0];
1407   s_kv.key[1] = key.as_u64[1];
1408
1409   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1410     {
1411       s = pool_elt_at_index (tsm->sessions, s_value.value);
1412     }
1413   else
1414     {
1415       l_key.addr = ip->src_address;
1416       l_key.port = udp->src_port;
1417       l_key.protocol = proto;
1418       l_key.fib_index = rx_fib_index;
1419       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0))
1420         return 0;
1421
1422       u_key.addr = ip->src_address;
1423       u_key.fib_index = rx_fib_index;
1424       kv.key = u_key.as_u64;
1425
1426       /* Ever heard of the "user" = src ip4 address before? */
1427       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1428         {
1429           /* no, make a new one */
1430           pool_get (tsm->users, u);
1431           memset (u, 0, sizeof (*u));
1432           u->addr = ip->src_address;
1433           u->fib_index = rx_fib_index;
1434
1435           pool_get (tsm->list_pool, head);
1436           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1437
1438           clib_dlist_init (tsm->list_pool,
1439                            u->sessions_per_user_list_head_index);
1440
1441           kv.value = u - tsm->users;
1442
1443           /* add user */
1444           if (clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1))
1445             clib_warning ("user key add failed");
1446         }
1447       else
1448         {
1449           u = pool_elt_at_index (tsm->users, value.value);
1450         }
1451
1452       /* Create a new session */
1453       pool_get (tsm->sessions, s);
1454       memset (s, 0, sizeof (*s));
1455
1456       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1457       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1458       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1459       s->outside_address_index = ~0;
1460       s->in2out = l_key;
1461       s->out2in = e_key;
1462       u->nstaticsessions++;
1463
1464       /* Create list elts */
1465       pool_get (tsm->list_pool, elt);
1466       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1467       elt->value = s - tsm->sessions;
1468       s->per_user_index = elt - tsm->list_pool;
1469       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1470       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1471                           s->per_user_index);
1472
1473       /* Add to lookup tables */
1474       s_kv.value = s - tsm->sessions;
1475       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1476         clib_warning ("in2out-ed key add failed");
1477
1478       key.l_addr = e_key.addr;
1479       key.fib_index = e_key.fib_index;
1480       key.l_port = e_key.port;
1481       s_kv.key[0] = key.as_u64[0];
1482       s_kv.key[1] = key.as_u64[1];
1483       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1484         clib_warning ("out2in-ed key add failed");
1485     }
1486
1487   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1488
1489   /* Update IP checksum */
1490   sum = ip->checksum;
1491   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1492   ip->checksum = ip_csum_fold (sum);
1493
1494   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1495     {
1496       old_port = tcp->src_port;
1497       tcp->src_port = s->out2in.port;
1498       new_port = tcp->src_port;
1499
1500       sum = tcp->checksum;
1501       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1502       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1503       tcp->checksum = ip_csum_fold(sum);
1504     }
1505   else
1506     {
1507       udp->src_port = s->out2in.port;
1508       udp->checksum = 0;
1509     }
1510
1511   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1512     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1513
1514   /* Accounting */
1515   s->last_heard = now;
1516   s->total_pkts++;
1517   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1518   return s;
1519 }
1520
1521 static inline uword
1522 snat_in2out_node_fn_inline (vlib_main_t * vm,
1523                             vlib_node_runtime_t * node,
1524                             vlib_frame_t * frame, int is_slow_path,
1525                             int is_output_feature)
1526 {
1527   u32 n_left_from, * from, * to_next;
1528   snat_in2out_next_t next_index;
1529   u32 pkts_processed = 0;
1530   snat_main_t * sm = &snat_main;
1531   f64 now = vlib_time_now (vm);
1532   u32 stats_node_index;
1533   u32 thread_index = vlib_get_thread_index ();
1534
1535   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1536     snat_in2out_node.index;
1537
1538   from = vlib_frame_vector_args (frame);
1539   n_left_from = frame->n_vectors;
1540   next_index = node->cached_next_index;
1541
1542   while (n_left_from > 0)
1543     {
1544       u32 n_left_to_next;
1545
1546       vlib_get_next_frame (vm, node, next_index,
1547                            to_next, n_left_to_next);
1548
1549       while (n_left_from >= 4 && n_left_to_next >= 2)
1550         {
1551           u32 bi0, bi1;
1552           vlib_buffer_t * b0, * b1;
1553           u32 next0, next1;
1554           u32 sw_if_index0, sw_if_index1;
1555           ip4_header_t * ip0, * ip1;
1556           ip_csum_t sum0, sum1;
1557           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1558           u16 old_port0, new_port0, old_port1, new_port1;
1559           udp_header_t * udp0, * udp1;
1560           tcp_header_t * tcp0, * tcp1;
1561           icmp46_header_t * icmp0, * icmp1;
1562           snat_session_key_t key0, key1;
1563           u32 rx_fib_index0, rx_fib_index1;
1564           u32 proto0, proto1;
1565           snat_session_t * s0 = 0, * s1 = 0;
1566           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1567           u32 iph_offset0 = 0, iph_offset1 = 0;
1568
1569           /* Prefetch next iteration. */
1570           {
1571             vlib_buffer_t * p2, * p3;
1572
1573             p2 = vlib_get_buffer (vm, from[2]);
1574             p3 = vlib_get_buffer (vm, from[3]);
1575
1576             vlib_prefetch_buffer_header (p2, LOAD);
1577             vlib_prefetch_buffer_header (p3, LOAD);
1578
1579             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1580             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1581           }
1582
1583           /* speculatively enqueue b0 and b1 to the current next frame */
1584           to_next[0] = bi0 = from[0];
1585           to_next[1] = bi1 = from[1];
1586           from += 2;
1587           to_next += 2;
1588           n_left_from -= 2;
1589           n_left_to_next -= 2;
1590
1591           b0 = vlib_get_buffer (vm, bi0);
1592           b1 = vlib_get_buffer (vm, bi1);
1593
1594           if (is_output_feature)
1595             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1596
1597           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1598                  iph_offset0);
1599
1600           udp0 = ip4_next_header (ip0);
1601           tcp0 = (tcp_header_t *) udp0;
1602           icmp0 = (icmp46_header_t *) udp0;
1603
1604           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1605           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1606                                    sw_if_index0);
1607
1608           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1609
1610           if (PREDICT_FALSE(ip0->ttl == 1))
1611             {
1612               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1613               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1614                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1615                                            0);
1616               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1617               goto trace00;
1618             }
1619
1620           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1621
1622           /* Next configured feature, probably ip4-lookup */
1623           if (is_slow_path)
1624             {
1625               if (PREDICT_FALSE (proto0 == ~0))
1626                 {
1627                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1628                                              thread_index, now, vm);
1629                   goto trace00;
1630                 }
1631
1632               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1633                 {
1634                   next0 = icmp_in2out_slow_path
1635                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1636                      node, next0, now, thread_index, &s0);
1637                   goto trace00;
1638                 }
1639             }
1640           else
1641             {
1642               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1643                 {
1644                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1645                   goto trace00;
1646                 }
1647             }
1648
1649           key0.addr = ip0->src_address;
1650           key0.port = udp0->src_port;
1651           key0.protocol = proto0;
1652           key0.fib_index = rx_fib_index0;
1653
1654           kv0.key = key0.as_u64;
1655
1656           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
1657             {
1658               if (is_slow_path)
1659                 {
1660                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1661                       ip0, proto0, rx_fib_index0)) && !is_output_feature)
1662                     goto trace00;
1663
1664                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1665                                      &s0, node, next0, thread_index);
1666                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1667                     goto trace00;
1668                 }
1669               else
1670                 {
1671                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1672                   goto trace00;
1673                 }
1674             }
1675           else
1676             {
1677               if (PREDICT_FALSE (value0.value == ~0ULL))
1678                 {
1679                   if (is_slow_path)
1680                     {
1681                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1682                                      now, vm);
1683                       goto trace00;
1684                     }
1685                   else
1686                     {
1687                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1688                       goto trace00;
1689                     }
1690                 }
1691               else
1692                 {
1693                   s0 = pool_elt_at_index (
1694                     sm->per_thread_data[thread_index].sessions,
1695                     value0.value);
1696                 }
1697             }
1698
1699           old_addr0 = ip0->src_address.as_u32;
1700           ip0->src_address = s0->out2in.addr;
1701           new_addr0 = ip0->src_address.as_u32;
1702           if (!is_output_feature)
1703             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1704
1705           sum0 = ip0->checksum;
1706           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1707                                  ip4_header_t,
1708                                  src_address /* changed member */);
1709           ip0->checksum = ip_csum_fold (sum0);
1710
1711           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1712             {
1713               old_port0 = tcp0->src_port;
1714               tcp0->src_port = s0->out2in.port;
1715               new_port0 = tcp0->src_port;
1716
1717               sum0 = tcp0->checksum;
1718               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1719                                      ip4_header_t,
1720                                      dst_address /* changed member */);
1721               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1722                                      ip4_header_t /* cheat */,
1723                                      length /* changed member */);
1724               tcp0->checksum = ip_csum_fold(sum0);
1725             }
1726           else
1727             {
1728               old_port0 = udp0->src_port;
1729               udp0->src_port = s0->out2in.port;
1730               udp0->checksum = 0;
1731             }
1732
1733           /* Hairpinning */
1734           if (!is_output_feature)
1735             snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1736
1737           /* Accounting */
1738           s0->last_heard = now;
1739           s0->total_pkts++;
1740           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1741           /* Per-user LRU list maintenance for dynamic translation */
1742           if (!snat_is_session_static (s0))
1743             {
1744               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1745                                  s0->per_user_index);
1746               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1747                                   s0->per_user_list_head_index,
1748                                   s0->per_user_index);
1749             }
1750         trace00:
1751
1752           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1753                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1754             {
1755               snat_in2out_trace_t *t =
1756                  vlib_add_trace (vm, node, b0, sizeof (*t));
1757               t->is_slow_path = is_slow_path;
1758               t->sw_if_index = sw_if_index0;
1759               t->next_index = next0;
1760                   t->session_index = ~0;
1761               if (s0)
1762                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1763             }
1764
1765           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1766
1767           if (is_output_feature)
1768             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1769
1770           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1771                  iph_offset1);
1772
1773           udp1 = ip4_next_header (ip1);
1774           tcp1 = (tcp_header_t *) udp1;
1775           icmp1 = (icmp46_header_t *) udp1;
1776
1777           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1778           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1779                                    sw_if_index1);
1780
1781           if (PREDICT_FALSE(ip1->ttl == 1))
1782             {
1783               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1784               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1785                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1786                                            0);
1787               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1788               goto trace01;
1789             }
1790
1791           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1792
1793           /* Next configured feature, probably ip4-lookup */
1794           if (is_slow_path)
1795             {
1796               if (PREDICT_FALSE (proto1 == ~0))
1797                 {
1798                   snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1799                                              thread_index, now, vm);
1800                   goto trace01;
1801                 }
1802
1803               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1804                 {
1805                   next1 = icmp_in2out_slow_path
1806                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1807                      next1, now, thread_index, &s1);
1808                   goto trace01;
1809                 }
1810             }
1811           else
1812             {
1813               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1814                 {
1815                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1816                   goto trace01;
1817                 }
1818             }
1819
1820           key1.addr = ip1->src_address;
1821           key1.port = udp1->src_port;
1822           key1.protocol = proto1;
1823           key1.fib_index = rx_fib_index1;
1824
1825           kv1.key = key1.as_u64;
1826
1827             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
1828             {
1829               if (is_slow_path)
1830                 {
1831                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1832                       ip1, proto1, rx_fib_index1)) && !is_output_feature)
1833                     goto trace01;
1834
1835                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1836                                      &s1, node, next1, thread_index);
1837                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1838                     goto trace01;
1839                 }
1840               else
1841                 {
1842                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1843                   goto trace01;
1844                 }
1845             }
1846           else
1847             {
1848               if (PREDICT_FALSE (value1.value == ~0ULL))
1849                 {
1850                   if (is_slow_path)
1851                     {
1852                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1853                                      now, vm);
1854                       goto trace01;
1855                     }
1856                   else
1857                     {
1858                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1859                       goto trace01;
1860                     }
1861                 }
1862               else
1863                 {
1864                   s1 = pool_elt_at_index (
1865                     sm->per_thread_data[thread_index].sessions,
1866                     value1.value);
1867                 }
1868             }
1869
1870           old_addr1 = ip1->src_address.as_u32;
1871           ip1->src_address = s1->out2in.addr;
1872           new_addr1 = ip1->src_address.as_u32;
1873           if (!is_output_feature)
1874             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1875
1876           sum1 = ip1->checksum;
1877           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1878                                  ip4_header_t,
1879                                  src_address /* changed member */);
1880           ip1->checksum = ip_csum_fold (sum1);
1881
1882           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1883             {
1884               old_port1 = tcp1->src_port;
1885               tcp1->src_port = s1->out2in.port;
1886               new_port1 = tcp1->src_port;
1887
1888               sum1 = tcp1->checksum;
1889               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1890                                      ip4_header_t,
1891                                      dst_address /* changed member */);
1892               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1893                                      ip4_header_t /* cheat */,
1894                                      length /* changed member */);
1895               tcp1->checksum = ip_csum_fold(sum1);
1896             }
1897           else
1898             {
1899               old_port1 = udp1->src_port;
1900               udp1->src_port = s1->out2in.port;
1901               udp1->checksum = 0;
1902             }
1903
1904           /* Hairpinning */
1905           if (!is_output_feature)
1906             snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1907
1908           /* Accounting */
1909           s1->last_heard = now;
1910           s1->total_pkts++;
1911           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1912           /* Per-user LRU list maintenance for dynamic translation */
1913           if (!snat_is_session_static (s1))
1914             {
1915               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1916                                  s1->per_user_index);
1917               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1918                                   s1->per_user_list_head_index,
1919                                   s1->per_user_index);
1920             }
1921         trace01:
1922
1923           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1924                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1925             {
1926               snat_in2out_trace_t *t =
1927                  vlib_add_trace (vm, node, b1, sizeof (*t));
1928               t->sw_if_index = sw_if_index1;
1929               t->next_index = next1;
1930               t->session_index = ~0;
1931               if (s1)
1932                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1933             }
1934
1935           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1936
1937           /* verify speculative enqueues, maybe switch current next frame */
1938           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1939                                            to_next, n_left_to_next,
1940                                            bi0, bi1, next0, next1);
1941         }
1942
1943       while (n_left_from > 0 && n_left_to_next > 0)
1944         {
1945           u32 bi0;
1946           vlib_buffer_t * b0;
1947           u32 next0;
1948           u32 sw_if_index0;
1949           ip4_header_t * ip0;
1950           ip_csum_t sum0;
1951           u32 new_addr0, old_addr0;
1952           u16 old_port0, new_port0;
1953           udp_header_t * udp0;
1954           tcp_header_t * tcp0;
1955           icmp46_header_t * icmp0;
1956           snat_session_key_t key0;
1957           u32 rx_fib_index0;
1958           u32 proto0;
1959           snat_session_t * s0 = 0;
1960           clib_bihash_kv_8_8_t kv0, value0;
1961           u32 iph_offset0 = 0;
1962
1963           /* speculatively enqueue b0 to the current next frame */
1964           bi0 = from[0];
1965           to_next[0] = bi0;
1966           from += 1;
1967           to_next += 1;
1968           n_left_from -= 1;
1969           n_left_to_next -= 1;
1970
1971           b0 = vlib_get_buffer (vm, bi0);
1972           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1973
1974           if (is_output_feature)
1975             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1976
1977           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1978                  iph_offset0);
1979
1980           udp0 = ip4_next_header (ip0);
1981           tcp0 = (tcp_header_t *) udp0;
1982           icmp0 = (icmp46_header_t *) udp0;
1983
1984           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1985           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1986                                    sw_if_index0);
1987
1988           if (PREDICT_FALSE(ip0->ttl == 1))
1989             {
1990               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1991               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1992                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1993                                            0);
1994               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1995               goto trace0;
1996             }
1997
1998           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1999
2000           /* Next configured feature, probably ip4-lookup */
2001           if (is_slow_path)
2002             {
2003               if (PREDICT_FALSE (proto0 == ~0))
2004                 {
2005                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2006                                              thread_index, now, vm);
2007                   goto trace0;
2008                 }
2009
2010               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2011                 {
2012                   next0 = icmp_in2out_slow_path
2013                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2014                      next0, now, thread_index, &s0);
2015                   goto trace0;
2016                 }
2017             }
2018           else
2019             {
2020               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2021                 {
2022                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2023                   goto trace0;
2024                 }
2025             }
2026
2027           key0.addr = ip0->src_address;
2028           key0.port = udp0->src_port;
2029           key0.protocol = proto0;
2030           key0.fib_index = rx_fib_index0;
2031
2032           kv0.key = key0.as_u64;
2033
2034           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
2035             {
2036               if (is_slow_path)
2037                 {
2038                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2039                       ip0, proto0, rx_fib_index0)) && !is_output_feature)
2040                     goto trace0;
2041
2042                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2043                                      &s0, node, next0, thread_index);
2044
2045                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2046                     goto trace0;
2047                 }
2048               else
2049                 {
2050                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2051                   goto trace0;
2052                 }
2053             }
2054           else
2055             {
2056               if (PREDICT_FALSE (value0.value == ~0ULL))
2057                 {
2058                   if (is_slow_path)
2059                     {
2060                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0, thread_index,
2061                                      now, vm);
2062                       goto trace0;
2063                     }
2064                   else
2065                     {
2066                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2067                       goto trace0;
2068                     }
2069                 }
2070               else
2071                 {
2072                   s0 = pool_elt_at_index (
2073                     sm->per_thread_data[thread_index].sessions,
2074                     value0.value);
2075                 }
2076             }
2077
2078           old_addr0 = ip0->src_address.as_u32;
2079           ip0->src_address = s0->out2in.addr;
2080           new_addr0 = ip0->src_address.as_u32;
2081           if (!is_output_feature)
2082             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2083
2084           sum0 = ip0->checksum;
2085           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2086                                  ip4_header_t,
2087                                  src_address /* changed member */);
2088           ip0->checksum = ip_csum_fold (sum0);
2089
2090           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2091             {
2092               old_port0 = tcp0->src_port;
2093               tcp0->src_port = s0->out2in.port;
2094               new_port0 = tcp0->src_port;
2095
2096               sum0 = tcp0->checksum;
2097               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2098                                      ip4_header_t,
2099                                      dst_address /* changed member */);
2100               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2101                                      ip4_header_t /* cheat */,
2102                                      length /* changed member */);
2103               tcp0->checksum = ip_csum_fold(sum0);
2104             }
2105           else
2106             {
2107               old_port0 = udp0->src_port;
2108               udp0->src_port = s0->out2in.port;
2109               udp0->checksum = 0;
2110             }
2111
2112           /* Hairpinning */
2113           if (!is_output_feature)
2114             snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2115
2116           /* Accounting */
2117           s0->last_heard = now;
2118           s0->total_pkts++;
2119           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2120           /* Per-user LRU list maintenance for dynamic translation */
2121           if (!snat_is_session_static (s0))
2122             {
2123               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2124                                  s0->per_user_index);
2125               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2126                                   s0->per_user_list_head_index,
2127                                   s0->per_user_index);
2128             }
2129
2130         trace0:
2131           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2132                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2133             {
2134               snat_in2out_trace_t *t =
2135                  vlib_add_trace (vm, node, b0, sizeof (*t));
2136               t->is_slow_path = is_slow_path;
2137               t->sw_if_index = sw_if_index0;
2138               t->next_index = next0;
2139                   t->session_index = ~0;
2140               if (s0)
2141                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2142             }
2143
2144           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2145
2146           /* verify speculative enqueue, maybe switch current next frame */
2147           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2148                                            to_next, n_left_to_next,
2149                                            bi0, next0);
2150         }
2151
2152       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2153     }
2154
2155   vlib_node_increment_counter (vm, stats_node_index,
2156                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2157                                pkts_processed);
2158   return frame->n_vectors;
2159 }
2160
2161 static uword
2162 snat_in2out_fast_path_fn (vlib_main_t * vm,
2163                           vlib_node_runtime_t * node,
2164                           vlib_frame_t * frame)
2165 {
2166   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2167 }
2168
2169 VLIB_REGISTER_NODE (snat_in2out_node) = {
2170   .function = snat_in2out_fast_path_fn,
2171   .name = "nat44-in2out",
2172   .vector_size = sizeof (u32),
2173   .format_trace = format_snat_in2out_trace,
2174   .type = VLIB_NODE_TYPE_INTERNAL,
2175
2176   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2177   .error_strings = snat_in2out_error_strings,
2178
2179   .runtime_data_bytes = sizeof (snat_runtime_t),
2180
2181   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2182
2183   /* edit / add dispositions here */
2184   .next_nodes = {
2185     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2186     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2187     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2188     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2189   },
2190 };
2191
2192 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2193
2194 static uword
2195 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2196                                  vlib_node_runtime_t * node,
2197                                  vlib_frame_t * frame)
2198 {
2199   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2200 }
2201
2202 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2203   .function = snat_in2out_output_fast_path_fn,
2204   .name = "nat44-in2out-output",
2205   .vector_size = sizeof (u32),
2206   .format_trace = format_snat_in2out_trace,
2207   .type = VLIB_NODE_TYPE_INTERNAL,
2208
2209   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2210   .error_strings = snat_in2out_error_strings,
2211
2212   .runtime_data_bytes = sizeof (snat_runtime_t),
2213
2214   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2215
2216   /* edit / add dispositions here */
2217   .next_nodes = {
2218     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2219     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2220     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2221     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2222   },
2223 };
2224
2225 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2226                               snat_in2out_output_fast_path_fn);
2227
2228 static uword
2229 snat_in2out_slow_path_fn (vlib_main_t * vm,
2230                           vlib_node_runtime_t * node,
2231                           vlib_frame_t * frame)
2232 {
2233   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2234 }
2235
2236 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2237   .function = snat_in2out_slow_path_fn,
2238   .name = "nat44-in2out-slowpath",
2239   .vector_size = sizeof (u32),
2240   .format_trace = format_snat_in2out_trace,
2241   .type = VLIB_NODE_TYPE_INTERNAL,
2242
2243   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2244   .error_strings = snat_in2out_error_strings,
2245
2246   .runtime_data_bytes = sizeof (snat_runtime_t),
2247
2248   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2249
2250   /* edit / add dispositions here */
2251   .next_nodes = {
2252     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2253     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2254     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2255     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2256   },
2257 };
2258
2259 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2260                               snat_in2out_slow_path_fn);
2261
2262 static uword
2263 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2264                                  vlib_node_runtime_t * node,
2265                                  vlib_frame_t * frame)
2266 {
2267   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2268 }
2269
2270 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2271   .function = snat_in2out_output_slow_path_fn,
2272   .name = "nat44-in2out-output-slowpath",
2273   .vector_size = sizeof (u32),
2274   .format_trace = format_snat_in2out_trace,
2275   .type = VLIB_NODE_TYPE_INTERNAL,
2276
2277   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2278   .error_strings = snat_in2out_error_strings,
2279
2280   .runtime_data_bytes = sizeof (snat_runtime_t),
2281
2282   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2283
2284   /* edit / add dispositions here */
2285   .next_nodes = {
2286     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2287     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2288     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2289     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2290   },
2291 };
2292
2293 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2294                               snat_in2out_output_slow_path_fn);
2295
2296 /**************************/
2297 /*** deterministic mode ***/
2298 /**************************/
2299 static uword
2300 snat_det_in2out_node_fn (vlib_main_t * vm,
2301                          vlib_node_runtime_t * node,
2302                          vlib_frame_t * frame)
2303 {
2304   u32 n_left_from, * from, * to_next;
2305   snat_in2out_next_t next_index;
2306   u32 pkts_processed = 0;
2307   snat_main_t * sm = &snat_main;
2308   u32 now = (u32) vlib_time_now (vm);
2309   u32 thread_index = vlib_get_thread_index ();
2310
2311   from = vlib_frame_vector_args (frame);
2312   n_left_from = frame->n_vectors;
2313   next_index = node->cached_next_index;
2314
2315   while (n_left_from > 0)
2316     {
2317       u32 n_left_to_next;
2318
2319       vlib_get_next_frame (vm, node, next_index,
2320                            to_next, n_left_to_next);
2321
2322       while (n_left_from >= 4 && n_left_to_next >= 2)
2323         {
2324           u32 bi0, bi1;
2325           vlib_buffer_t * b0, * b1;
2326           u32 next0, next1;
2327           u32 sw_if_index0, sw_if_index1;
2328           ip4_header_t * ip0, * ip1;
2329           ip_csum_t sum0, sum1;
2330           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2331           u16 old_port0, new_port0, lo_port0, i0;
2332           u16 old_port1, new_port1, lo_port1, i1;
2333           udp_header_t * udp0, * udp1;
2334           tcp_header_t * tcp0, * tcp1;
2335           u32 proto0, proto1;
2336           snat_det_out_key_t key0, key1;
2337           snat_det_map_t * dm0, * dm1;
2338           snat_det_session_t * ses0 = 0, * ses1 = 0;
2339           u32 rx_fib_index0, rx_fib_index1;
2340           icmp46_header_t * icmp0, * icmp1;
2341
2342           /* Prefetch next iteration. */
2343           {
2344             vlib_buffer_t * p2, * p3;
2345
2346             p2 = vlib_get_buffer (vm, from[2]);
2347             p3 = vlib_get_buffer (vm, from[3]);
2348
2349             vlib_prefetch_buffer_header (p2, LOAD);
2350             vlib_prefetch_buffer_header (p3, LOAD);
2351
2352             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2353             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2354           }
2355
2356           /* speculatively enqueue b0 and b1 to the current next frame */
2357           to_next[0] = bi0 = from[0];
2358           to_next[1] = bi1 = from[1];
2359           from += 2;
2360           to_next += 2;
2361           n_left_from -= 2;
2362           n_left_to_next -= 2;
2363
2364           b0 = vlib_get_buffer (vm, bi0);
2365           b1 = vlib_get_buffer (vm, bi1);
2366
2367           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2368           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2369
2370           ip0 = vlib_buffer_get_current (b0);
2371           udp0 = ip4_next_header (ip0);
2372           tcp0 = (tcp_header_t *) udp0;
2373
2374           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2375
2376           if (PREDICT_FALSE(ip0->ttl == 1))
2377             {
2378               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2379               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2380                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2381                                            0);
2382               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2383               goto trace0;
2384             }
2385
2386           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2387
2388           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2389             {
2390               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2391               icmp0 = (icmp46_header_t *) udp0;
2392
2393               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2394                                   rx_fib_index0, node, next0, thread_index,
2395                                   &ses0, &dm0);
2396               goto trace0;
2397             }
2398
2399           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2400           if (PREDICT_FALSE(!dm0))
2401             {
2402               clib_warning("no match for internal host %U",
2403                            format_ip4_address, &ip0->src_address);
2404               next0 = SNAT_IN2OUT_NEXT_DROP;
2405               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2406               goto trace0;
2407             }
2408
2409           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2410
2411           key0.ext_host_addr = ip0->dst_address;
2412           key0.ext_host_port = tcp0->dst;
2413
2414           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2415           if (PREDICT_FALSE(!ses0))
2416             {
2417               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2418                 {
2419                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2420                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2421
2422                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2423                     continue;
2424
2425                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2426                   break;
2427                 }
2428               if (PREDICT_FALSE(!ses0))
2429                 {
2430                   /* too many sessions for user, send ICMP error packet */
2431
2432                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2433                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2434                                                ICMP4_destination_unreachable_destination_unreachable_host,
2435                                                0);
2436                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2437                   goto trace0;
2438                 }
2439             }
2440
2441           new_port0 = ses0->out.out_port;
2442
2443           old_addr0.as_u32 = ip0->src_address.as_u32;
2444           ip0->src_address.as_u32 = new_addr0.as_u32;
2445           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2446
2447           sum0 = ip0->checksum;
2448           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2449                                  ip4_header_t,
2450                                  src_address /* changed member */);
2451           ip0->checksum = ip_csum_fold (sum0);
2452
2453           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2454             {
2455               if (tcp0->flags & TCP_FLAG_SYN)
2456                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2457               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2458                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2459               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2460                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2461               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2462                 snat_det_ses_close(dm0, ses0);
2463               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2464                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2465               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2466                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2467
2468               old_port0 = tcp0->src;
2469               tcp0->src = new_port0;
2470
2471               sum0 = tcp0->checksum;
2472               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2473                                      ip4_header_t,
2474                                      dst_address /* changed member */);
2475               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2476                                      ip4_header_t /* cheat */,
2477                                      length /* changed member */);
2478               tcp0->checksum = ip_csum_fold(sum0);
2479             }
2480           else
2481             {
2482               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2483               old_port0 = udp0->src_port;
2484               udp0->src_port = new_port0;
2485               udp0->checksum = 0;
2486             }
2487
2488           switch(ses0->state)
2489             {
2490             case SNAT_SESSION_UDP_ACTIVE:
2491                 ses0->expire = now + sm->udp_timeout;
2492                 break;
2493             case SNAT_SESSION_TCP_SYN_SENT:
2494             case SNAT_SESSION_TCP_FIN_WAIT:
2495             case SNAT_SESSION_TCP_CLOSE_WAIT:
2496             case SNAT_SESSION_TCP_LAST_ACK:
2497                 ses0->expire = now + sm->tcp_transitory_timeout;
2498                 break;
2499             case SNAT_SESSION_TCP_ESTABLISHED:
2500                 ses0->expire = now + sm->tcp_established_timeout;
2501                 break;
2502             }
2503
2504         trace0:
2505           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2506                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2507             {
2508               snat_in2out_trace_t *t =
2509                  vlib_add_trace (vm, node, b0, sizeof (*t));
2510               t->is_slow_path = 0;
2511               t->sw_if_index = sw_if_index0;
2512               t->next_index = next0;
2513               t->session_index = ~0;
2514               if (ses0)
2515                 t->session_index = ses0 - dm0->sessions;
2516             }
2517
2518           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2519
2520           ip1 = vlib_buffer_get_current (b1);
2521           udp1 = ip4_next_header (ip1);
2522           tcp1 = (tcp_header_t *) udp1;
2523
2524           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2525
2526           if (PREDICT_FALSE(ip1->ttl == 1))
2527             {
2528               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2529               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2530                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2531                                            0);
2532               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2533               goto trace1;
2534             }
2535
2536           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2537
2538           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2539             {
2540               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2541               icmp1 = (icmp46_header_t *) udp1;
2542
2543               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2544                                   rx_fib_index1, node, next1, thread_index,
2545                                   &ses1, &dm1);
2546               goto trace1;
2547             }
2548
2549           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2550           if (PREDICT_FALSE(!dm1))
2551             {
2552               clib_warning("no match for internal host %U",
2553                            format_ip4_address, &ip0->src_address);
2554               next1 = SNAT_IN2OUT_NEXT_DROP;
2555               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2556               goto trace1;
2557             }
2558
2559           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2560
2561           key1.ext_host_addr = ip1->dst_address;
2562           key1.ext_host_port = tcp1->dst;
2563
2564           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2565           if (PREDICT_FALSE(!ses1))
2566             {
2567               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2568                 {
2569                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2570                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2571
2572                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2573                     continue;
2574
2575                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2576                   break;
2577                 }
2578               if (PREDICT_FALSE(!ses1))
2579                 {
2580                   /* too many sessions for user, send ICMP error packet */
2581
2582                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2583                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2584                                                ICMP4_destination_unreachable_destination_unreachable_host,
2585                                                0);
2586                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2587                   goto trace1;
2588                 }
2589             }
2590
2591           new_port1 = ses1->out.out_port;
2592
2593           old_addr1.as_u32 = ip1->src_address.as_u32;
2594           ip1->src_address.as_u32 = new_addr1.as_u32;
2595           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2596
2597           sum1 = ip1->checksum;
2598           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2599                                  ip4_header_t,
2600                                  src_address /* changed member */);
2601           ip1->checksum = ip_csum_fold (sum1);
2602
2603           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2604             {
2605               if (tcp1->flags & TCP_FLAG_SYN)
2606                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2607               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2608                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2609               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2610                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2611               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2612                 snat_det_ses_close(dm1, ses1);
2613               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2614                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2615               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2616                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2617
2618               old_port1 = tcp1->src;
2619               tcp1->src = new_port1;
2620
2621               sum1 = tcp1->checksum;
2622               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2623                                      ip4_header_t,
2624                                      dst_address /* changed member */);
2625               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2626                                      ip4_header_t /* cheat */,
2627                                      length /* changed member */);
2628               tcp1->checksum = ip_csum_fold(sum1);
2629             }
2630           else
2631             {
2632               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2633               old_port1 = udp1->src_port;
2634               udp1->src_port = new_port1;
2635               udp1->checksum = 0;
2636             }
2637
2638           switch(ses1->state)
2639             {
2640             case SNAT_SESSION_UDP_ACTIVE:
2641                 ses1->expire = now + sm->udp_timeout;
2642                 break;
2643             case SNAT_SESSION_TCP_SYN_SENT:
2644             case SNAT_SESSION_TCP_FIN_WAIT:
2645             case SNAT_SESSION_TCP_CLOSE_WAIT:
2646             case SNAT_SESSION_TCP_LAST_ACK:
2647                 ses1->expire = now + sm->tcp_transitory_timeout;
2648                 break;
2649             case SNAT_SESSION_TCP_ESTABLISHED:
2650                 ses1->expire = now + sm->tcp_established_timeout;
2651                 break;
2652             }
2653
2654         trace1:
2655           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2656                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2657             {
2658               snat_in2out_trace_t *t =
2659                  vlib_add_trace (vm, node, b1, sizeof (*t));
2660               t->is_slow_path = 0;
2661               t->sw_if_index = sw_if_index1;
2662               t->next_index = next1;
2663               t->session_index = ~0;
2664               if (ses1)
2665                 t->session_index = ses1 - dm1->sessions;
2666             }
2667
2668           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2669
2670           /* verify speculative enqueues, maybe switch current next frame */
2671           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2672                                            to_next, n_left_to_next,
2673                                            bi0, bi1, next0, next1);
2674          }
2675
2676       while (n_left_from > 0 && n_left_to_next > 0)
2677         {
2678           u32 bi0;
2679           vlib_buffer_t * b0;
2680           u32 next0;
2681           u32 sw_if_index0;
2682           ip4_header_t * ip0;
2683           ip_csum_t sum0;
2684           ip4_address_t new_addr0, old_addr0;
2685           u16 old_port0, new_port0, lo_port0, i0;
2686           udp_header_t * udp0;
2687           tcp_header_t * tcp0;
2688           u32 proto0;
2689           snat_det_out_key_t key0;
2690           snat_det_map_t * dm0;
2691           snat_det_session_t * ses0 = 0;
2692           u32 rx_fib_index0;
2693           icmp46_header_t * icmp0;
2694
2695           /* speculatively enqueue b0 to the current next frame */
2696           bi0 = from[0];
2697           to_next[0] = bi0;
2698           from += 1;
2699           to_next += 1;
2700           n_left_from -= 1;
2701           n_left_to_next -= 1;
2702
2703           b0 = vlib_get_buffer (vm, bi0);
2704           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2705
2706           ip0 = vlib_buffer_get_current (b0);
2707           udp0 = ip4_next_header (ip0);
2708           tcp0 = (tcp_header_t *) udp0;
2709
2710           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2711
2712           if (PREDICT_FALSE(ip0->ttl == 1))
2713             {
2714               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2715               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2716                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2717                                            0);
2718               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2719               goto trace00;
2720             }
2721
2722           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2723
2724           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2725             {
2726               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2727               icmp0 = (icmp46_header_t *) udp0;
2728
2729               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2730                                   rx_fib_index0, node, next0, thread_index,
2731                                   &ses0, &dm0);
2732               goto trace00;
2733             }
2734
2735           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2736           if (PREDICT_FALSE(!dm0))
2737             {
2738               clib_warning("no match for internal host %U",
2739                            format_ip4_address, &ip0->src_address);
2740               next0 = SNAT_IN2OUT_NEXT_DROP;
2741               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2742               goto trace00;
2743             }
2744
2745           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2746
2747           key0.ext_host_addr = ip0->dst_address;
2748           key0.ext_host_port = tcp0->dst;
2749
2750           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2751           if (PREDICT_FALSE(!ses0))
2752             {
2753               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2754                 {
2755                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2756                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2757
2758                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2759                     continue;
2760
2761                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2762                   break;
2763                 }
2764               if (PREDICT_FALSE(!ses0))
2765                 {
2766                   /* too many sessions for user, send ICMP error packet */
2767
2768                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2769                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2770                                                ICMP4_destination_unreachable_destination_unreachable_host,
2771                                                0);
2772                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2773                   goto trace00;
2774                 }
2775             }
2776
2777           new_port0 = ses0->out.out_port;
2778
2779           old_addr0.as_u32 = ip0->src_address.as_u32;
2780           ip0->src_address.as_u32 = new_addr0.as_u32;
2781           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2782
2783           sum0 = ip0->checksum;
2784           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2785                                  ip4_header_t,
2786                                  src_address /* changed member */);
2787           ip0->checksum = ip_csum_fold (sum0);
2788
2789           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2790             {
2791               if (tcp0->flags & TCP_FLAG_SYN)
2792                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2793               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2794                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2795               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2796                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2797               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2798                 snat_det_ses_close(dm0, ses0);
2799               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2800                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2801               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2802                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2803
2804               old_port0 = tcp0->src;
2805               tcp0->src = new_port0;
2806
2807               sum0 = tcp0->checksum;
2808               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2809                                      ip4_header_t,
2810                                      dst_address /* changed member */);
2811               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2812                                      ip4_header_t /* cheat */,
2813                                      length /* changed member */);
2814               tcp0->checksum = ip_csum_fold(sum0);
2815             }
2816           else
2817             {
2818               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2819               old_port0 = udp0->src_port;
2820               udp0->src_port = new_port0;
2821               udp0->checksum = 0;
2822             }
2823
2824           switch(ses0->state)
2825             {
2826             case SNAT_SESSION_UDP_ACTIVE:
2827                 ses0->expire = now + sm->udp_timeout;
2828                 break;
2829             case SNAT_SESSION_TCP_SYN_SENT:
2830             case SNAT_SESSION_TCP_FIN_WAIT:
2831             case SNAT_SESSION_TCP_CLOSE_WAIT:
2832             case SNAT_SESSION_TCP_LAST_ACK:
2833                 ses0->expire = now + sm->tcp_transitory_timeout;
2834                 break;
2835             case SNAT_SESSION_TCP_ESTABLISHED:
2836                 ses0->expire = now + sm->tcp_established_timeout;
2837                 break;
2838             }
2839
2840         trace00:
2841           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2842                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2843             {
2844               snat_in2out_trace_t *t =
2845                  vlib_add_trace (vm, node, b0, sizeof (*t));
2846               t->is_slow_path = 0;
2847               t->sw_if_index = sw_if_index0;
2848               t->next_index = next0;
2849               t->session_index = ~0;
2850               if (ses0)
2851                 t->session_index = ses0 - dm0->sessions;
2852             }
2853
2854           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2855
2856           /* verify speculative enqueue, maybe switch current next frame */
2857           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2858                                            to_next, n_left_to_next,
2859                                            bi0, next0);
2860         }
2861
2862       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2863     }
2864
2865   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2866                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2867                                pkts_processed);
2868   return frame->n_vectors;
2869 }
2870
2871 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2872   .function = snat_det_in2out_node_fn,
2873   .name = "nat44-det-in2out",
2874   .vector_size = sizeof (u32),
2875   .format_trace = format_snat_in2out_trace,
2876   .type = VLIB_NODE_TYPE_INTERNAL,
2877
2878   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2879   .error_strings = snat_in2out_error_strings,
2880
2881   .runtime_data_bytes = sizeof (snat_runtime_t),
2882
2883   .n_next_nodes = 3,
2884
2885   /* edit / add dispositions here */
2886   .next_nodes = {
2887     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2888     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2889     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2890   },
2891 };
2892
2893 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2894
2895 /**
2896  * Get address and port values to be used for ICMP packet translation
2897  * and create session if needed
2898  *
2899  * @param[in,out] sm             NAT main
2900  * @param[in,out] node           NAT node runtime
2901  * @param[in] thread_index       thread index
2902  * @param[in,out] b0             buffer containing packet to be translated
2903  * @param[out] p_proto           protocol used for matching
2904  * @param[out] p_value           address and port after NAT translation
2905  * @param[out] p_dont_translate  if packet should not be translated
2906  * @param d                      optional parameter
2907  * @param e                      optional parameter
2908  */
2909 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
2910                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2911                           snat_session_key_t *p_value,
2912                           u8 *p_dont_translate, void *d, void *e)
2913 {
2914   ip4_header_t *ip0;
2915   icmp46_header_t *icmp0;
2916   u32 sw_if_index0;
2917   u32 rx_fib_index0;
2918   u8 protocol;
2919   snat_det_out_key_t key0;
2920   u8 dont_translate = 0;
2921   u32 next0 = ~0;
2922   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2923   ip4_header_t *inner_ip0;
2924   void *l4_header = 0;
2925   icmp46_header_t *inner_icmp0;
2926   snat_det_map_t * dm0 = 0;
2927   ip4_address_t new_addr0;
2928   u16 lo_port0, i0;
2929   snat_det_session_t * ses0 = 0;
2930   ip4_address_t in_addr;
2931   u16 in_port;
2932
2933   ip0 = vlib_buffer_get_current (b0);
2934   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2935   echo0 = (icmp_echo_header_t *)(icmp0+1);
2936   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2937   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2938
2939   if (!icmp_is_error_message (icmp0))
2940     {
2941       protocol = SNAT_PROTOCOL_ICMP;
2942       in_addr = ip0->src_address;
2943       in_port = echo0->identifier;
2944     }
2945   else
2946     {
2947       inner_ip0 = (ip4_header_t *)(echo0+1);
2948       l4_header = ip4_next_header (inner_ip0);
2949       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2950       in_addr = inner_ip0->dst_address;
2951       switch (protocol)
2952         {
2953         case SNAT_PROTOCOL_ICMP:
2954           inner_icmp0 = (icmp46_header_t*)l4_header;
2955           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2956           in_port = inner_echo0->identifier;
2957           break;
2958         case SNAT_PROTOCOL_UDP:
2959         case SNAT_PROTOCOL_TCP:
2960           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2961           break;
2962         default:
2963           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
2964           next0 = SNAT_IN2OUT_NEXT_DROP;
2965           goto out;
2966         }
2967     }
2968
2969   dm0 = snat_det_map_by_user(sm, &in_addr);
2970   if (PREDICT_FALSE(!dm0))
2971     {
2972       clib_warning("no match for internal host %U",
2973                    format_ip4_address, &in_addr);
2974       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2975           IP_PROTOCOL_ICMP, rx_fib_index0)))
2976         {
2977           dont_translate = 1;
2978           goto out;
2979         }
2980       next0 = SNAT_IN2OUT_NEXT_DROP;
2981       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2982       goto out;
2983     }
2984
2985   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
2986
2987   key0.ext_host_addr = ip0->dst_address;
2988   key0.ext_host_port = 0;
2989
2990   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
2991   if (PREDICT_FALSE(!ses0))
2992     {
2993       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2994           IP_PROTOCOL_ICMP, rx_fib_index0)))
2995         {
2996           dont_translate = 1;
2997           goto out;
2998         }
2999       if (icmp0->type != ICMP4_echo_request)
3000         {
3001           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3002           next0 = SNAT_IN2OUT_NEXT_DROP;
3003           goto out;
3004         }
3005       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3006         {
3007           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3008             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3009
3010           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3011             continue;
3012
3013           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3014           break;
3015         }
3016       if (PREDICT_FALSE(!ses0))
3017         {
3018           next0 = SNAT_IN2OUT_NEXT_DROP;
3019           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3020           goto out;
3021         }
3022     }
3023
3024   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3025                     !icmp_is_error_message (icmp0)))
3026     {
3027       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3028       next0 = SNAT_IN2OUT_NEXT_DROP;
3029       goto out;
3030     }
3031
3032   u32 now = (u32) vlib_time_now (sm->vlib_main);
3033
3034   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3035   ses0->expire = now + sm->icmp_timeout;
3036
3037 out:
3038   *p_proto = protocol;
3039   if (ses0)
3040     {
3041       p_value->addr = new_addr0;
3042       p_value->fib_index = sm->outside_fib_index;
3043       p_value->port = ses0->out.out_port;
3044     }
3045   *p_dont_translate = dont_translate;
3046   if (d)
3047     *(snat_det_session_t**)d = ses0;
3048   if (e)
3049     *(snat_det_map_t**)e = dm0;
3050   return next0;
3051 }
3052
3053 /**********************/
3054 /*** worker handoff ***/
3055 /**********************/
3056 static inline uword
3057 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3058                                       vlib_node_runtime_t * node,
3059                                       vlib_frame_t * frame,
3060                                       u8 is_output)
3061 {
3062   snat_main_t *sm = &snat_main;
3063   vlib_thread_main_t *tm = vlib_get_thread_main ();
3064   u32 n_left_from, *from, *to_next = 0;
3065   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3066   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3067     = 0;
3068   vlib_frame_queue_elt_t *hf = 0;
3069   vlib_frame_t *f = 0;
3070   int i;
3071   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3072   u32 next_worker_index = 0;
3073   u32 current_worker_index = ~0;
3074   u32 thread_index = vlib_get_thread_index ();
3075   u32 fq_index;
3076   u32 to_node_index;
3077
3078   ASSERT (vec_len (sm->workers));
3079
3080   if (is_output)
3081     {
3082       fq_index = sm->fq_in2out_output_index;
3083       to_node_index = sm->in2out_output_node_index;
3084     }
3085   else
3086     {
3087       fq_index = sm->fq_in2out_index;
3088       to_node_index = sm->in2out_node_index;
3089     }
3090
3091   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3092     {
3093       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3094
3095       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3096                                sm->first_worker_index + sm->num_workers - 1,
3097                                (vlib_frame_queue_t *) (~0));
3098     }
3099
3100   from = vlib_frame_vector_args (frame);
3101   n_left_from = frame->n_vectors;
3102
3103   while (n_left_from > 0)
3104     {
3105       u32 bi0;
3106       vlib_buffer_t *b0;
3107       u32 sw_if_index0;
3108       u32 rx_fib_index0;
3109       ip4_header_t * ip0;
3110       u8 do_handoff;
3111
3112       bi0 = from[0];
3113       from += 1;
3114       n_left_from -= 1;
3115
3116       b0 = vlib_get_buffer (vm, bi0);
3117
3118       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3119       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3120
3121       ip0 = vlib_buffer_get_current (b0);
3122
3123       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3124
3125       if (PREDICT_FALSE (next_worker_index != thread_index))
3126         {
3127           do_handoff = 1;
3128
3129           if (next_worker_index != current_worker_index)
3130             {
3131               if (hf)
3132                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3133
3134               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3135                                                       next_worker_index,
3136                                                       handoff_queue_elt_by_worker_index);
3137
3138               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3139               to_next_worker = &hf->buffer_index[hf->n_vectors];
3140               current_worker_index = next_worker_index;
3141             }
3142
3143           /* enqueue to correct worker thread */
3144           to_next_worker[0] = bi0;
3145           to_next_worker++;
3146           n_left_to_next_worker--;
3147
3148           if (n_left_to_next_worker == 0)
3149             {
3150               hf->n_vectors = VLIB_FRAME_SIZE;
3151               vlib_put_frame_queue_elt (hf);
3152               current_worker_index = ~0;
3153               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3154               hf = 0;
3155             }
3156         }
3157       else
3158         {
3159           do_handoff = 0;
3160           /* if this is 1st frame */
3161           if (!f)
3162             {
3163               f = vlib_get_frame_to_node (vm, to_node_index);
3164               to_next = vlib_frame_vector_args (f);
3165             }
3166
3167           to_next[0] = bi0;
3168           to_next += 1;
3169           f->n_vectors++;
3170         }
3171
3172       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3173                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3174         {
3175           snat_in2out_worker_handoff_trace_t *t =
3176             vlib_add_trace (vm, node, b0, sizeof (*t));
3177           t->next_worker_index = next_worker_index;
3178           t->do_handoff = do_handoff;
3179         }
3180     }
3181
3182   if (f)
3183     vlib_put_frame_to_node (vm, to_node_index, f);
3184
3185   if (hf)
3186     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3187
3188   /* Ship frames to the worker nodes */
3189   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3190     {
3191       if (handoff_queue_elt_by_worker_index[i])
3192         {
3193           hf = handoff_queue_elt_by_worker_index[i];
3194           /*
3195            * It works better to let the handoff node
3196            * rate-adapt, always ship the handoff queue element.
3197            */
3198           if (1 || hf->n_vectors == hf->last_n_vectors)
3199             {
3200               vlib_put_frame_queue_elt (hf);
3201               handoff_queue_elt_by_worker_index[i] = 0;
3202             }
3203           else
3204             hf->last_n_vectors = hf->n_vectors;
3205         }
3206       congested_handoff_queue_by_worker_index[i] =
3207         (vlib_frame_queue_t *) (~0);
3208     }
3209   hf = 0;
3210   current_worker_index = ~0;
3211   return frame->n_vectors;
3212 }
3213
3214 static uword
3215 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3216                                vlib_node_runtime_t * node,
3217                                vlib_frame_t * frame)
3218 {
3219   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3220 }
3221
3222 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3223   .function = snat_in2out_worker_handoff_fn,
3224   .name = "nat44-in2out-worker-handoff",
3225   .vector_size = sizeof (u32),
3226   .format_trace = format_snat_in2out_worker_handoff_trace,
3227   .type = VLIB_NODE_TYPE_INTERNAL,
3228
3229   .n_next_nodes = 1,
3230
3231   .next_nodes = {
3232     [0] = "error-drop",
3233   },
3234 };
3235
3236 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3237                               snat_in2out_worker_handoff_fn);
3238
3239 static uword
3240 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3241                                       vlib_node_runtime_t * node,
3242                                       vlib_frame_t * frame)
3243 {
3244   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3245 }
3246
3247 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3248   .function = snat_in2out_output_worker_handoff_fn,
3249   .name = "nat44-in2out-output-worker-handoff",
3250   .vector_size = sizeof (u32),
3251   .format_trace = format_snat_in2out_worker_handoff_trace,
3252   .type = VLIB_NODE_TYPE_INTERNAL,
3253
3254   .n_next_nodes = 1,
3255
3256   .next_nodes = {
3257     [0] = "error-drop",
3258   },
3259 };
3260
3261 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3262                               snat_in2out_output_worker_handoff_fn);
3263
3264 static_always_inline int
3265 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3266 {
3267   snat_address_t * ap;
3268   clib_bihash_kv_8_8_t kv, value;
3269   snat_session_key_t m_key;
3270
3271   vec_foreach (ap, sm->addresses)
3272     {
3273       if (ap->addr.as_u32 == dst_addr->as_u32)
3274         return 1;
3275     }
3276
3277   m_key.addr.as_u32 = dst_addr->as_u32;
3278   m_key.fib_index = sm->outside_fib_index;
3279   m_key.port = 0;
3280   m_key.protocol = 0;
3281   kv.key = m_key.as_u64;
3282   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3283     return 1;
3284
3285   return 0;
3286 }
3287
3288 static uword
3289 snat_hairpin_dst_fn (vlib_main_t * vm,
3290                      vlib_node_runtime_t * node,
3291                      vlib_frame_t * frame)
3292 {
3293   u32 n_left_from, * from, * to_next;
3294   snat_in2out_next_t next_index;
3295   u32 pkts_processed = 0;
3296   snat_main_t * sm = &snat_main;
3297
3298   from = vlib_frame_vector_args (frame);
3299   n_left_from = frame->n_vectors;
3300   next_index = node->cached_next_index;
3301
3302   while (n_left_from > 0)
3303     {
3304       u32 n_left_to_next;
3305
3306       vlib_get_next_frame (vm, node, next_index,
3307                            to_next, n_left_to_next);
3308
3309       while (n_left_from > 0 && n_left_to_next > 0)
3310         {
3311           u32 bi0;
3312           vlib_buffer_t * b0;
3313           u32 next0;
3314           ip4_header_t * ip0;
3315           u32 proto0;
3316
3317           /* speculatively enqueue b0 to the current next frame */
3318           bi0 = from[0];
3319           to_next[0] = bi0;
3320           from += 1;
3321           to_next += 1;
3322           n_left_from -= 1;
3323           n_left_to_next -= 1;
3324
3325           b0 = vlib_get_buffer (vm, bi0);
3326           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3327           ip0 = vlib_buffer_get_current (b0);
3328
3329           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3330
3331           vnet_buffer (b0)->snat.flags = 0;
3332           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3333             {
3334               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3335                 {
3336                   udp_header_t * udp0 = ip4_next_header (ip0);
3337                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3338
3339                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3340                 }
3341               else if (proto0 == SNAT_PROTOCOL_ICMP)
3342                 {
3343                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3344
3345                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3346                 }
3347               else
3348                 {
3349                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3350                 }
3351
3352               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3353               clib_warning("is hairpinning");
3354             }
3355
3356           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3357
3358           /* verify speculative enqueue, maybe switch current next frame */
3359           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3360                                            to_next, n_left_to_next,
3361                                            bi0, next0);
3362          }
3363
3364       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3365     }
3366
3367   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3368                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3369                                pkts_processed);
3370   return frame->n_vectors;
3371 }
3372
3373 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3374   .function = snat_hairpin_dst_fn,
3375   .name = "nat44-hairpin-dst",
3376   .vector_size = sizeof (u32),
3377   .type = VLIB_NODE_TYPE_INTERNAL,
3378   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3379   .error_strings = snat_in2out_error_strings,
3380   .n_next_nodes = 2,
3381   .next_nodes = {
3382     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3383     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3384   },
3385 };
3386
3387 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3388                               snat_hairpin_dst_fn);
3389
3390 static uword
3391 snat_hairpin_src_fn (vlib_main_t * vm,
3392                      vlib_node_runtime_t * node,
3393                      vlib_frame_t * frame)
3394 {
3395   u32 n_left_from, * from, * to_next;
3396   snat_in2out_next_t next_index;
3397   u32 pkts_processed = 0;
3398   snat_main_t *sm = &snat_main;
3399
3400   from = vlib_frame_vector_args (frame);
3401   n_left_from = frame->n_vectors;
3402   next_index = node->cached_next_index;
3403
3404   while (n_left_from > 0)
3405     {
3406       u32 n_left_to_next;
3407
3408       vlib_get_next_frame (vm, node, next_index,
3409                            to_next, n_left_to_next);
3410
3411       while (n_left_from > 0 && n_left_to_next > 0)
3412         {
3413           u32 bi0;
3414           vlib_buffer_t * b0;
3415           u32 next0;
3416           snat_interface_t *i;
3417           u32 sw_if_index0;
3418
3419           /* speculatively enqueue b0 to the current next frame */
3420           bi0 = from[0];
3421           to_next[0] = bi0;
3422           from += 1;
3423           to_next += 1;
3424           n_left_from -= 1;
3425           n_left_to_next -= 1;
3426
3427           b0 = vlib_get_buffer (vm, bi0);
3428           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3429           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3430
3431           pool_foreach (i, sm->output_feature_interfaces,
3432           ({
3433             /* Only packets from NAT inside interface */
3434             if ((i->is_inside == 1) && (sw_if_index0 == i->sw_if_index))
3435               {
3436                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3437                                     SNAT_FLAG_HAIRPINNING))
3438                   {
3439                     if (PREDICT_TRUE (sm->num_workers > 1))
3440                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3441                     else
3442                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3443                   }
3444                 break;
3445               }
3446           }));
3447
3448           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3449
3450           /* verify speculative enqueue, maybe switch current next frame */
3451           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3452                                            to_next, n_left_to_next,
3453                                            bi0, next0);
3454          }
3455
3456       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3457     }
3458
3459   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3460                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3461                                pkts_processed);
3462   return frame->n_vectors;
3463 }
3464
3465 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3466   .function = snat_hairpin_src_fn,
3467   .name = "nat44-hairpin-src",
3468   .vector_size = sizeof (u32),
3469   .type = VLIB_NODE_TYPE_INTERNAL,
3470   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3471   .error_strings = snat_in2out_error_strings,
3472   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3473   .next_nodes = {
3474      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3475      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3476      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3477      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3478   },
3479 };
3480
3481 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3482                               snat_hairpin_src_fn);
3483
3484 static uword
3485 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3486                                 vlib_node_runtime_t * node,
3487                                 vlib_frame_t * frame)
3488 {
3489   u32 n_left_from, * from, * to_next;
3490   snat_in2out_next_t next_index;
3491   u32 pkts_processed = 0;
3492   snat_main_t * sm = &snat_main;
3493   u32 stats_node_index;
3494
3495   stats_node_index = snat_in2out_fast_node.index;
3496
3497   from = vlib_frame_vector_args (frame);
3498   n_left_from = frame->n_vectors;
3499   next_index = node->cached_next_index;
3500
3501   while (n_left_from > 0)
3502     {
3503       u32 n_left_to_next;
3504
3505       vlib_get_next_frame (vm, node, next_index,
3506                            to_next, n_left_to_next);
3507
3508       while (n_left_from > 0 && n_left_to_next > 0)
3509         {
3510           u32 bi0;
3511           vlib_buffer_t * b0;
3512           u32 next0;
3513           u32 sw_if_index0;
3514           ip4_header_t * ip0;
3515           ip_csum_t sum0;
3516           u32 new_addr0, old_addr0;
3517           u16 old_port0, new_port0;
3518           udp_header_t * udp0;
3519           tcp_header_t * tcp0;
3520           icmp46_header_t * icmp0;
3521           snat_session_key_t key0, sm0;
3522           u32 proto0;
3523           u32 rx_fib_index0;
3524
3525           /* speculatively enqueue b0 to the current next frame */
3526           bi0 = from[0];
3527           to_next[0] = bi0;
3528           from += 1;
3529           to_next += 1;
3530           n_left_from -= 1;
3531           n_left_to_next -= 1;
3532
3533           b0 = vlib_get_buffer (vm, bi0);
3534           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3535
3536           ip0 = vlib_buffer_get_current (b0);
3537           udp0 = ip4_next_header (ip0);
3538           tcp0 = (tcp_header_t *) udp0;
3539           icmp0 = (icmp46_header_t *) udp0;
3540
3541           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3542           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3543
3544           if (PREDICT_FALSE(ip0->ttl == 1))
3545             {
3546               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3547               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3548                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3549                                            0);
3550               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3551               goto trace0;
3552             }
3553
3554           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3555
3556           if (PREDICT_FALSE (proto0 == ~0))
3557               goto trace0;
3558
3559           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3560             {
3561               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3562                                   rx_fib_index0, node, next0, ~0, 0, 0);
3563               goto trace0;
3564             }
3565
3566           key0.addr = ip0->src_address;
3567           key0.protocol = proto0;
3568           key0.port = udp0->src_port;
3569           key0.fib_index = rx_fib_index0;
3570
3571           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
3572             {
3573               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3574               next0= SNAT_IN2OUT_NEXT_DROP;
3575               goto trace0;
3576             }
3577
3578           new_addr0 = sm0.addr.as_u32;
3579           new_port0 = sm0.port;
3580           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3581           old_addr0 = ip0->src_address.as_u32;
3582           ip0->src_address.as_u32 = new_addr0;
3583
3584           sum0 = ip0->checksum;
3585           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3586                                  ip4_header_t,
3587                                  src_address /* changed member */);
3588           ip0->checksum = ip_csum_fold (sum0);
3589
3590           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3591             {
3592               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3593                 {
3594                   old_port0 = tcp0->src_port;
3595                   tcp0->src_port = new_port0;
3596
3597                   sum0 = tcp0->checksum;
3598                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3599                                          ip4_header_t,
3600                                          dst_address /* changed member */);
3601                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3602                                          ip4_header_t /* cheat */,
3603                                          length /* changed member */);
3604                   tcp0->checksum = ip_csum_fold(sum0);
3605                 }
3606               else
3607                 {
3608                   old_port0 = udp0->src_port;
3609                   udp0->src_port = new_port0;
3610                   udp0->checksum = 0;
3611                 }
3612             }
3613           else
3614             {
3615               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3616                 {
3617                   sum0 = tcp0->checksum;
3618                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3619                                          ip4_header_t,
3620                                          dst_address /* changed member */);
3621                   tcp0->checksum = ip_csum_fold(sum0);
3622                 }
3623             }
3624
3625           /* Hairpinning */
3626           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3627
3628         trace0:
3629           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3630                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3631             {
3632               snat_in2out_trace_t *t =
3633                  vlib_add_trace (vm, node, b0, sizeof (*t));
3634               t->sw_if_index = sw_if_index0;
3635               t->next_index = next0;
3636             }
3637
3638           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3639
3640           /* verify speculative enqueue, maybe switch current next frame */
3641           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3642                                            to_next, n_left_to_next,
3643                                            bi0, next0);
3644         }
3645
3646       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3647     }
3648
3649   vlib_node_increment_counter (vm, stats_node_index,
3650                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3651                                pkts_processed);
3652   return frame->n_vectors;
3653 }
3654
3655
3656 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3657   .function = snat_in2out_fast_static_map_fn,
3658   .name = "nat44-in2out-fast",
3659   .vector_size = sizeof (u32),
3660   .format_trace = format_snat_in2out_fast_trace,
3661   .type = VLIB_NODE_TYPE_INTERNAL,
3662
3663   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3664   .error_strings = snat_in2out_error_strings,
3665
3666   .runtime_data_bytes = sizeof (snat_runtime_t),
3667
3668   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3669
3670   /* edit / add dispositions here */
3671   .next_nodes = {
3672     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3673     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3674     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3675     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3676   },
3677 };
3678
3679 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);