src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91 vlib_node_registration_t snat_in2out_output_node;
  92 vlib_node_registration_t snat_in2out_output_slowpath_node;
  93 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
  94 vlib_node_registration_t snat_hairpin_dst_node;
  95 vlib_node_registration_t snat_hairpin_src_node;
  96 vlib_node_registration_t nat44_hairpinning_node;
  97
  98
  99 #define foreach_snat_in2out_error                       \
 100 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 101 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 102 _(OUT_OF_PORTS, "Out of ports")                         \
 103 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 104 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 105 _(NO_TRANSLATION, "No translation")                     \
 106 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")
 107
 108 typedef enum {
 109 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 110   foreach_snat_in2out_error
 111 #undef _
 112   SNAT_IN2OUT_N_ERROR,
 113 } snat_in2out_error_t;
 114
 115 static char * snat_in2out_error_strings[] = {
 116 #define _(sym,string) string,
 117   foreach_snat_in2out_error
 118 #undef _
 119 };
 120
 121 typedef enum {
 122   SNAT_IN2OUT_NEXT_LOOKUP,
 123   SNAT_IN2OUT_NEXT_DROP,
 124   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 125   SNAT_IN2OUT_NEXT_SLOW_PATH,
 126   SNAT_IN2OUT_N_NEXT,
 127 } snat_in2out_next_t;
 128
 129 typedef enum {
 130   SNAT_HAIRPIN_SRC_NEXT_DROP,
 131   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 132   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 133   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 134   SNAT_HAIRPIN_SRC_N_NEXT,
 135 } snat_hairpin_next_t;
 136
 137 /**
 138  * @brief Check if packet should be translated
 139  *
 140  * Packets aimed at outside interface and external addresss with active session
 141  * should be translated.
 142  *
 143  * @param sm            NAT main
 144  * @param rt            NAT runtime data
 145  * @param sw_if_index0  index of the inside interface
 146  * @param ip0           IPv4 header
 147  * @param proto0        NAT protocol
 148  * @param rx_fib_index0 RX FIB index
 149  *
 150  * @returns 0 if packet should be translated otherwise 1
 151  */
 152 static inline int
 153 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 154                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 155                          u32 rx_fib_index0)
 156 {
 157   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 158   fib_prefix_t pfx = {
 159     .fp_proto = FIB_PROTOCOL_IP4,
 160     .fp_len = 32,
 161     .fp_addr = {
 162         .ip4.as_u32 = ip0->dst_address.as_u32,
 163     },
 164   };
 165
 166   /* Don't NAT packet aimed at the intfc address */
 167   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 168                                       ip0->dst_address.as_u32)))
 169     return 1;
 170
 171   fei = fib_table_lookup (rx_fib_index0, &pfx);
 172   if (FIB_NODE_INDEX_INVALID != fei)
 173     {
 174       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 175       if (sw_if_index == ~0)
 176         {
 177           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 178           if (FIB_NODE_INDEX_INVALID != fei)
 179             sw_if_index = fib_entry_get_resolving_interface (fei);
 180         }
 181       snat_interface_t *i;
 182       pool_foreach (i, sm->interfaces,
 183       ({
 184         /* NAT packet aimed at outside interface */
 185         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 186           return 0;
 187       }));
 188     }
 189
 190   return 1;
 191 }
 192
 193 static inline int
 194 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 195                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 196                     u32 rx_fib_index0, u32 thread_index)
 197 {
 198   udp_header_t * udp0 = ip4_next_header (ip0);
 199   snat_session_key_t key0, sm0;
 200   clib_bihash_kv_8_8_t kv0, value0;
 201
 202   key0.addr = ip0->dst_address;
 203   key0.port = udp0->dst_port;
 204   key0.protocol = proto0;
 205   key0.fib_index = sm->outside_fib_index;
 206   kv0.key = key0.as_u64;
 207
 208   /* NAT packet aimed at external address if */
 209   /* has active sessions */
 210   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 211                               &value0))
 212     {
 213       /* or is static mappings */
 214       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 215         return 0;
 216     }
 217   else
 218     return 0;
 219
 220   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 221                                  rx_fib_index0);
 222 }
 223
 224 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 225                       ip4_header_t * ip0,
 226                       u32 rx_fib_index0,
 227                       snat_session_key_t * key0,
 228                       snat_session_t ** sessionp,
 229                       vlib_node_runtime_t * node,
 230                       u32 next0,
 231                       u32 thread_index)
 232 {
 233   snat_user_t *u;
 234   snat_user_key_t user_key;
 235   snat_session_t *s;
 236   clib_bihash_kv_8_8_t kv0, value0;
 237   u32 oldest_per_user_translation_list_index;
 238   dlist_elt_t * oldest_per_user_translation_list_elt;
 239   dlist_elt_t * per_user_translation_list_elt;
 240   dlist_elt_t * per_user_list_head_elt;
 241   u32 session_index;
 242   snat_session_key_t key1;
 243   u32 address_index = ~0;
 244   u32 outside_fib_index;
 245   uword * p;
 246
 247   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 248     {
 249       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 250       return SNAT_IN2OUT_NEXT_DROP;
 251     }
 252
 253   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 254   if (! p)
 255     {
 256       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 257       return SNAT_IN2OUT_NEXT_DROP;
 258     }
 259   outside_fib_index = p[0];
 260
 261   key1.protocol = key0->protocol;
 262   user_key.addr = ip0->src_address;
 263   user_key.fib_index = rx_fib_index0;
 264   kv0.key = user_key.as_u64;
 265
 266   /* Ever heard of the "user" = src ip4 address before? */
 267   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 268                               &kv0, &value0))
 269     {
 270       /* no, make a new one */
 271       pool_get (sm->per_thread_data[thread_index].users, u);
 272       memset (u, 0, sizeof (*u));
 273       u->addr = ip0->src_address;
 274       u->fib_index = rx_fib_index0;
 275
 276       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 277
 278       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 279         sm->per_thread_data[thread_index].list_pool;
 280
 281       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 282                        u->sessions_per_user_list_head_index);
 283
 284       kv0.value = u - sm->per_thread_data[thread_index].users;
 285
 286       /* add user */
 287       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 288                                &kv0, 1 /* is_add */);
 289     }
 290   else
 291     {
 292       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 293                              value0.value);
 294     }
 295
 296   /* Over quota? Recycle the least recently used dynamic translation */
 297   if (u->nsessions >= sm->max_translations_per_user)
 298     {
 299       /* Remove the oldest dynamic translation */
 300       do {
 301           oldest_per_user_translation_list_index =
 302             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 303                                     u->sessions_per_user_list_head_index);
 304
 305           ASSERT (oldest_per_user_translation_list_index != ~0);
 306
 307           /* add it back to the end of the LRU list */
 308           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 309                               u->sessions_per_user_list_head_index,
 310                               oldest_per_user_translation_list_index);
 311           /* Get the list element */
 312           oldest_per_user_translation_list_elt =
 313             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 314                                oldest_per_user_translation_list_index);
 315
 316           /* Get the session index from the list element */
 317           session_index = oldest_per_user_translation_list_elt->value;
 318
 319           /* Get the session */
 320           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 321                                  session_index);
 322       } while (snat_is_session_static (s));
 323
 324       if (snat_is_unk_proto_session (s))
 325         {
 326           clib_bihash_kv_16_8_t up_kv;
 327           nat_ed_ses_key_t key;
 328
 329           /* Remove from lookup tables */
 330           key.l_addr = s->in2out.addr;
 331           key.r_addr = s->ext_host_addr;
 332           key.fib_index = s->in2out.fib_index;
 333           key.proto = s->in2out.port;
 334           key.rsvd = 0;
 335           key.l_port = 0;
 336           up_kv.key[0] = key.as_u64[0];
 337           up_kv.key[1] = key.as_u64[1];
 338           if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &up_kv, 0))
 339             clib_warning ("in2out key del failed");
 340
 341           key.l_addr = s->out2in.addr;
 342           key.fib_index = s->out2in.fib_index;
 343           up_kv.key[0] = key.as_u64[0];
 344           up_kv.key[1] = key.as_u64[1];
 345           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &up_kv, 0))
 346             clib_warning ("out2in key del failed");
 347         }
 348       else
 349         {
 350           /* Remove in2out, out2in keys */
 351           kv0.key = s->in2out.as_u64;
 352           if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out,
 353                                        &kv0, 0 /* is_add */))
 354               clib_warning ("in2out key delete failed");
 355           kv0.key = s->out2in.as_u64;
 356           if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in,
 357                                        &kv0, 0 /* is_add */))
 358               clib_warning ("out2in key delete failed");
 359
 360           /* log NAT event */
 361           snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 362                                               s->out2in.addr.as_u32,
 363                                               s->in2out.protocol,
 364                                               s->in2out.port,
 365                                               s->out2in.port,
 366                                               s->in2out.fib_index);
 367
 368           snat_free_outside_address_and_port
 369             (sm, thread_index, &s->out2in, s->outside_address_index);
 370         }
 371       s->outside_address_index = ~0;
 372
 373       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, thread_index,
 374                                                &key1, &address_index))
 375         {
 376           ASSERT(0);
 377
 378           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 379           return SNAT_IN2OUT_NEXT_DROP;
 380         }
 381       s->outside_address_index = address_index;
 382     }
 383   else
 384     {
 385       u8 static_mapping = 1;
 386
 387       /* First try to match static mapping by local address and port */
 388       if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
 389         {
 390           static_mapping = 0;
 391           /* Try to create dynamic translation */
 392           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0,
 393                                                    thread_index, &key1,
 394                                                    &address_index))
 395             {
 396               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 397               return SNAT_IN2OUT_NEXT_DROP;
 398             }
 399         }
 400
 401       /* Create a new session */
 402       pool_get (sm->per_thread_data[thread_index].sessions, s);
 403       memset (s, 0, sizeof (*s));
 404
 405       s->outside_address_index = address_index;
 406
 407       if (static_mapping)
 408         {
 409           u->nstaticsessions++;
 410           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 411         }
 412       else
 413         {
 414           u->nsessions++;
 415         }
 416
 417       /* Create list elts */
 418       pool_get (sm->per_thread_data[thread_index].list_pool,
 419                 per_user_translation_list_elt);
 420       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 421                        per_user_translation_list_elt -
 422                        sm->per_thread_data[thread_index].list_pool);
 423
 424       per_user_translation_list_elt->value =
 425         s - sm->per_thread_data[thread_index].sessions;
 426       s->per_user_index = per_user_translation_list_elt -
 427                           sm->per_thread_data[thread_index].list_pool;
 428       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 429
 430       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 431                           s->per_user_list_head_index,
 432                           per_user_translation_list_elt -
 433                           sm->per_thread_data[thread_index].list_pool);
 434    }
 435
 436   s->in2out = *key0;
 437   s->out2in = key1;
 438   s->out2in.protocol = key0->protocol;
 439   s->out2in.fib_index = outside_fib_index;
 440   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 441   *sessionp = s;
 442
 443   /* Add to translation hashes */
 444   kv0.key = s->in2out.as_u64;
 445   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 446   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 447                                1 /* is_add */))
 448       clib_warning ("in2out key add failed");
 449
 450   kv0.key = s->out2in.as_u64;
 451   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 452
 453   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 454                                1 /* is_add */))
 455       clib_warning ("out2in key add failed");
 456
 457   /* log NAT event */
 458   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 459                                       s->out2in.addr.as_u32,
 460                                       s->in2out.protocol,
 461                                       s->in2out.port,
 462                                       s->out2in.port,
 463                                       s->in2out.fib_index);
 464   return next0;
 465 }
 466
 467 static_always_inline
 468 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 469                                  snat_session_key_t *p_key0)
 470 {
 471   icmp46_header_t *icmp0;
 472   snat_session_key_t key0;
 473   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 474   ip4_header_t *inner_ip0 = 0;
 475   void *l4_header = 0;
 476   icmp46_header_t *inner_icmp0;
 477
 478   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 479   echo0 = (icmp_echo_header_t *)(icmp0+1);
 480
 481   if (!icmp_is_error_message (icmp0))
 482     {
 483       key0.protocol = SNAT_PROTOCOL_ICMP;
 484       key0.addr = ip0->src_address;
 485       key0.port = echo0->identifier;
 486     }
 487   else
 488     {
 489       inner_ip0 = (ip4_header_t *)(echo0+1);
 490       l4_header = ip4_next_header (inner_ip0);
 491       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 492       key0.addr = inner_ip0->dst_address;
 493       switch (key0.protocol)
 494         {
 495         case SNAT_PROTOCOL_ICMP:
 496           inner_icmp0 = (icmp46_header_t*)l4_header;
 497           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 498           key0.port = inner_echo0->identifier;
 499           break;
 500         case SNAT_PROTOCOL_UDP:
 501         case SNAT_PROTOCOL_TCP:
 502           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 503           break;
 504         default:
 505           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 506         }
 507     }
 508   *p_key0 = key0;
 509   return -1; /* success */
 510 }
 511
 512 /**
 513  * Get address and port values to be used for ICMP packet translation
 514  * and create session if needed
 515  *
 516  * @param[in,out] sm             NAT main
 517  * @param[in,out] node           NAT node runtime
 518  * @param[in] thread_index       thread index
 519  * @param[in,out] b0             buffer containing packet to be translated
 520  * @param[out] p_proto           protocol used for matching
 521  * @param[out] p_value           address and port after NAT translation
 522  * @param[out] p_dont_translate  if packet should not be translated
 523  * @param d                      optional parameter
 524  * @param e                      optional parameter
 525  */
 526 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 527                            u32 thread_index, vlib_buffer_t *b0,
 528                            ip4_header_t *ip0, u8 *p_proto,
 529                            snat_session_key_t *p_value,
 530                            u8 *p_dont_translate, void *d, void *e)
 531 {
 532   icmp46_header_t *icmp0;
 533   u32 sw_if_index0;
 534   u32 rx_fib_index0;
 535   snat_session_key_t key0;
 536   snat_session_t *s0 = 0;
 537   u8 dont_translate = 0;
 538   clib_bihash_kv_8_8_t kv0, value0;
 539   u32 next0 = ~0;
 540   int err;
 541
 542   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 543   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 544   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 545
 546   err = icmp_get_key (ip0, &key0);
 547   if (err != -1)
 548     {
 549       b0->error = node->errors[err];
 550       next0 = SNAT_IN2OUT_NEXT_DROP;
 551       goto out;
 552     }
 553   key0.fib_index = rx_fib_index0;
 554
 555   kv0.key = key0.as_u64;
 556
 557   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 558                               &value0))
 559     {
 560       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 561           IP_PROTOCOL_ICMP, rx_fib_index0, thread_index) &&
 562           vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
 563         {
 564           dont_translate = 1;
 565           goto out;
 566         }
 567
 568       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 569         {
 570           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 571           next0 = SNAT_IN2OUT_NEXT_DROP;
 572           goto out;
 573         }
 574
 575       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 576                          &s0, node, next0, thread_index);
 577
 578       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 579         goto out;
 580     }
 581   else
 582     {
 583       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 584                         icmp0->type != ICMP4_echo_reply &&
 585                         !icmp_is_error_message (icmp0)))
 586         {
 587           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 588           next0 = SNAT_IN2OUT_NEXT_DROP;
 589           goto out;
 590         }
 591
 592       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 593                               value0.value);
 594     }
 595
 596 out:
 597   *p_proto = key0.protocol;
 598   if (s0)
 599     *p_value = s0->out2in;
 600   *p_dont_translate = dont_translate;
 601   if (d)
 602     *(snat_session_t**)d = s0;
 603   return next0;
 604 }
 605
 606 /**
 607  * Get address and port values to be used for ICMP packet translation
 608  *
 609  * @param[in] sm                 NAT main
 610  * @param[in,out] node           NAT node runtime
 611  * @param[in] thread_index       thread index
 612  * @param[in,out] b0             buffer containing packet to be translated
 613  * @param[out] p_proto           protocol used for matching
 614  * @param[out] p_value           address and port after NAT translation
 615  * @param[out] p_dont_translate  if packet should not be translated
 616  * @param d                      optional parameter
 617  * @param e                      optional parameter
 618  */
 619 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 620                            u32 thread_index, vlib_buffer_t *b0,
 621                            ip4_header_t *ip0, u8 *p_proto,
 622                            snat_session_key_t *p_value,
 623                            u8 *p_dont_translate, void *d, void *e)
 624 {
 625   icmp46_header_t *icmp0;
 626   u32 sw_if_index0;
 627   u32 rx_fib_index0;
 628   snat_session_key_t key0;
 629   snat_session_key_t sm0;
 630   u8 dont_translate = 0;
 631   u8 is_addr_only;
 632   u32 next0 = ~0;
 633   int err;
 634
 635   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 636   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 637   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 638
 639   err = icmp_get_key (ip0, &key0);
 640   if (err != -1)
 641     {
 642       b0->error = node->errors[err];
 643       next0 = SNAT_IN2OUT_NEXT_DROP;
 644       goto out2;
 645     }
 646   key0.fib_index = rx_fib_index0;
 647
 648   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
 649     {
 650       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 651           IP_PROTOCOL_ICMP, rx_fib_index0)))
 652         {
 653           dont_translate = 1;
 654           goto out;
 655         }
 656
 657       if (icmp_is_error_message (icmp0))
 658         {
 659           next0 = SNAT_IN2OUT_NEXT_DROP;
 660           goto out;
 661         }
 662
 663       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 664       next0 = SNAT_IN2OUT_NEXT_DROP;
 665       goto out;
 666     }
 667
 668   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 669                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 670                     !icmp_is_error_message (icmp0)))
 671     {
 672       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 673       next0 = SNAT_IN2OUT_NEXT_DROP;
 674       goto out;
 675     }
 676
 677 out:
 678   *p_value = sm0;
 679 out2:
 680   *p_proto = key0.protocol;
 681   *p_dont_translate = dont_translate;
 682   return next0;
 683 }
 684
 685 static inline u32 icmp_in2out (snat_main_t *sm,
 686                                vlib_buffer_t * b0,
 687                                ip4_header_t * ip0,
 688                                icmp46_header_t * icmp0,
 689                                u32 sw_if_index0,
 690                                u32 rx_fib_index0,
 691                                vlib_node_runtime_t * node,
 692                                u32 next0,
 693                                u32 thread_index,
 694                                void *d,
 695                                void *e)
 696 {
 697   snat_session_key_t sm0;
 698   u8 protocol;
 699   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 700   ip4_header_t *inner_ip0;
 701   void *l4_header = 0;
 702   icmp46_header_t *inner_icmp0;
 703   u8 dont_translate;
 704   u32 new_addr0, old_addr0;
 705   u16 old_id0, new_id0;
 706   ip_csum_t sum0;
 707   u16 checksum0;
 708   u32 next0_tmp;
 709
 710   echo0 = (icmp_echo_header_t *)(icmp0+1);
 711
 712   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 713                                        &protocol, &sm0, &dont_translate, d, e);
 714   if (next0_tmp != ~0)
 715     next0 = next0_tmp;
 716   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 717     goto out;
 718
 719   sum0 = ip_incremental_checksum (0, icmp0,
 720                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 721   checksum0 = ~ip_csum_fold (sum0);
 722   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 723     {
 724       next0 = SNAT_IN2OUT_NEXT_DROP;
 725       goto out;
 726     }
 727
 728   old_addr0 = ip0->src_address.as_u32;
 729   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 730   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 731     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 732
 733   sum0 = ip0->checksum;
 734   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 735                          src_address /* changed member */);
 736   ip0->checksum = ip_csum_fold (sum0);
 737
 738   if (!icmp_is_error_message (icmp0))
 739     {
 740       new_id0 = sm0.port;
 741       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 742         {
 743           old_id0 = echo0->identifier;
 744           new_id0 = sm0.port;
 745           echo0->identifier = new_id0;
 746
 747           sum0 = icmp0->checksum;
 748           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 749                                  identifier);
 750           icmp0->checksum = ip_csum_fold (sum0);
 751         }
 752     }
 753   else
 754     {
 755       inner_ip0 = (ip4_header_t *)(echo0+1);
 756       l4_header = ip4_next_header (inner_ip0);
 757
 758       if (!ip4_header_checksum_is_valid (inner_ip0))
 759         {
 760           next0 = SNAT_IN2OUT_NEXT_DROP;
 761           goto out;
 762         }
 763
 764       old_addr0 = inner_ip0->dst_address.as_u32;
 765       inner_ip0->dst_address = sm0.addr;
 766       new_addr0 = inner_ip0->dst_address.as_u32;
 767
 768       sum0 = icmp0->checksum;
 769       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 770                              dst_address /* changed member */);
 771       icmp0->checksum = ip_csum_fold (sum0);
 772
 773       switch (protocol)
 774         {
 775           case SNAT_PROTOCOL_ICMP:
 776             inner_icmp0 = (icmp46_header_t*)l4_header;
 777             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 778
 779             old_id0 = inner_echo0->identifier;
 780             new_id0 = sm0.port;
 781             inner_echo0->identifier = new_id0;
 782
 783             sum0 = icmp0->checksum;
 784             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 785                                    identifier);
 786             icmp0->checksum = ip_csum_fold (sum0);
 787             break;
 788           case SNAT_PROTOCOL_UDP:
 789           case SNAT_PROTOCOL_TCP:
 790             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 791             new_id0 = sm0.port;
 792             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 793
 794             sum0 = icmp0->checksum;
 795             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 796                                    dst_port);
 797             icmp0->checksum = ip_csum_fold (sum0);
 798             break;
 799           default:
 800             ASSERT(0);
 801         }
 802     }
 803
 804 out:
 805   return next0;
 806 }
 807
 808 /**
 809  * @brief Hairpinning
 810  *
 811  * Hairpinning allows two endpoints on the internal side of the NAT to
 812  * communicate even if they only use each other's external IP addresses
 813  * and ports.
 814  *
 815  * @param sm     NAT main.
 816  * @param b0     Vlib buffer.
 817  * @param ip0    IP header.
 818  * @param udp0   UDP header.
 819  * @param tcp0   TCP header.
 820  * @param proto0 NAT protocol.
 821  */
 822 static inline int
 823 snat_hairpinning (snat_main_t *sm,
 824                   vlib_buffer_t * b0,
 825                   ip4_header_t * ip0,
 826                   udp_header_t * udp0,
 827                   tcp_header_t * tcp0,
 828                   u32 proto0)
 829 {
 830   snat_session_key_t key0, sm0;
 831   snat_session_t * s0;
 832   clib_bihash_kv_8_8_t kv0, value0;
 833   ip_csum_t sum0;
 834   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 835   u16 new_dst_port0, old_dst_port0;
 836
 837   key0.addr = ip0->dst_address;
 838   key0.port = udp0->dst_port;
 839   key0.protocol = proto0;
 840   key0.fib_index = sm->outside_fib_index;
 841   kv0.key = key0.as_u64;
 842
 843   /* Check if destination is static mappings */
 844   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 845     {
 846       new_dst_addr0 = sm0.addr.as_u32;
 847       new_dst_port0 = sm0.port;
 848       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 849     }
 850   /* or active session */
 851   else
 852     {
 853       if (sm->num_workers > 1)
 854         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 855       else
 856         ti = sm->num_workers;
 857
 858       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 859         {
 860           si = value0.value;
 861
 862           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 863           new_dst_addr0 = s0->in2out.addr.as_u32;
 864           new_dst_port0 = s0->in2out.port;
 865           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 866         }
 867     }
 868
 869   /* Destination is behind the same NAT, use internal address and port */
 870   if (new_dst_addr0)
 871     {
 872       old_dst_addr0 = ip0->dst_address.as_u32;
 873       ip0->dst_address.as_u32 = new_dst_addr0;
 874       sum0 = ip0->checksum;
 875       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 876                              ip4_header_t, dst_address);
 877       ip0->checksum = ip_csum_fold (sum0);
 878
 879       old_dst_port0 = tcp0->dst;
 880       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 881         {
 882           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 883             {
 884               tcp0->dst = new_dst_port0;
 885               sum0 = tcp0->checksum;
 886               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 887                                      ip4_header_t, dst_address);
 888               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 889                                      ip4_header_t /* cheat */, length);
 890               tcp0->checksum = ip_csum_fold(sum0);
 891             }
 892           else
 893             {
 894               udp0->dst_port = new_dst_port0;
 895               udp0->checksum = 0;
 896             }
 897         }
 898       else
 899         {
 900           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 901             {
 902               sum0 = tcp0->checksum;
 903               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 904                                      ip4_header_t, dst_address);
 905               tcp0->checksum = ip_csum_fold(sum0);
 906             }
 907         }
 908       return 1;
 909     }
 910   return 0;
 911 }
 912
 913 static inline void
 914 snat_icmp_hairpinning (snat_main_t *sm,
 915                        vlib_buffer_t * b0,
 916                        ip4_header_t * ip0,
 917                        icmp46_header_t * icmp0)
 918 {
 919   snat_session_key_t key0, sm0;
 920   clib_bihash_kv_8_8_t kv0, value0;
 921   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 922   ip_csum_t sum0;
 923   snat_session_t *s0;
 924
 925   if (!icmp_is_error_message (icmp0))
 926     {
 927       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 928       u16 icmp_id0 = echo0->identifier;
 929       key0.addr = ip0->dst_address;
 930       key0.port = icmp_id0;
 931       key0.protocol = SNAT_PROTOCOL_ICMP;
 932       key0.fib_index = sm->outside_fib_index;
 933       kv0.key = key0.as_u64;
 934
 935       if (sm->num_workers > 1)
 936         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 937       else
 938         ti = sm->num_workers;
 939
 940       /* Check if destination is in active sessions */
 941       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 942                                   &value0))
 943         {
 944           /* or static mappings */
 945           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 946             {
 947               new_dst_addr0 = sm0.addr.as_u32;
 948               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 949             }
 950         }
 951       else
 952         {
 953           si = value0.value;
 954
 955           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 956           new_dst_addr0 = s0->in2out.addr.as_u32;
 957           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 958           echo0->identifier = s0->in2out.port;
 959           sum0 = icmp0->checksum;
 960           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 961                                  icmp_echo_header_t, identifier);
 962           icmp0->checksum = ip_csum_fold (sum0);
 963         }
 964
 965       /* Destination is behind the same NAT, use internal address and port */
 966       if (new_dst_addr0)
 967         {
 968           old_dst_addr0 = ip0->dst_address.as_u32;
 969           ip0->dst_address.as_u32 = new_dst_addr0;
 970           sum0 = ip0->checksum;
 971           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 972                                  ip4_header_t, dst_address);
 973           ip0->checksum = ip_csum_fold (sum0);
 974         }
 975     }
 976
 977 }
 978
 979 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 980                                          vlib_buffer_t * b0,
 981                                          ip4_header_t * ip0,
 982                                          icmp46_header_t * icmp0,
 983                                          u32 sw_if_index0,
 984                                          u32 rx_fib_index0,
 985                                          vlib_node_runtime_t * node,
 986                                          u32 next0,
 987                                          f64 now,
 988                                          u32 thread_index,
 989                                          snat_session_t ** p_s0)
 990 {
 991   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 992                       next0, thread_index, p_s0, 0);
 993   snat_session_t * s0 = *p_s0;
 994   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 995     {
 996       /* Hairpinning */
 997       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
 998         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
 999       /* Accounting */
1000       s0->last_heard = now;
1001       s0->total_pkts++;
1002       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1003       /* Per-user LRU list maintenance for dynamic translations */
1004       if (!snat_is_session_static (s0))
1005         {
1006           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1007                              s0->per_user_index);
1008           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1009                               s0->per_user_list_head_index,
1010                               s0->per_user_index);
1011         }
1012     }
1013   return next0;
1014 }
1015 static inline void
1016 snat_hairpinning_unknown_proto (snat_main_t *sm,
1017                                 vlib_buffer_t * b,
1018                                 ip4_header_t * ip)
1019 {
1020   u32 old_addr, new_addr = 0, ti = 0;
1021   clib_bihash_kv_8_8_t kv, value;
1022   clib_bihash_kv_16_8_t s_kv, s_value;
1023   nat_ed_ses_key_t key;
1024   snat_session_key_t m_key;
1025   snat_static_mapping_t *m;
1026   ip_csum_t sum;
1027   snat_session_t *s;
1028
1029   old_addr = ip->dst_address.as_u32;
1030   key.l_addr.as_u32 = ip->dst_address.as_u32;
1031   key.r_addr.as_u32 = ip->src_address.as_u32;
1032   key.fib_index = sm->outside_fib_index;
1033   key.proto = ip->protocol;
1034   key.rsvd = 0;
1035   key.l_port = 0;
1036   s_kv.key[0] = key.as_u64[0];
1037   s_kv.key[1] = key.as_u64[1];
1038   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1039     {
1040       m_key.addr = ip->dst_address;
1041       m_key.fib_index = sm->outside_fib_index;
1042       m_key.port = 0;
1043       m_key.protocol = 0;
1044       kv.key = m_key.as_u64;
1045       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1046         return;
1047
1048       m = pool_elt_at_index (sm->static_mappings, value.value);
1049       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1050         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1051       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1052     }
1053   else
1054     {
1055       if (sm->num_workers > 1)
1056         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1057       else
1058         ti = sm->num_workers;
1059
1060       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1061       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1062         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1063       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1064     }
1065   sum = ip->checksum;
1066   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1067   ip->checksum = ip_csum_fold (sum);
1068 }
1069
1070 static snat_session_t *
1071 snat_in2out_unknown_proto (snat_main_t *sm,
1072                            vlib_buffer_t * b,
1073                            ip4_header_t * ip,
1074                            u32 rx_fib_index,
1075                            u32 thread_index,
1076                            f64 now,
1077                            vlib_main_t * vm,
1078                            vlib_node_runtime_t * node)
1079 {
1080   clib_bihash_kv_8_8_t kv, value;
1081   clib_bihash_kv_16_8_t s_kv, s_value;
1082   snat_static_mapping_t *m;
1083   snat_session_key_t m_key;
1084   u32 old_addr, new_addr = 0;
1085   ip_csum_t sum;
1086   snat_user_key_t u_key;
1087   snat_user_t *u;
1088   dlist_elt_t *head, *elt, *oldest;
1089   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1090   u32 elt_index, head_index, ses_index, oldest_index;
1091   snat_session_t * s;
1092   nat_ed_ses_key_t key;
1093   u32 address_index = ~0;
1094   int i;
1095   u8 is_sm = 0;
1096
1097   old_addr = ip->src_address.as_u32;
1098
1099   key.l_addr = ip->src_address;
1100   key.r_addr = ip->dst_address;
1101   key.fib_index = rx_fib_index;
1102   key.proto = ip->protocol;
1103   key.rsvd = 0;
1104   key.l_port = 0;
1105   s_kv.key[0] = key.as_u64[0];
1106   s_kv.key[1] = key.as_u64[1];
1107
1108   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1109     {
1110       s = pool_elt_at_index (tsm->sessions, s_value.value);
1111       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1112     }
1113   else
1114     {
1115       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1116         {
1117           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1118           return 0;
1119         }
1120
1121       u_key.addr = ip->src_address;
1122       u_key.fib_index = rx_fib_index;
1123       kv.key = u_key.as_u64;
1124
1125       /* Ever heard of the "user" = src ip4 address before? */
1126       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1127         {
1128           /* no, make a new one */
1129           pool_get (tsm->users, u);
1130           memset (u, 0, sizeof (*u));
1131           u->addr = ip->src_address;
1132           u->fib_index = rx_fib_index;
1133
1134           pool_get (tsm->list_pool, head);
1135           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1136
1137           clib_dlist_init (tsm->list_pool,
1138                            u->sessions_per_user_list_head_index);
1139
1140           kv.value = u - tsm->users;
1141
1142           /* add user */
1143           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
1144         }
1145       else
1146         {
1147           u = pool_elt_at_index (tsm->users, value.value);
1148         }
1149
1150       m_key.addr = ip->src_address;
1151       m_key.port = 0;
1152       m_key.protocol = 0;
1153       m_key.fib_index = rx_fib_index;
1154       kv.key = m_key.as_u64;
1155
1156       /* Try to find static mapping first */
1157       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1158         {
1159           m = pool_elt_at_index (sm->static_mappings, value.value);
1160           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1161           is_sm = 1;
1162           goto create_ses;
1163         }
1164       /* Fallback to 3-tuple key */
1165       else
1166         {
1167           /* Choose same out address as for TCP/UDP session to same destination */
1168           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1169             {
1170               head_index = u->sessions_per_user_list_head_index;
1171               head = pool_elt_at_index (tsm->list_pool, head_index);
1172               elt_index = head->next;
1173               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1174               ses_index = elt->value;
1175               while (ses_index != ~0)
1176                 {
1177                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1178                   elt_index = elt->next;
1179                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1180                   ses_index = elt->value;
1181
1182                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1183                     {
1184                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1185                       address_index = s->outside_address_index;
1186
1187                       key.fib_index = sm->outside_fib_index;
1188                       key.l_addr.as_u32 = new_addr;
1189                       s_kv.key[0] = key.as_u64[0];
1190                       s_kv.key[1] = key.as_u64[1];
1191                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1192                         break;
1193
1194                       goto create_ses;
1195                     }
1196                 }
1197             }
1198           key.fib_index = sm->outside_fib_index;
1199           for (i = 0; i < vec_len (sm->addresses); i++)
1200             {
1201               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1202               s_kv.key[0] = key.as_u64[0];
1203               s_kv.key[1] = key.as_u64[1];
1204               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1205                 {
1206                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1207                   address_index = i;
1208                   goto create_ses;
1209                 }
1210             }
1211           return 0;
1212         }
1213
1214 create_ses:
1215       /* Over quota? Recycle the least recently used dynamic translation */
1216       if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1217         {
1218           /* Remove the oldest dynamic translation */
1219           do {
1220               oldest_index = clib_dlist_remove_head (
1221                 tsm->list_pool, u->sessions_per_user_list_head_index);
1222
1223               ASSERT (oldest_index != ~0);
1224
1225               /* add it back to the end of the LRU list */
1226               clib_dlist_addtail (tsm->list_pool,
1227                                   u->sessions_per_user_list_head_index,
1228                                   oldest_index);
1229               /* Get the list element */
1230               oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1231
1232               /* Get the session index from the list element */
1233               ses_index = oldest->value;
1234
1235               /* Get the session */
1236               s = pool_elt_at_index (tsm->sessions, ses_index);
1237           } while (snat_is_session_static (s));
1238
1239           if (snat_is_unk_proto_session (s))
1240             {
1241               /* Remove from lookup tables */
1242               key.l_addr = s->in2out.addr;
1243               key.r_addr = s->ext_host_addr;
1244               key.fib_index = s->in2out.fib_index;
1245               key.proto = s->in2out.port;
1246               s_kv.key[0] = key.as_u64[0];
1247               s_kv.key[1] = key.as_u64[1];
1248               if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 0))
1249                 clib_warning ("in2out key del failed");
1250
1251               key.l_addr = s->out2in.addr;
1252               key.fib_index = s->out2in.fib_index;
1253               s_kv.key[0] = key.as_u64[0];
1254               s_kv.key[1] = key.as_u64[1];
1255               if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 0))
1256                 clib_warning ("out2in key del failed");
1257             }
1258           else
1259             {
1260               /* log NAT event */
1261               snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1262                                                   s->out2in.addr.as_u32,
1263                                                   s->in2out.protocol,
1264                                                   s->in2out.port,
1265                                                   s->out2in.port,
1266                                                   s->in2out.fib_index);
1267
1268               snat_free_outside_address_and_port (sm, thread_index, &s->out2in,
1269                                                   s->outside_address_index);
1270
1271               /* Remove in2out, out2in keys */
1272               kv.key = s->in2out.as_u64;
1273               if (clib_bihash_add_del_8_8 (
1274                     &sm->per_thread_data[thread_index].in2out, &kv, 0))
1275                 clib_warning ("in2out key del failed");
1276               kv.key = s->out2in.as_u64;
1277               if (clib_bihash_add_del_8_8 (
1278                     &sm->per_thread_data[thread_index].out2in, &kv, 0))
1279                 clib_warning ("out2in key del failed");
1280             }
1281         }
1282       else
1283         {
1284           /* Create a new session */
1285           pool_get (tsm->sessions, s);
1286           memset (s, 0, sizeof (*s));
1287
1288           /* Create list elts */
1289           pool_get (tsm->list_pool, elt);
1290           clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1291           elt->value = s - tsm->sessions;
1292           s->per_user_index = elt - tsm->list_pool;
1293           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1294           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1295                               s->per_user_index);
1296         }
1297
1298       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1299       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1300       s->outside_address_index = address_index;
1301       s->out2in.addr.as_u32 = new_addr;
1302       s->out2in.fib_index = sm->outside_fib_index;
1303       s->in2out.addr.as_u32 = old_addr;
1304       s->in2out.fib_index = rx_fib_index;
1305       s->in2out.port = s->out2in.port = ip->protocol;
1306       if (is_sm)
1307         {
1308           u->nstaticsessions++;
1309           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1310         }
1311       else
1312         {
1313           u->nsessions++;
1314         }
1315
1316       /* Add to lookup tables */
1317       key.l_addr.as_u32 = old_addr;
1318       key.r_addr = ip->dst_address;
1319       key.proto = ip->protocol;
1320       key.fib_index = rx_fib_index;
1321       s_kv.key[0] = key.as_u64[0];
1322       s_kv.key[1] = key.as_u64[1];
1323       s_kv.value = s - tsm->sessions;
1324       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1325         clib_warning ("in2out key add failed");
1326
1327       key.l_addr.as_u32 = new_addr;
1328       key.fib_index = sm->outside_fib_index;
1329       s_kv.key[0] = key.as_u64[0];
1330       s_kv.key[1] = key.as_u64[1];
1331       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1332         clib_warning ("out2in key add failed");
1333   }
1334
1335   /* Update IP checksum */
1336   sum = ip->checksum;
1337   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1338   ip->checksum = ip_csum_fold (sum);
1339
1340   /* Accounting */
1341   s->last_heard = now;
1342   s->total_pkts++;
1343   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1344   /* Per-user LRU list maintenance */
1345   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1346   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1347                       s->per_user_index);
1348
1349   /* Hairpinning */
1350   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1351     snat_hairpinning_unknown_proto(sm, b, ip);
1352
1353   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1354     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1355
1356   return s;
1357 }
1358
1359 static snat_session_t *
1360 snat_in2out_lb (snat_main_t *sm,
1361                 vlib_buffer_t * b,
1362                 ip4_header_t * ip,
1363                 u32 rx_fib_index,
1364                 u32 thread_index,
1365                 f64 now,
1366                 vlib_main_t * vm,
1367                 vlib_node_runtime_t * node)
1368 {
1369   nat_ed_ses_key_t key;
1370   clib_bihash_kv_16_8_t s_kv, s_value;
1371   udp_header_t *udp = ip4_next_header (ip);
1372   tcp_header_t *tcp = (tcp_header_t *) udp;
1373   snat_session_t *s = 0;
1374   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1375   u32 old_addr, new_addr;
1376   u16 new_port, old_port;
1377   ip_csum_t sum;
1378   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1379   snat_session_key_t e_key, l_key;
1380   clib_bihash_kv_8_8_t kv, value;
1381   snat_user_key_t u_key;
1382   snat_user_t *u;
1383   dlist_elt_t *head, *elt;
1384
1385   old_addr = ip->src_address.as_u32;
1386
1387   key.l_addr = ip->src_address;
1388   key.r_addr = ip->dst_address;
1389   key.fib_index = rx_fib_index;
1390   key.proto = ip->protocol;
1391   key.rsvd = 0;
1392   key.l_port = udp->src_port;
1393   s_kv.key[0] = key.as_u64[0];
1394   s_kv.key[1] = key.as_u64[1];
1395
1396   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1397     {
1398       s = pool_elt_at_index (tsm->sessions, s_value.value);
1399     }
1400   else
1401     {
1402       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1403         {
1404           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1405           return 0;
1406         }
1407
1408       l_key.addr = ip->src_address;
1409       l_key.port = udp->src_port;
1410       l_key.protocol = proto;
1411       l_key.fib_index = rx_fib_index;
1412       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0))
1413         return 0;
1414
1415       u_key.addr = ip->src_address;
1416       u_key.fib_index = rx_fib_index;
1417       kv.key = u_key.as_u64;
1418
1419       /* Ever heard of the "user" = src ip4 address before? */
1420       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1421         {
1422           /* no, make a new one */
1423           pool_get (tsm->users, u);
1424           memset (u, 0, sizeof (*u));
1425           u->addr = ip->src_address;
1426           u->fib_index = rx_fib_index;
1427
1428           pool_get (tsm->list_pool, head);
1429           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1430
1431           clib_dlist_init (tsm->list_pool,
1432                            u->sessions_per_user_list_head_index);
1433
1434           kv.value = u - tsm->users;
1435
1436           /* add user */
1437           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
1438             clib_warning ("user key add failed");
1439         }
1440       else
1441         {
1442           u = pool_elt_at_index (tsm->users, value.value);
1443         }
1444
1445       /* Create a new session */
1446       pool_get (tsm->sessions, s);
1447       memset (s, 0, sizeof (*s));
1448
1449       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1450       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1451       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1452       s->outside_address_index = ~0;
1453       s->in2out = l_key;
1454       s->out2in = e_key;
1455       u->nstaticsessions++;
1456
1457       /* Create list elts */
1458       pool_get (tsm->list_pool, elt);
1459       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1460       elt->value = s - tsm->sessions;
1461       s->per_user_index = elt - tsm->list_pool;
1462       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1463       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1464                           s->per_user_index);
1465
1466       /* Add to lookup tables */
1467       s_kv.value = s - tsm->sessions;
1468       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1469         clib_warning ("in2out-ed key add failed");
1470
1471       key.l_addr = e_key.addr;
1472       key.fib_index = e_key.fib_index;
1473       key.l_port = e_key.port;
1474       s_kv.key[0] = key.as_u64[0];
1475       s_kv.key[1] = key.as_u64[1];
1476       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1477         clib_warning ("out2in-ed key add failed");
1478     }
1479
1480   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1481
1482   /* Update IP checksum */
1483   sum = ip->checksum;
1484   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1485   ip->checksum = ip_csum_fold (sum);
1486
1487   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1488     {
1489       old_port = tcp->src_port;
1490       tcp->src_port = s->out2in.port;
1491       new_port = tcp->src_port;
1492
1493       sum = tcp->checksum;
1494       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1495       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1496       tcp->checksum = ip_csum_fold(sum);
1497     }
1498   else
1499     {
1500       udp->src_port = s->out2in.port;
1501       udp->checksum = 0;
1502     }
1503
1504   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1505     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1506
1507   /* Accounting */
1508   s->last_heard = now;
1509   s->total_pkts++;
1510   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1511   return s;
1512 }
1513
1514 static inline uword
1515 snat_in2out_node_fn_inline (vlib_main_t * vm,
1516                             vlib_node_runtime_t * node,
1517                             vlib_frame_t * frame, int is_slow_path,
1518                             int is_output_feature)
1519 {
1520   u32 n_left_from, * from, * to_next;
1521   snat_in2out_next_t next_index;
1522   u32 pkts_processed = 0;
1523   snat_main_t * sm = &snat_main;
1524   f64 now = vlib_time_now (vm);
1525   u32 stats_node_index;
1526   u32 thread_index = vlib_get_thread_index ();
1527
1528   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1529     snat_in2out_node.index;
1530
1531   from = vlib_frame_vector_args (frame);
1532   n_left_from = frame->n_vectors;
1533   next_index = node->cached_next_index;
1534
1535   while (n_left_from > 0)
1536     {
1537       u32 n_left_to_next;
1538
1539       vlib_get_next_frame (vm, node, next_index,
1540                            to_next, n_left_to_next);
1541
1542       while (n_left_from >= 4 && n_left_to_next >= 2)
1543         {
1544           u32 bi0, bi1;
1545           vlib_buffer_t * b0, * b1;
1546           u32 next0, next1;
1547           u32 sw_if_index0, sw_if_index1;
1548           ip4_header_t * ip0, * ip1;
1549           ip_csum_t sum0, sum1;
1550           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1551           u16 old_port0, new_port0, old_port1, new_port1;
1552           udp_header_t * udp0, * udp1;
1553           tcp_header_t * tcp0, * tcp1;
1554           icmp46_header_t * icmp0, * icmp1;
1555           snat_session_key_t key0, key1;
1556           u32 rx_fib_index0, rx_fib_index1;
1557           u32 proto0, proto1;
1558           snat_session_t * s0 = 0, * s1 = 0;
1559           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1560           u32 iph_offset0 = 0, iph_offset1 = 0;
1561
1562           /* Prefetch next iteration. */
1563           {
1564             vlib_buffer_t * p2, * p3;
1565
1566             p2 = vlib_get_buffer (vm, from[2]);
1567             p3 = vlib_get_buffer (vm, from[3]);
1568
1569             vlib_prefetch_buffer_header (p2, LOAD);
1570             vlib_prefetch_buffer_header (p3, LOAD);
1571
1572             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1573             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1574           }
1575
1576           /* speculatively enqueue b0 and b1 to the current next frame */
1577           to_next[0] = bi0 = from[0];
1578           to_next[1] = bi1 = from[1];
1579           from += 2;
1580           to_next += 2;
1581           n_left_from -= 2;
1582           n_left_to_next -= 2;
1583
1584           b0 = vlib_get_buffer (vm, bi0);
1585           b1 = vlib_get_buffer (vm, bi1);
1586
1587           if (is_output_feature)
1588             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1589
1590           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1591                  iph_offset0);
1592
1593           udp0 = ip4_next_header (ip0);
1594           tcp0 = (tcp_header_t *) udp0;
1595           icmp0 = (icmp46_header_t *) udp0;
1596
1597           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1598           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1599                                    sw_if_index0);
1600
1601           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1602
1603           if (PREDICT_FALSE(ip0->ttl == 1))
1604             {
1605               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1606               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1607                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1608                                            0);
1609               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1610               goto trace00;
1611             }
1612
1613           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1614
1615           /* Next configured feature, probably ip4-lookup */
1616           if (is_slow_path)
1617             {
1618               if (PREDICT_FALSE (proto0 == ~0))
1619                 {
1620                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1621                                                   thread_index, now, vm, node);
1622                   if (!s0)
1623                     next0 = SNAT_IN2OUT_NEXT_DROP;
1624                   goto trace00;
1625                 }
1626
1627               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1628                 {
1629                   next0 = icmp_in2out_slow_path
1630                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1631                      node, next0, now, thread_index, &s0);
1632                   goto trace00;
1633                 }
1634             }
1635           else
1636             {
1637               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1638                 {
1639                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1640                   goto trace00;
1641                 }
1642             }
1643
1644           key0.addr = ip0->src_address;
1645           key0.port = udp0->src_port;
1646           key0.protocol = proto0;
1647           key0.fib_index = rx_fib_index0;
1648
1649           kv0.key = key0.as_u64;
1650
1651           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1652               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1653             {
1654               if (is_slow_path)
1655                 {
1656                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1657                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1658                     goto trace00;
1659
1660                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1661                                      &s0, node, next0, thread_index);
1662                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1663                     goto trace00;
1664                 }
1665               else
1666                 {
1667                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1668                   goto trace00;
1669                 }
1670             }
1671           else
1672             {
1673               if (PREDICT_FALSE (value0.value == ~0ULL))
1674                 {
1675                   if (is_slow_path)
1676                     {
1677                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1678                                           thread_index, now, vm, node);
1679                       if (!s0)
1680                         next0 = SNAT_IN2OUT_NEXT_DROP;
1681                       goto trace00;
1682                     }
1683                   else
1684                     {
1685                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1686                       goto trace00;
1687                     }
1688                 }
1689               else
1690                 {
1691                   s0 = pool_elt_at_index (
1692                     sm->per_thread_data[thread_index].sessions,
1693                     value0.value);
1694                 }
1695             }
1696
1697           b0->flags |= VNET_BUFFER_F_IS_NATED;
1698
1699           old_addr0 = ip0->src_address.as_u32;
1700           ip0->src_address = s0->out2in.addr;
1701           new_addr0 = ip0->src_address.as_u32;
1702           if (!is_output_feature)
1703             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1704
1705           sum0 = ip0->checksum;
1706           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1707                                  ip4_header_t,
1708                                  src_address /* changed member */);
1709           ip0->checksum = ip_csum_fold (sum0);
1710
1711           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1712             {
1713               old_port0 = tcp0->src_port;
1714               tcp0->src_port = s0->out2in.port;
1715               new_port0 = tcp0->src_port;
1716
1717               sum0 = tcp0->checksum;
1718               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1719                                      ip4_header_t,
1720                                      dst_address /* changed member */);
1721               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1722                                      ip4_header_t /* cheat */,
1723                                      length /* changed member */);
1724               tcp0->checksum = ip_csum_fold(sum0);
1725             }
1726           else
1727             {
1728               old_port0 = udp0->src_port;
1729               udp0->src_port = s0->out2in.port;
1730               udp0->checksum = 0;
1731             }
1732
1733           /* Accounting */
1734           s0->last_heard = now;
1735           s0->total_pkts++;
1736           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1737           /* Per-user LRU list maintenance for dynamic translation */
1738           if (!snat_is_session_static (s0))
1739             {
1740               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1741                                  s0->per_user_index);
1742               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1743                                   s0->per_user_list_head_index,
1744                                   s0->per_user_index);
1745             }
1746         trace00:
1747
1748           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1749                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1750             {
1751               snat_in2out_trace_t *t =
1752                  vlib_add_trace (vm, node, b0, sizeof (*t));
1753               t->is_slow_path = is_slow_path;
1754               t->sw_if_index = sw_if_index0;
1755               t->next_index = next0;
1756                   t->session_index = ~0;
1757               if (s0)
1758                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1759             }
1760
1761           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1762
1763           if (is_output_feature)
1764             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1765
1766           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1767                  iph_offset1);
1768
1769           udp1 = ip4_next_header (ip1);
1770           tcp1 = (tcp_header_t *) udp1;
1771           icmp1 = (icmp46_header_t *) udp1;
1772
1773           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1774           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1775                                    sw_if_index1);
1776
1777           if (PREDICT_FALSE(ip1->ttl == 1))
1778             {
1779               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1780               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1781                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1782                                            0);
1783               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1784               goto trace01;
1785             }
1786
1787           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1788
1789           /* Next configured feature, probably ip4-lookup */
1790           if (is_slow_path)
1791             {
1792               if (PREDICT_FALSE (proto1 == ~0))
1793                 {
1794                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1795                                                   thread_index, now, vm, node);
1796                   if (!s1)
1797                     next1 = SNAT_IN2OUT_NEXT_DROP;
1798                   goto trace01;
1799                 }
1800
1801               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1802                 {
1803                   next1 = icmp_in2out_slow_path
1804                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1805                      next1, now, thread_index, &s1);
1806                   goto trace01;
1807                 }
1808             }
1809           else
1810             {
1811               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1812                 {
1813                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1814                   goto trace01;
1815                 }
1816             }
1817
1818           b1->flags |= VNET_BUFFER_F_IS_NATED;
1819
1820           key1.addr = ip1->src_address;
1821           key1.port = udp1->src_port;
1822           key1.protocol = proto1;
1823           key1.fib_index = rx_fib_index1;
1824
1825           kv1.key = key1.as_u64;
1826
1827             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1828                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1829             {
1830               if (is_slow_path)
1831                 {
1832                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1833                       ip1, proto1, rx_fib_index1, thread_index)) && !is_output_feature)
1834                     goto trace01;
1835
1836                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1837                                      &s1, node, next1, thread_index);
1838                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1839                     goto trace01;
1840                 }
1841               else
1842                 {
1843                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1844                   goto trace01;
1845                 }
1846             }
1847           else
1848             {
1849               if (PREDICT_FALSE (value1.value == ~0ULL))
1850                 {
1851                   if (is_slow_path)
1852                     {
1853                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1854                                           thread_index, now, vm, node);
1855                       if (!s1)
1856                         next1 = SNAT_IN2OUT_NEXT_DROP;
1857                       goto trace01;
1858                     }
1859                   else
1860                     {
1861                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1862                       goto trace01;
1863                     }
1864                 }
1865               else
1866                 {
1867                   s1 = pool_elt_at_index (
1868                     sm->per_thread_data[thread_index].sessions,
1869                     value1.value);
1870                 }
1871             }
1872
1873           old_addr1 = ip1->src_address.as_u32;
1874           ip1->src_address = s1->out2in.addr;
1875           new_addr1 = ip1->src_address.as_u32;
1876           if (!is_output_feature)
1877             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1878
1879           sum1 = ip1->checksum;
1880           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1881                                  ip4_header_t,
1882                                  src_address /* changed member */);
1883           ip1->checksum = ip_csum_fold (sum1);
1884
1885           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1886             {
1887               old_port1 = tcp1->src_port;
1888               tcp1->src_port = s1->out2in.port;
1889               new_port1 = tcp1->src_port;
1890
1891               sum1 = tcp1->checksum;
1892               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1893                                      ip4_header_t,
1894                                      dst_address /* changed member */);
1895               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1896                                      ip4_header_t /* cheat */,
1897                                      length /* changed member */);
1898               tcp1->checksum = ip_csum_fold(sum1);
1899             }
1900           else
1901             {
1902               old_port1 = udp1->src_port;
1903               udp1->src_port = s1->out2in.port;
1904               udp1->checksum = 0;
1905             }
1906
1907           /* Accounting */
1908           s1->last_heard = now;
1909           s1->total_pkts++;
1910           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1911           /* Per-user LRU list maintenance for dynamic translation */
1912           if (!snat_is_session_static (s1))
1913             {
1914               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1915                                  s1->per_user_index);
1916               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1917                                   s1->per_user_list_head_index,
1918                                   s1->per_user_index);
1919             }
1920         trace01:
1921
1922           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1923                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1924             {
1925               snat_in2out_trace_t *t =
1926                  vlib_add_trace (vm, node, b1, sizeof (*t));
1927               t->sw_if_index = sw_if_index1;
1928               t->next_index = next1;
1929               t->session_index = ~0;
1930               if (s1)
1931                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1932             }
1933
1934           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1935
1936           /* verify speculative enqueues, maybe switch current next frame */
1937           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1938                                            to_next, n_left_to_next,
1939                                            bi0, bi1, next0, next1);
1940         }
1941
1942       while (n_left_from > 0 && n_left_to_next > 0)
1943         {
1944           u32 bi0;
1945           vlib_buffer_t * b0;
1946           u32 next0;
1947           u32 sw_if_index0;
1948           ip4_header_t * ip0;
1949           ip_csum_t sum0;
1950           u32 new_addr0, old_addr0;
1951           u16 old_port0, new_port0;
1952           udp_header_t * udp0;
1953           tcp_header_t * tcp0;
1954           icmp46_header_t * icmp0;
1955           snat_session_key_t key0;
1956           u32 rx_fib_index0;
1957           u32 proto0;
1958           snat_session_t * s0 = 0;
1959           clib_bihash_kv_8_8_t kv0, value0;
1960           u32 iph_offset0 = 0;
1961
1962           /* speculatively enqueue b0 to the current next frame */
1963           bi0 = from[0];
1964           to_next[0] = bi0;
1965           from += 1;
1966           to_next += 1;
1967           n_left_from -= 1;
1968           n_left_to_next -= 1;
1969
1970           b0 = vlib_get_buffer (vm, bi0);
1971           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1972
1973           if (is_output_feature)
1974             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1975
1976           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1977                  iph_offset0);
1978
1979           udp0 = ip4_next_header (ip0);
1980           tcp0 = (tcp_header_t *) udp0;
1981           icmp0 = (icmp46_header_t *) udp0;
1982
1983           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1984           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1985                                    sw_if_index0);
1986
1987           if (PREDICT_FALSE(ip0->ttl == 1))
1988             {
1989               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1990               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1991                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1992                                            0);
1993               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1994               goto trace0;
1995             }
1996
1997           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1998
1999           /* Next configured feature, probably ip4-lookup */
2000           if (is_slow_path)
2001             {
2002               if (PREDICT_FALSE (proto0 == ~0))
2003                 {
2004                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2005                                                   thread_index, now, vm, node);
2006                   if (!s0)
2007                     next0 = SNAT_IN2OUT_NEXT_DROP;
2008                   goto trace0;
2009                 }
2010
2011               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2012                 {
2013                   next0 = icmp_in2out_slow_path
2014                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2015                      next0, now, thread_index, &s0);
2016                   goto trace0;
2017                 }
2018             }
2019           else
2020             {
2021               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2022                 {
2023                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2024                   goto trace0;
2025                 }
2026             }
2027
2028           key0.addr = ip0->src_address;
2029           key0.port = udp0->src_port;
2030           key0.protocol = proto0;
2031           key0.fib_index = rx_fib_index0;
2032
2033           kv0.key = key0.as_u64;
2034
2035           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2036                                       &kv0, &value0))
2037             {
2038               if (is_slow_path)
2039                 {
2040                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2041                       ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
2042                     goto trace0;
2043
2044                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2045                                      &s0, node, next0, thread_index);
2046
2047                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2048                     goto trace0;
2049                 }
2050               else
2051                 {
2052                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2053                   goto trace0;
2054                 }
2055             }
2056           else
2057             {
2058               if (PREDICT_FALSE (value0.value == ~0ULL))
2059                 {
2060                   if (is_slow_path)
2061                     {
2062                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2063                                           thread_index, now, vm, node);
2064                       if (!s0)
2065                         next0 = SNAT_IN2OUT_NEXT_DROP;
2066                       goto trace0;
2067                     }
2068                   else
2069                     {
2070                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2071                       goto trace0;
2072                     }
2073                 }
2074               else
2075                 {
2076                   s0 = pool_elt_at_index (
2077                     sm->per_thread_data[thread_index].sessions,
2078                     value0.value);
2079                 }
2080             }
2081
2082           b0->flags |= VNET_BUFFER_F_IS_NATED;
2083
2084           old_addr0 = ip0->src_address.as_u32;
2085           ip0->src_address = s0->out2in.addr;
2086           new_addr0 = ip0->src_address.as_u32;
2087           if (!is_output_feature)
2088             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2089
2090           sum0 = ip0->checksum;
2091           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2092                                  ip4_header_t,
2093                                  src_address /* changed member */);
2094           ip0->checksum = ip_csum_fold (sum0);
2095
2096           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2097             {
2098               old_port0 = tcp0->src_port;
2099               tcp0->src_port = s0->out2in.port;
2100               new_port0 = tcp0->src_port;
2101
2102               sum0 = tcp0->checksum;
2103               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2104                                      ip4_header_t,
2105                                      dst_address /* changed member */);
2106               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2107                                      ip4_header_t /* cheat */,
2108                                      length /* changed member */);
2109               tcp0->checksum = ip_csum_fold(sum0);
2110             }
2111           else
2112             {
2113               old_port0 = udp0->src_port;
2114               udp0->src_port = s0->out2in.port;
2115               udp0->checksum = 0;
2116             }
2117
2118           /* Accounting */
2119           s0->last_heard = now;
2120           s0->total_pkts++;
2121           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2122           /* Per-user LRU list maintenance for dynamic translation */
2123           if (!snat_is_session_static (s0))
2124             {
2125               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2126                                  s0->per_user_index);
2127               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2128                                   s0->per_user_list_head_index,
2129                                   s0->per_user_index);
2130             }
2131
2132         trace0:
2133           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2134                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2135             {
2136               snat_in2out_trace_t *t =
2137                  vlib_add_trace (vm, node, b0, sizeof (*t));
2138               t->is_slow_path = is_slow_path;
2139               t->sw_if_index = sw_if_index0;
2140               t->next_index = next0;
2141                   t->session_index = ~0;
2142               if (s0)
2143                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2144             }
2145
2146           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2147
2148           /* verify speculative enqueue, maybe switch current next frame */
2149           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2150                                            to_next, n_left_to_next,
2151                                            bi0, next0);
2152         }
2153
2154       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2155     }
2156
2157   vlib_node_increment_counter (vm, stats_node_index,
2158                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2159                                pkts_processed);
2160   return frame->n_vectors;
2161 }
2162
2163 static uword
2164 snat_in2out_fast_path_fn (vlib_main_t * vm,
2165                           vlib_node_runtime_t * node,
2166                           vlib_frame_t * frame)
2167 {
2168   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2169 }
2170
2171 VLIB_REGISTER_NODE (snat_in2out_node) = {
2172   .function = snat_in2out_fast_path_fn,
2173   .name = "nat44-in2out",
2174   .vector_size = sizeof (u32),
2175   .format_trace = format_snat_in2out_trace,
2176   .type = VLIB_NODE_TYPE_INTERNAL,
2177
2178   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2179   .error_strings = snat_in2out_error_strings,
2180
2181   .runtime_data_bytes = sizeof (snat_runtime_t),
2182
2183   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2184
2185   /* edit / add dispositions here */
2186   .next_nodes = {
2187     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2188     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2189     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2190     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2191   },
2192 };
2193
2194 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2195
2196 static uword
2197 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2198                                  vlib_node_runtime_t * node,
2199                                  vlib_frame_t * frame)
2200 {
2201   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2202 }
2203
2204 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2205   .function = snat_in2out_output_fast_path_fn,
2206   .name = "nat44-in2out-output",
2207   .vector_size = sizeof (u32),
2208   .format_trace = format_snat_in2out_trace,
2209   .type = VLIB_NODE_TYPE_INTERNAL,
2210
2211   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2212   .error_strings = snat_in2out_error_strings,
2213
2214   .runtime_data_bytes = sizeof (snat_runtime_t),
2215
2216   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2217
2218   /* edit / add dispositions here */
2219   .next_nodes = {
2220     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2221     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2222     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2223     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2224   },
2225 };
2226
2227 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2228                               snat_in2out_output_fast_path_fn);
2229
2230 static uword
2231 snat_in2out_slow_path_fn (vlib_main_t * vm,
2232                           vlib_node_runtime_t * node,
2233                           vlib_frame_t * frame)
2234 {
2235   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2236 }
2237
2238 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2239   .function = snat_in2out_slow_path_fn,
2240   .name = "nat44-in2out-slowpath",
2241   .vector_size = sizeof (u32),
2242   .format_trace = format_snat_in2out_trace,
2243   .type = VLIB_NODE_TYPE_INTERNAL,
2244
2245   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2246   .error_strings = snat_in2out_error_strings,
2247
2248   .runtime_data_bytes = sizeof (snat_runtime_t),
2249
2250   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2251
2252   /* edit / add dispositions here */
2253   .next_nodes = {
2254     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2255     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2256     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2257     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2258   },
2259 };
2260
2261 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2262                               snat_in2out_slow_path_fn);
2263
2264 static uword
2265 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2266                                  vlib_node_runtime_t * node,
2267                                  vlib_frame_t * frame)
2268 {
2269   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2270 }
2271
2272 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2273   .function = snat_in2out_output_slow_path_fn,
2274   .name = "nat44-in2out-output-slowpath",
2275   .vector_size = sizeof (u32),
2276   .format_trace = format_snat_in2out_trace,
2277   .type = VLIB_NODE_TYPE_INTERNAL,
2278
2279   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2280   .error_strings = snat_in2out_error_strings,
2281
2282   .runtime_data_bytes = sizeof (snat_runtime_t),
2283
2284   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2285
2286   /* edit / add dispositions here */
2287   .next_nodes = {
2288     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2289     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2290     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2291     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2292   },
2293 };
2294
2295 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2296                               snat_in2out_output_slow_path_fn);
2297
2298 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2299
2300 static uword
2301 nat44_hairpinning_fn (vlib_main_t * vm,
2302                       vlib_node_runtime_t * node,
2303                       vlib_frame_t * frame)
2304 {
2305   u32 n_left_from, * from, * to_next;
2306   snat_in2out_next_t next_index;
2307   u32 pkts_processed = 0;
2308   snat_main_t * sm = &snat_main;
2309   vnet_feature_main_t *fm = &feature_main;
2310   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2311   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2312
2313   from = vlib_frame_vector_args (frame);
2314   n_left_from = frame->n_vectors;
2315   next_index = node->cached_next_index;
2316
2317   while (n_left_from > 0)
2318     {
2319       u32 n_left_to_next;
2320
2321       vlib_get_next_frame (vm, node, next_index,
2322                            to_next, n_left_to_next);
2323
2324       while (n_left_from > 0 && n_left_to_next > 0)
2325         {
2326           u32 bi0;
2327           vlib_buffer_t * b0;
2328           u32 next0;
2329           ip4_header_t * ip0;
2330           u32 proto0;
2331           udp_header_t * udp0;
2332           tcp_header_t * tcp0;
2333
2334           /* speculatively enqueue b0 to the current next frame */
2335           bi0 = from[0];
2336           to_next[0] = bi0;
2337           from += 1;
2338           to_next += 1;
2339           n_left_from -= 1;
2340           n_left_to_next -= 1;
2341
2342           b0 = vlib_get_buffer (vm, bi0);
2343           ip0 = vlib_buffer_get_current (b0);
2344           udp0 = ip4_next_header (ip0);
2345           tcp0 = (tcp_header_t *) udp0;
2346
2347           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2348
2349           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2350                                 &next0, 0);
2351
2352           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2353             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2354
2355           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2356
2357           /* verify speculative enqueue, maybe switch current next frame */
2358           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2359                                            to_next, n_left_to_next,
2360                                            bi0, next0);
2361          }
2362
2363       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2364     }
2365
2366   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2367                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2368                                pkts_processed);
2369   return frame->n_vectors;
2370 }
2371
2372 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2373   .function = nat44_hairpinning_fn,
2374   .name = "nat44-hairpinning",
2375   .vector_size = sizeof (u32),
2376   .type = VLIB_NODE_TYPE_INTERNAL,
2377   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2378   .error_strings = snat_in2out_error_strings,
2379   .n_next_nodes = 2,
2380   .next_nodes = {
2381     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2382     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2383   },
2384 };
2385
2386 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2387                               nat44_hairpinning_fn);
2388
2389 /**************************/
2390 /*** deterministic mode ***/
2391 /**************************/
2392 static uword
2393 snat_det_in2out_node_fn (vlib_main_t * vm,
2394                          vlib_node_runtime_t * node,
2395                          vlib_frame_t * frame)
2396 {
2397   u32 n_left_from, * from, * to_next;
2398   snat_in2out_next_t next_index;
2399   u32 pkts_processed = 0;
2400   snat_main_t * sm = &snat_main;
2401   u32 now = (u32) vlib_time_now (vm);
2402   u32 thread_index = vlib_get_thread_index ();
2403
2404   from = vlib_frame_vector_args (frame);
2405   n_left_from = frame->n_vectors;
2406   next_index = node->cached_next_index;
2407
2408   while (n_left_from > 0)
2409     {
2410       u32 n_left_to_next;
2411
2412       vlib_get_next_frame (vm, node, next_index,
2413                            to_next, n_left_to_next);
2414
2415       while (n_left_from >= 4 && n_left_to_next >= 2)
2416         {
2417           u32 bi0, bi1;
2418           vlib_buffer_t * b0, * b1;
2419           u32 next0, next1;
2420           u32 sw_if_index0, sw_if_index1;
2421           ip4_header_t * ip0, * ip1;
2422           ip_csum_t sum0, sum1;
2423           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2424           u16 old_port0, new_port0, lo_port0, i0;
2425           u16 old_port1, new_port1, lo_port1, i1;
2426           udp_header_t * udp0, * udp1;
2427           tcp_header_t * tcp0, * tcp1;
2428           u32 proto0, proto1;
2429           snat_det_out_key_t key0, key1;
2430           snat_det_map_t * dm0, * dm1;
2431           snat_det_session_t * ses0 = 0, * ses1 = 0;
2432           u32 rx_fib_index0, rx_fib_index1;
2433           icmp46_header_t * icmp0, * icmp1;
2434
2435           /* Prefetch next iteration. */
2436           {
2437             vlib_buffer_t * p2, * p3;
2438
2439             p2 = vlib_get_buffer (vm, from[2]);
2440             p3 = vlib_get_buffer (vm, from[3]);
2441
2442             vlib_prefetch_buffer_header (p2, LOAD);
2443             vlib_prefetch_buffer_header (p3, LOAD);
2444
2445             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2446             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2447           }
2448
2449           /* speculatively enqueue b0 and b1 to the current next frame */
2450           to_next[0] = bi0 = from[0];
2451           to_next[1] = bi1 = from[1];
2452           from += 2;
2453           to_next += 2;
2454           n_left_from -= 2;
2455           n_left_to_next -= 2;
2456
2457           b0 = vlib_get_buffer (vm, bi0);
2458           b1 = vlib_get_buffer (vm, bi1);
2459
2460           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2461           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2462
2463           ip0 = vlib_buffer_get_current (b0);
2464           udp0 = ip4_next_header (ip0);
2465           tcp0 = (tcp_header_t *) udp0;
2466
2467           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2468
2469           if (PREDICT_FALSE(ip0->ttl == 1))
2470             {
2471               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2472               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2473                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2474                                            0);
2475               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2476               goto trace0;
2477             }
2478
2479           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2480
2481           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2482             {
2483               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2484               icmp0 = (icmp46_header_t *) udp0;
2485
2486               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2487                                   rx_fib_index0, node, next0, thread_index,
2488                                   &ses0, &dm0);
2489               goto trace0;
2490             }
2491
2492           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2493           if (PREDICT_FALSE(!dm0))
2494             {
2495               clib_warning("no match for internal host %U",
2496                            format_ip4_address, &ip0->src_address);
2497               next0 = SNAT_IN2OUT_NEXT_DROP;
2498               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2499               goto trace0;
2500             }
2501
2502           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2503
2504           key0.ext_host_addr = ip0->dst_address;
2505           key0.ext_host_port = tcp0->dst;
2506
2507           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2508           if (PREDICT_FALSE(!ses0))
2509             {
2510               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2511                 {
2512                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2513                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2514
2515                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2516                     continue;
2517
2518                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2519                   break;
2520                 }
2521               if (PREDICT_FALSE(!ses0))
2522                 {
2523                   /* too many sessions for user, send ICMP error packet */
2524
2525                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2526                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2527                                                ICMP4_destination_unreachable_destination_unreachable_host,
2528                                                0);
2529                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2530                   goto trace0;
2531                 }
2532             }
2533
2534           new_port0 = ses0->out.out_port;
2535
2536           old_addr0.as_u32 = ip0->src_address.as_u32;
2537           ip0->src_address.as_u32 = new_addr0.as_u32;
2538           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2539
2540           sum0 = ip0->checksum;
2541           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2542                                  ip4_header_t,
2543                                  src_address /* changed member */);
2544           ip0->checksum = ip_csum_fold (sum0);
2545
2546           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2547             {
2548               if (tcp0->flags & TCP_FLAG_SYN)
2549                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2550               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2551                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2552               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2553                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2554               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2555                 snat_det_ses_close(dm0, ses0);
2556               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2557                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2558               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2559                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2560
2561               old_port0 = tcp0->src;
2562               tcp0->src = new_port0;
2563
2564               sum0 = tcp0->checksum;
2565               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2566                                      ip4_header_t,
2567                                      dst_address /* changed member */);
2568               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2569                                      ip4_header_t /* cheat */,
2570                                      length /* changed member */);
2571               tcp0->checksum = ip_csum_fold(sum0);
2572             }
2573           else
2574             {
2575               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2576               old_port0 = udp0->src_port;
2577               udp0->src_port = new_port0;
2578               udp0->checksum = 0;
2579             }
2580
2581           switch(ses0->state)
2582             {
2583             case SNAT_SESSION_UDP_ACTIVE:
2584                 ses0->expire = now + sm->udp_timeout;
2585                 break;
2586             case SNAT_SESSION_TCP_SYN_SENT:
2587             case SNAT_SESSION_TCP_FIN_WAIT:
2588             case SNAT_SESSION_TCP_CLOSE_WAIT:
2589             case SNAT_SESSION_TCP_LAST_ACK:
2590                 ses0->expire = now + sm->tcp_transitory_timeout;
2591                 break;
2592             case SNAT_SESSION_TCP_ESTABLISHED:
2593                 ses0->expire = now + sm->tcp_established_timeout;
2594                 break;
2595             }
2596
2597         trace0:
2598           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2599                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2600             {
2601               snat_in2out_trace_t *t =
2602                  vlib_add_trace (vm, node, b0, sizeof (*t));
2603               t->is_slow_path = 0;
2604               t->sw_if_index = sw_if_index0;
2605               t->next_index = next0;
2606               t->session_index = ~0;
2607               if (ses0)
2608                 t->session_index = ses0 - dm0->sessions;
2609             }
2610
2611           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2612
2613           ip1 = vlib_buffer_get_current (b1);
2614           udp1 = ip4_next_header (ip1);
2615           tcp1 = (tcp_header_t *) udp1;
2616
2617           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2618
2619           if (PREDICT_FALSE(ip1->ttl == 1))
2620             {
2621               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2622               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2623                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2624                                            0);
2625               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2626               goto trace1;
2627             }
2628
2629           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2630
2631           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2632             {
2633               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2634               icmp1 = (icmp46_header_t *) udp1;
2635
2636               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2637                                   rx_fib_index1, node, next1, thread_index,
2638                                   &ses1, &dm1);
2639               goto trace1;
2640             }
2641
2642           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2643           if (PREDICT_FALSE(!dm1))
2644             {
2645               clib_warning("no match for internal host %U",
2646                            format_ip4_address, &ip0->src_address);
2647               next1 = SNAT_IN2OUT_NEXT_DROP;
2648               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2649               goto trace1;
2650             }
2651
2652           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2653
2654           key1.ext_host_addr = ip1->dst_address;
2655           key1.ext_host_port = tcp1->dst;
2656
2657           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2658           if (PREDICT_FALSE(!ses1))
2659             {
2660               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2661                 {
2662                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2663                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2664
2665                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2666                     continue;
2667
2668                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2669                   break;
2670                 }
2671               if (PREDICT_FALSE(!ses1))
2672                 {
2673                   /* too many sessions for user, send ICMP error packet */
2674
2675                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2676                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2677                                                ICMP4_destination_unreachable_destination_unreachable_host,
2678                                                0);
2679                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2680                   goto trace1;
2681                 }
2682             }
2683
2684           new_port1 = ses1->out.out_port;
2685
2686           old_addr1.as_u32 = ip1->src_address.as_u32;
2687           ip1->src_address.as_u32 = new_addr1.as_u32;
2688           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2689
2690           sum1 = ip1->checksum;
2691           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2692                                  ip4_header_t,
2693                                  src_address /* changed member */);
2694           ip1->checksum = ip_csum_fold (sum1);
2695
2696           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2697             {
2698               if (tcp1->flags & TCP_FLAG_SYN)
2699                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2700               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2701                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2702               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2703                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2704               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2705                 snat_det_ses_close(dm1, ses1);
2706               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2707                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2708               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2709                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2710
2711               old_port1 = tcp1->src;
2712               tcp1->src = new_port1;
2713
2714               sum1 = tcp1->checksum;
2715               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2716                                      ip4_header_t,
2717                                      dst_address /* changed member */);
2718               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2719                                      ip4_header_t /* cheat */,
2720                                      length /* changed member */);
2721               tcp1->checksum = ip_csum_fold(sum1);
2722             }
2723           else
2724             {
2725               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2726               old_port1 = udp1->src_port;
2727               udp1->src_port = new_port1;
2728               udp1->checksum = 0;
2729             }
2730
2731           switch(ses1->state)
2732             {
2733             case SNAT_SESSION_UDP_ACTIVE:
2734                 ses1->expire = now + sm->udp_timeout;
2735                 break;
2736             case SNAT_SESSION_TCP_SYN_SENT:
2737             case SNAT_SESSION_TCP_FIN_WAIT:
2738             case SNAT_SESSION_TCP_CLOSE_WAIT:
2739             case SNAT_SESSION_TCP_LAST_ACK:
2740                 ses1->expire = now + sm->tcp_transitory_timeout;
2741                 break;
2742             case SNAT_SESSION_TCP_ESTABLISHED:
2743                 ses1->expire = now + sm->tcp_established_timeout;
2744                 break;
2745             }
2746
2747         trace1:
2748           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2749                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2750             {
2751               snat_in2out_trace_t *t =
2752                  vlib_add_trace (vm, node, b1, sizeof (*t));
2753               t->is_slow_path = 0;
2754               t->sw_if_index = sw_if_index1;
2755               t->next_index = next1;
2756               t->session_index = ~0;
2757               if (ses1)
2758                 t->session_index = ses1 - dm1->sessions;
2759             }
2760
2761           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2762
2763           /* verify speculative enqueues, maybe switch current next frame */
2764           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2765                                            to_next, n_left_to_next,
2766                                            bi0, bi1, next0, next1);
2767          }
2768
2769       while (n_left_from > 0 && n_left_to_next > 0)
2770         {
2771           u32 bi0;
2772           vlib_buffer_t * b0;
2773           u32 next0;
2774           u32 sw_if_index0;
2775           ip4_header_t * ip0;
2776           ip_csum_t sum0;
2777           ip4_address_t new_addr0, old_addr0;
2778           u16 old_port0, new_port0, lo_port0, i0;
2779           udp_header_t * udp0;
2780           tcp_header_t * tcp0;
2781           u32 proto0;
2782           snat_det_out_key_t key0;
2783           snat_det_map_t * dm0;
2784           snat_det_session_t * ses0 = 0;
2785           u32 rx_fib_index0;
2786           icmp46_header_t * icmp0;
2787
2788           /* speculatively enqueue b0 to the current next frame */
2789           bi0 = from[0];
2790           to_next[0] = bi0;
2791           from += 1;
2792           to_next += 1;
2793           n_left_from -= 1;
2794           n_left_to_next -= 1;
2795
2796           b0 = vlib_get_buffer (vm, bi0);
2797           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2798
2799           ip0 = vlib_buffer_get_current (b0);
2800           udp0 = ip4_next_header (ip0);
2801           tcp0 = (tcp_header_t *) udp0;
2802
2803           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2804
2805           if (PREDICT_FALSE(ip0->ttl == 1))
2806             {
2807               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2808               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2809                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2810                                            0);
2811               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2812               goto trace00;
2813             }
2814
2815           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2816
2817           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2818             {
2819               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2820               icmp0 = (icmp46_header_t *) udp0;
2821
2822               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2823                                   rx_fib_index0, node, next0, thread_index,
2824                                   &ses0, &dm0);
2825               goto trace00;
2826             }
2827
2828           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2829           if (PREDICT_FALSE(!dm0))
2830             {
2831               clib_warning("no match for internal host %U",
2832                            format_ip4_address, &ip0->src_address);
2833               next0 = SNAT_IN2OUT_NEXT_DROP;
2834               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2835               goto trace00;
2836             }
2837
2838           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2839
2840           key0.ext_host_addr = ip0->dst_address;
2841           key0.ext_host_port = tcp0->dst;
2842
2843           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2844           if (PREDICT_FALSE(!ses0))
2845             {
2846               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2847                 {
2848                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2849                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2850
2851                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2852                     continue;
2853
2854                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2855                   break;
2856                 }
2857               if (PREDICT_FALSE(!ses0))
2858                 {
2859                   /* too many sessions for user, send ICMP error packet */
2860
2861                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2862                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2863                                                ICMP4_destination_unreachable_destination_unreachable_host,
2864                                                0);
2865                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2866                   goto trace00;
2867                 }
2868             }
2869
2870           new_port0 = ses0->out.out_port;
2871
2872           old_addr0.as_u32 = ip0->src_address.as_u32;
2873           ip0->src_address.as_u32 = new_addr0.as_u32;
2874           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2875
2876           sum0 = ip0->checksum;
2877           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2878                                  ip4_header_t,
2879                                  src_address /* changed member */);
2880           ip0->checksum = ip_csum_fold (sum0);
2881
2882           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2883             {
2884               if (tcp0->flags & TCP_FLAG_SYN)
2885                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2886               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2887                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2888               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2889                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2890               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2891                 snat_det_ses_close(dm0, ses0);
2892               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2893                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2894               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2895                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2896
2897               old_port0 = tcp0->src;
2898               tcp0->src = new_port0;
2899
2900               sum0 = tcp0->checksum;
2901               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2902                                      ip4_header_t,
2903                                      dst_address /* changed member */);
2904               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2905                                      ip4_header_t /* cheat */,
2906                                      length /* changed member */);
2907               tcp0->checksum = ip_csum_fold(sum0);
2908             }
2909           else
2910             {
2911               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2912               old_port0 = udp0->src_port;
2913               udp0->src_port = new_port0;
2914               udp0->checksum = 0;
2915             }
2916
2917           switch(ses0->state)
2918             {
2919             case SNAT_SESSION_UDP_ACTIVE:
2920                 ses0->expire = now + sm->udp_timeout;
2921                 break;
2922             case SNAT_SESSION_TCP_SYN_SENT:
2923             case SNAT_SESSION_TCP_FIN_WAIT:
2924             case SNAT_SESSION_TCP_CLOSE_WAIT:
2925             case SNAT_SESSION_TCP_LAST_ACK:
2926                 ses0->expire = now + sm->tcp_transitory_timeout;
2927                 break;
2928             case SNAT_SESSION_TCP_ESTABLISHED:
2929                 ses0->expire = now + sm->tcp_established_timeout;
2930                 break;
2931             }
2932
2933         trace00:
2934           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2935                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2936             {
2937               snat_in2out_trace_t *t =
2938                  vlib_add_trace (vm, node, b0, sizeof (*t));
2939               t->is_slow_path = 0;
2940               t->sw_if_index = sw_if_index0;
2941               t->next_index = next0;
2942               t->session_index = ~0;
2943               if (ses0)
2944                 t->session_index = ses0 - dm0->sessions;
2945             }
2946
2947           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2948
2949           /* verify speculative enqueue, maybe switch current next frame */
2950           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2951                                            to_next, n_left_to_next,
2952                                            bi0, next0);
2953         }
2954
2955       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2956     }
2957
2958   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2959                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2960                                pkts_processed);
2961   return frame->n_vectors;
2962 }
2963
2964 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2965   .function = snat_det_in2out_node_fn,
2966   .name = "nat44-det-in2out",
2967   .vector_size = sizeof (u32),
2968   .format_trace = format_snat_in2out_trace,
2969   .type = VLIB_NODE_TYPE_INTERNAL,
2970
2971   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2972   .error_strings = snat_in2out_error_strings,
2973
2974   .runtime_data_bytes = sizeof (snat_runtime_t),
2975
2976   .n_next_nodes = 3,
2977
2978   /* edit / add dispositions here */
2979   .next_nodes = {
2980     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2981     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2982     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2983   },
2984 };
2985
2986 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2987
2988 /**
2989  * Get address and port values to be used for ICMP packet translation
2990  * and create session if needed
2991  *
2992  * @param[in,out] sm             NAT main
2993  * @param[in,out] node           NAT node runtime
2994  * @param[in] thread_index       thread index
2995  * @param[in,out] b0             buffer containing packet to be translated
2996  * @param[out] p_proto           protocol used for matching
2997  * @param[out] p_value           address and port after NAT translation
2998  * @param[out] p_dont_translate  if packet should not be translated
2999  * @param d                      optional parameter
3000  * @param e                      optional parameter
3001  */
3002 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3003                           u32 thread_index, vlib_buffer_t *b0,
3004                           ip4_header_t *ip0, u8 *p_proto,
3005                           snat_session_key_t *p_value,
3006                           u8 *p_dont_translate, void *d, void *e)
3007 {
3008   icmp46_header_t *icmp0;
3009   u32 sw_if_index0;
3010   u32 rx_fib_index0;
3011   u8 protocol;
3012   snat_det_out_key_t key0;
3013   u8 dont_translate = 0;
3014   u32 next0 = ~0;
3015   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3016   ip4_header_t *inner_ip0;
3017   void *l4_header = 0;
3018   icmp46_header_t *inner_icmp0;
3019   snat_det_map_t * dm0 = 0;
3020   ip4_address_t new_addr0;
3021   u16 lo_port0, i0;
3022   snat_det_session_t * ses0 = 0;
3023   ip4_address_t in_addr;
3024   u16 in_port;
3025
3026   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3027   echo0 = (icmp_echo_header_t *)(icmp0+1);
3028   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3029   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3030
3031   if (!icmp_is_error_message (icmp0))
3032     {
3033       protocol = SNAT_PROTOCOL_ICMP;
3034       in_addr = ip0->src_address;
3035       in_port = echo0->identifier;
3036     }
3037   else
3038     {
3039       inner_ip0 = (ip4_header_t *)(echo0+1);
3040       l4_header = ip4_next_header (inner_ip0);
3041       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3042       in_addr = inner_ip0->dst_address;
3043       switch (protocol)
3044         {
3045         case SNAT_PROTOCOL_ICMP:
3046           inner_icmp0 = (icmp46_header_t*)l4_header;
3047           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3048           in_port = inner_echo0->identifier;
3049           break;
3050         case SNAT_PROTOCOL_UDP:
3051         case SNAT_PROTOCOL_TCP:
3052           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3053           break;
3054         default:
3055           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3056           next0 = SNAT_IN2OUT_NEXT_DROP;
3057           goto out;
3058         }
3059     }
3060
3061   dm0 = snat_det_map_by_user(sm, &in_addr);
3062   if (PREDICT_FALSE(!dm0))
3063     {
3064       clib_warning("no match for internal host %U",
3065                    format_ip4_address, &in_addr);
3066       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3067           IP_PROTOCOL_ICMP, rx_fib_index0)))
3068         {
3069           dont_translate = 1;
3070           goto out;
3071         }
3072       next0 = SNAT_IN2OUT_NEXT_DROP;
3073       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3074       goto out;
3075     }
3076
3077   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3078
3079   key0.ext_host_addr = ip0->dst_address;
3080   key0.ext_host_port = 0;
3081
3082   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3083   if (PREDICT_FALSE(!ses0))
3084     {
3085       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3086           IP_PROTOCOL_ICMP, rx_fib_index0)))
3087         {
3088           dont_translate = 1;
3089           goto out;
3090         }
3091       if (icmp0->type != ICMP4_echo_request)
3092         {
3093           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3094           next0 = SNAT_IN2OUT_NEXT_DROP;
3095           goto out;
3096         }
3097       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3098         {
3099           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3100             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3101
3102           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3103             continue;
3104
3105           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3106           break;
3107         }
3108       if (PREDICT_FALSE(!ses0))
3109         {
3110           next0 = SNAT_IN2OUT_NEXT_DROP;
3111           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3112           goto out;
3113         }
3114     }
3115
3116   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3117                     !icmp_is_error_message (icmp0)))
3118     {
3119       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3120       next0 = SNAT_IN2OUT_NEXT_DROP;
3121       goto out;
3122     }
3123
3124   u32 now = (u32) vlib_time_now (sm->vlib_main);
3125
3126   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3127   ses0->expire = now + sm->icmp_timeout;
3128
3129 out:
3130   *p_proto = protocol;
3131   if (ses0)
3132     {
3133       p_value->addr = new_addr0;
3134       p_value->fib_index = sm->outside_fib_index;
3135       p_value->port = ses0->out.out_port;
3136     }
3137   *p_dont_translate = dont_translate;
3138   if (d)
3139     *(snat_det_session_t**)d = ses0;
3140   if (e)
3141     *(snat_det_map_t**)e = dm0;
3142   return next0;
3143 }
3144
3145 /**********************/
3146 /*** worker handoff ***/
3147 /**********************/
3148 static inline uword
3149 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3150                                       vlib_node_runtime_t * node,
3151                                       vlib_frame_t * frame,
3152                                       u8 is_output)
3153 {
3154   snat_main_t *sm = &snat_main;
3155   vlib_thread_main_t *tm = vlib_get_thread_main ();
3156   u32 n_left_from, *from, *to_next = 0;
3157   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3158   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3159     = 0;
3160   vlib_frame_queue_elt_t *hf = 0;
3161   vlib_frame_t *f = 0;
3162   int i;
3163   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3164   u32 next_worker_index = 0;
3165   u32 current_worker_index = ~0;
3166   u32 thread_index = vlib_get_thread_index ();
3167   u32 fq_index;
3168   u32 to_node_index;
3169
3170   ASSERT (vec_len (sm->workers));
3171
3172   if (is_output)
3173     {
3174       fq_index = sm->fq_in2out_output_index;
3175       to_node_index = sm->in2out_output_node_index;
3176     }
3177   else
3178     {
3179       fq_index = sm->fq_in2out_index;
3180       to_node_index = sm->in2out_node_index;
3181     }
3182
3183   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3184     {
3185       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3186
3187       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3188                                sm->first_worker_index + sm->num_workers - 1,
3189                                (vlib_frame_queue_t *) (~0));
3190     }
3191
3192   from = vlib_frame_vector_args (frame);
3193   n_left_from = frame->n_vectors;
3194
3195   while (n_left_from > 0)
3196     {
3197       u32 bi0;
3198       vlib_buffer_t *b0;
3199       u32 sw_if_index0;
3200       u32 rx_fib_index0;
3201       ip4_header_t * ip0;
3202       u8 do_handoff;
3203
3204       bi0 = from[0];
3205       from += 1;
3206       n_left_from -= 1;
3207
3208       b0 = vlib_get_buffer (vm, bi0);
3209
3210       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3211       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3212
3213       ip0 = vlib_buffer_get_current (b0);
3214
3215       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3216
3217       if (PREDICT_FALSE (next_worker_index != thread_index))
3218         {
3219           do_handoff = 1;
3220
3221           if (next_worker_index != current_worker_index)
3222             {
3223               if (hf)
3224                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3225
3226               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3227                                                       next_worker_index,
3228                                                       handoff_queue_elt_by_worker_index);
3229
3230               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3231               to_next_worker = &hf->buffer_index[hf->n_vectors];
3232               current_worker_index = next_worker_index;
3233             }
3234
3235           /* enqueue to correct worker thread */
3236           to_next_worker[0] = bi0;
3237           to_next_worker++;
3238           n_left_to_next_worker--;
3239
3240           if (n_left_to_next_worker == 0)
3241             {
3242               hf->n_vectors = VLIB_FRAME_SIZE;
3243               vlib_put_frame_queue_elt (hf);
3244               current_worker_index = ~0;
3245               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3246               hf = 0;
3247             }
3248         }
3249       else
3250         {
3251           do_handoff = 0;
3252           /* if this is 1st frame */
3253           if (!f)
3254             {
3255               f = vlib_get_frame_to_node (vm, to_node_index);
3256               to_next = vlib_frame_vector_args (f);
3257             }
3258
3259           to_next[0] = bi0;
3260           to_next += 1;
3261           f->n_vectors++;
3262         }
3263
3264       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3265                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3266         {
3267           snat_in2out_worker_handoff_trace_t *t =
3268             vlib_add_trace (vm, node, b0, sizeof (*t));
3269           t->next_worker_index = next_worker_index;
3270           t->do_handoff = do_handoff;
3271         }
3272     }
3273
3274   if (f)
3275     vlib_put_frame_to_node (vm, to_node_index, f);
3276
3277   if (hf)
3278     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3279
3280   /* Ship frames to the worker nodes */
3281   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3282     {
3283       if (handoff_queue_elt_by_worker_index[i])
3284         {
3285           hf = handoff_queue_elt_by_worker_index[i];
3286           /*
3287            * It works better to let the handoff node
3288            * rate-adapt, always ship the handoff queue element.
3289            */
3290           if (1 || hf->n_vectors == hf->last_n_vectors)
3291             {
3292               vlib_put_frame_queue_elt (hf);
3293               handoff_queue_elt_by_worker_index[i] = 0;
3294             }
3295           else
3296             hf->last_n_vectors = hf->n_vectors;
3297         }
3298       congested_handoff_queue_by_worker_index[i] =
3299         (vlib_frame_queue_t *) (~0);
3300     }
3301   hf = 0;
3302   current_worker_index = ~0;
3303   return frame->n_vectors;
3304 }
3305
3306 static uword
3307 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3308                                vlib_node_runtime_t * node,
3309                                vlib_frame_t * frame)
3310 {
3311   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3312 }
3313
3314 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3315   .function = snat_in2out_worker_handoff_fn,
3316   .name = "nat44-in2out-worker-handoff",
3317   .vector_size = sizeof (u32),
3318   .format_trace = format_snat_in2out_worker_handoff_trace,
3319   .type = VLIB_NODE_TYPE_INTERNAL,
3320
3321   .n_next_nodes = 1,
3322
3323   .next_nodes = {
3324     [0] = "error-drop",
3325   },
3326 };
3327
3328 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3329                               snat_in2out_worker_handoff_fn);
3330
3331 static uword
3332 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3333                                       vlib_node_runtime_t * node,
3334                                       vlib_frame_t * frame)
3335 {
3336   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3337 }
3338
3339 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3340   .function = snat_in2out_output_worker_handoff_fn,
3341   .name = "nat44-in2out-output-worker-handoff",
3342   .vector_size = sizeof (u32),
3343   .format_trace = format_snat_in2out_worker_handoff_trace,
3344   .type = VLIB_NODE_TYPE_INTERNAL,
3345
3346   .n_next_nodes = 1,
3347
3348   .next_nodes = {
3349     [0] = "error-drop",
3350   },
3351 };
3352
3353 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3354                               snat_in2out_output_worker_handoff_fn);
3355
3356 static_always_inline int
3357 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3358 {
3359   snat_address_t * ap;
3360   clib_bihash_kv_8_8_t kv, value;
3361   snat_session_key_t m_key;
3362
3363   vec_foreach (ap, sm->addresses)
3364     {
3365       if (ap->addr.as_u32 == dst_addr->as_u32)
3366         return 1;
3367     }
3368
3369   m_key.addr.as_u32 = dst_addr->as_u32;
3370   m_key.fib_index = sm->outside_fib_index;
3371   m_key.port = 0;
3372   m_key.protocol = 0;
3373   kv.key = m_key.as_u64;
3374   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3375     return 1;
3376
3377   return 0;
3378 }
3379
3380 static uword
3381 snat_hairpin_dst_fn (vlib_main_t * vm,
3382                      vlib_node_runtime_t * node,
3383                      vlib_frame_t * frame)
3384 {
3385   u32 n_left_from, * from, * to_next;
3386   snat_in2out_next_t next_index;
3387   u32 pkts_processed = 0;
3388   snat_main_t * sm = &snat_main;
3389
3390   from = vlib_frame_vector_args (frame);
3391   n_left_from = frame->n_vectors;
3392   next_index = node->cached_next_index;
3393
3394   while (n_left_from > 0)
3395     {
3396       u32 n_left_to_next;
3397
3398       vlib_get_next_frame (vm, node, next_index,
3399                            to_next, n_left_to_next);
3400
3401       while (n_left_from > 0 && n_left_to_next > 0)
3402         {
3403           u32 bi0;
3404           vlib_buffer_t * b0;
3405           u32 next0;
3406           ip4_header_t * ip0;
3407           u32 proto0;
3408
3409           /* speculatively enqueue b0 to the current next frame */
3410           bi0 = from[0];
3411           to_next[0] = bi0;
3412           from += 1;
3413           to_next += 1;
3414           n_left_from -= 1;
3415           n_left_to_next -= 1;
3416
3417           b0 = vlib_get_buffer (vm, bi0);
3418           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3419           ip0 = vlib_buffer_get_current (b0);
3420
3421           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3422
3423           vnet_buffer (b0)->snat.flags = 0;
3424           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3425             {
3426               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3427                 {
3428                   udp_header_t * udp0 = ip4_next_header (ip0);
3429                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3430
3431                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3432                 }
3433               else if (proto0 == SNAT_PROTOCOL_ICMP)
3434                 {
3435                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3436
3437                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3438                 }
3439               else
3440                 {
3441                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3442                 }
3443
3444               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3445               clib_warning("is hairpinning");
3446             }
3447
3448           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3449
3450           /* verify speculative enqueue, maybe switch current next frame */
3451           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3452                                            to_next, n_left_to_next,
3453                                            bi0, next0);
3454          }
3455
3456       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3457     }
3458
3459   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3460                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3461                                pkts_processed);
3462   return frame->n_vectors;
3463 }
3464
3465 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3466   .function = snat_hairpin_dst_fn,
3467   .name = "nat44-hairpin-dst",
3468   .vector_size = sizeof (u32),
3469   .type = VLIB_NODE_TYPE_INTERNAL,
3470   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3471   .error_strings = snat_in2out_error_strings,
3472   .n_next_nodes = 2,
3473   .next_nodes = {
3474     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3475     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3476   },
3477 };
3478
3479 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3480                               snat_hairpin_dst_fn);
3481
3482 static uword
3483 snat_hairpin_src_fn (vlib_main_t * vm,
3484                      vlib_node_runtime_t * node,
3485                      vlib_frame_t * frame)
3486 {
3487   u32 n_left_from, * from, * to_next;
3488   snat_in2out_next_t next_index;
3489   u32 pkts_processed = 0;
3490   snat_main_t *sm = &snat_main;
3491
3492   from = vlib_frame_vector_args (frame);
3493   n_left_from = frame->n_vectors;
3494   next_index = node->cached_next_index;
3495
3496   while (n_left_from > 0)
3497     {
3498       u32 n_left_to_next;
3499
3500       vlib_get_next_frame (vm, node, next_index,
3501                            to_next, n_left_to_next);
3502
3503       while (n_left_from > 0 && n_left_to_next > 0)
3504         {
3505           u32 bi0;
3506           vlib_buffer_t * b0;
3507           u32 next0;
3508           snat_interface_t *i;
3509           u32 sw_if_index0;
3510
3511           /* speculatively enqueue b0 to the current next frame */
3512           bi0 = from[0];
3513           to_next[0] = bi0;
3514           from += 1;
3515           to_next += 1;
3516           n_left_from -= 1;
3517           n_left_to_next -= 1;
3518
3519           b0 = vlib_get_buffer (vm, bi0);
3520           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3521           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3522
3523           pool_foreach (i, sm->output_feature_interfaces,
3524           ({
3525             /* Only packets from NAT inside interface */
3526             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3527               {
3528                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3529                                     SNAT_FLAG_HAIRPINNING))
3530                   {
3531                     if (PREDICT_TRUE (sm->num_workers > 1))
3532                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3533                     else
3534                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3535                   }
3536                 break;
3537               }
3538           }));
3539
3540           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3541
3542           /* verify speculative enqueue, maybe switch current next frame */
3543           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3544                                            to_next, n_left_to_next,
3545                                            bi0, next0);
3546          }
3547
3548       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3549     }
3550
3551   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3552                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3553                                pkts_processed);
3554   return frame->n_vectors;
3555 }
3556
3557 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3558   .function = snat_hairpin_src_fn,
3559   .name = "nat44-hairpin-src",
3560   .vector_size = sizeof (u32),
3561   .type = VLIB_NODE_TYPE_INTERNAL,
3562   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3563   .error_strings = snat_in2out_error_strings,
3564   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3565   .next_nodes = {
3566      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3567      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3568      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3569      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3570   },
3571 };
3572
3573 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3574                               snat_hairpin_src_fn);
3575
3576 static uword
3577 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3578                                 vlib_node_runtime_t * node,
3579                                 vlib_frame_t * frame)
3580 {
3581   u32 n_left_from, * from, * to_next;
3582   snat_in2out_next_t next_index;
3583   u32 pkts_processed = 0;
3584   snat_main_t * sm = &snat_main;
3585   u32 stats_node_index;
3586
3587   stats_node_index = snat_in2out_fast_node.index;
3588
3589   from = vlib_frame_vector_args (frame);
3590   n_left_from = frame->n_vectors;
3591   next_index = node->cached_next_index;
3592
3593   while (n_left_from > 0)
3594     {
3595       u32 n_left_to_next;
3596
3597       vlib_get_next_frame (vm, node, next_index,
3598                            to_next, n_left_to_next);
3599
3600       while (n_left_from > 0 && n_left_to_next > 0)
3601         {
3602           u32 bi0;
3603           vlib_buffer_t * b0;
3604           u32 next0;
3605           u32 sw_if_index0;
3606           ip4_header_t * ip0;
3607           ip_csum_t sum0;
3608           u32 new_addr0, old_addr0;
3609           u16 old_port0, new_port0;
3610           udp_header_t * udp0;
3611           tcp_header_t * tcp0;
3612           icmp46_header_t * icmp0;
3613           snat_session_key_t key0, sm0;
3614           u32 proto0;
3615           u32 rx_fib_index0;
3616
3617           /* speculatively enqueue b0 to the current next frame */
3618           bi0 = from[0];
3619           to_next[0] = bi0;
3620           from += 1;
3621           to_next += 1;
3622           n_left_from -= 1;
3623           n_left_to_next -= 1;
3624
3625           b0 = vlib_get_buffer (vm, bi0);
3626           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3627
3628           ip0 = vlib_buffer_get_current (b0);
3629           udp0 = ip4_next_header (ip0);
3630           tcp0 = (tcp_header_t *) udp0;
3631           icmp0 = (icmp46_header_t *) udp0;
3632
3633           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3634           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3635
3636           if (PREDICT_FALSE(ip0->ttl == 1))
3637             {
3638               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3639               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3640                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3641                                            0);
3642               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3643               goto trace0;
3644             }
3645
3646           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3647
3648           if (PREDICT_FALSE (proto0 == ~0))
3649               goto trace0;
3650
3651           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3652             {
3653               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3654                                   rx_fib_index0, node, next0, ~0, 0, 0);
3655               goto trace0;
3656             }
3657
3658           key0.addr = ip0->src_address;
3659           key0.protocol = proto0;
3660           key0.port = udp0->src_port;
3661           key0.fib_index = rx_fib_index0;
3662
3663           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
3664             {
3665               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3666               next0= SNAT_IN2OUT_NEXT_DROP;
3667               goto trace0;
3668             }
3669
3670           new_addr0 = sm0.addr.as_u32;
3671           new_port0 = sm0.port;
3672           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3673           old_addr0 = ip0->src_address.as_u32;
3674           ip0->src_address.as_u32 = new_addr0;
3675
3676           sum0 = ip0->checksum;
3677           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3678                                  ip4_header_t,
3679                                  src_address /* changed member */);
3680           ip0->checksum = ip_csum_fold (sum0);
3681
3682           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3683             {
3684               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3685                 {
3686                   old_port0 = tcp0->src_port;
3687                   tcp0->src_port = new_port0;
3688
3689                   sum0 = tcp0->checksum;
3690                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3691                                          ip4_header_t,
3692                                          dst_address /* changed member */);
3693                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3694                                          ip4_header_t /* cheat */,
3695                                          length /* changed member */);
3696                   tcp0->checksum = ip_csum_fold(sum0);
3697                 }
3698               else
3699                 {
3700                   old_port0 = udp0->src_port;
3701                   udp0->src_port = new_port0;
3702                   udp0->checksum = 0;
3703                 }
3704             }
3705           else
3706             {
3707               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3708                 {
3709                   sum0 = tcp0->checksum;
3710                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3711                                          ip4_header_t,
3712                                          dst_address /* changed member */);
3713                   tcp0->checksum = ip_csum_fold(sum0);
3714                 }
3715             }
3716
3717           /* Hairpinning */
3718           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3719
3720         trace0:
3721           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3722                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3723             {
3724               snat_in2out_trace_t *t =
3725                  vlib_add_trace (vm, node, b0, sizeof (*t));
3726               t->sw_if_index = sw_if_index0;
3727               t->next_index = next0;
3728             }
3729
3730           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3731
3732           /* verify speculative enqueue, maybe switch current next frame */
3733           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3734                                            to_next, n_left_to_next,
3735                                            bi0, next0);
3736         }
3737
3738       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3739     }
3740
3741   vlib_node_increment_counter (vm, stats_node_index,
3742                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3743                                pkts_processed);
3744   return frame->n_vectors;
3745 }
3746
3747
3748 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3749   .function = snat_in2out_fast_static_map_fn,
3750   .name = "nat44-in2out-fast",
3751   .vector_size = sizeof (u32),
3752   .format_trace = format_snat_in2out_fast_trace,
3753   .type = VLIB_NODE_TYPE_INTERNAL,
3754
3755   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3756   .error_strings = snat_in2out_error_strings,
3757
3758   .runtime_data_bytes = sizeof (snat_runtime_t),
3759
3760   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3761
3762   /* edit / add dispositions here */
3763   .next_nodes = {
3764     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3765     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3766     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3767     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3768   },
3769 };
3770
3771 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);