src/plugins/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25 #include <snat/snat_ipfix_logging.h>
  26 #include <snat/snat_det.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36   u32 is_slow_path;
  37 } snat_in2out_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_in2out_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  50   char * tag;
  51
  52   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  53
  54   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  55               t->sw_if_index, t->next_index, t->session_index);
  56
  57   return s;
  58 }
  59
  60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  65
  66   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68
  69   return s;
  70 }
  71
  72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  73 {
  74   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  75   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  76   snat_in2out_worker_handoff_trace_t * t =
  77     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  78   char * m;
  79
  80   m = t->do_handoff ? "next worker" : "same worker";
  81   s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  82
  83   return s;
  84 }
  85
  86 vlib_node_registration_t snat_in2out_node;
  87 vlib_node_registration_t snat_in2out_slowpath_node;
  88 vlib_node_registration_t snat_in2out_fast_node;
  89 vlib_node_registration_t snat_in2out_worker_handoff_node;
  90 vlib_node_registration_t snat_det_in2out_node;
  91
  92 #define foreach_snat_in2out_error                       \
  93 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  94 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  95 _(OUT_OF_PORTS, "Out of ports")                         \
  96 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  97 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  98 _(NO_TRANSLATION, "No translation")
  99
 100 typedef enum {
 101 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 102   foreach_snat_in2out_error
 103 #undef _
 104   SNAT_IN2OUT_N_ERROR,
 105 } snat_in2out_error_t;
 106
 107 static char * snat_in2out_error_strings[] = {
 108 #define _(sym,string) string,
 109   foreach_snat_in2out_error
 110 #undef _
 111 };
 112
 113 typedef enum {
 114   SNAT_IN2OUT_NEXT_LOOKUP,
 115   SNAT_IN2OUT_NEXT_DROP,
 116   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 117   SNAT_IN2OUT_NEXT_SLOW_PATH,
 118   SNAT_IN2OUT_N_NEXT,
 119 } snat_in2out_next_t;
 120
 121 /**
 122  * @brief Check if packet should be translated
 123  *
 124  * Packets aimed at outside interface and external addresss with active session
 125  * should be translated.
 126  *
 127  * @param sm            SNAT main
 128  * @param rt            SNAT runtime data
 129  * @param sw_if_index0  index of the inside interface
 130  * @param ip0           IPv4 header
 131  * @param proto0        SNAT protocol
 132  * @param rx_fib_index0 RX FIB index
 133  *
 134  * @returns 0 if packet should be translated otherwise 1
 135  */
 136 static inline int
 137 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 138                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 139                          u32 rx_fib_index0)
 140 {
 141   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 142   fib_prefix_t pfx = {
 143     .fp_proto = FIB_PROTOCOL_IP4,
 144     .fp_len = 32,
 145     .fp_addr = {
 146         .ip4.as_u32 = ip0->dst_address.as_u32,
 147     },
 148   };
 149
 150   /* Don't NAT packet aimed at the intfc address */
 151   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 152                                       ip0->dst_address.as_u32)))
 153     return 1;
 154
 155   fei = fib_table_lookup (rx_fib_index0, &pfx);
 156   if (FIB_NODE_INDEX_INVALID != fei)
 157     {
 158       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 159       if (sw_if_index == ~0)
 160         {
 161           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 162           if (FIB_NODE_INDEX_INVALID != fei)
 163             sw_if_index = fib_entry_get_resolving_interface (fei);
 164         }
 165       snat_interface_t *i;
 166       pool_foreach (i, sm->interfaces,
 167       ({
 168         /* NAT packet aimed at outside interface */
 169         if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
 170           return 0;
 171       }));
 172     }
 173
 174   return 1;
 175 }
 176
 177 static inline int
 178 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 179                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                     u32 rx_fib_index0)
 181 {
 182   udp_header_t * udp0 = ip4_next_header (ip0);
 183   snat_session_key_t key0, sm0;
 184   clib_bihash_kv_8_8_t kv0, value0;
 185
 186   key0.addr = ip0->dst_address;
 187   key0.port = udp0->dst_port;
 188   key0.protocol = proto0;
 189   key0.fib_index = sm->outside_fib_index;
 190   kv0.key = key0.as_u64;
 191
 192   /* NAT packet aimed at external address if */
 193   /* has active sessions */
 194   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 195     {
 196       /* or is static mappings */
 197       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 198         return 0;
 199     }
 200   else
 201     return 0;
 202
 203   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 204                                  rx_fib_index0);
 205 }
 206
 207 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 208                       ip4_header_t * ip0,
 209                       u32 rx_fib_index0,
 210                       snat_session_key_t * key0,
 211                       snat_session_t ** sessionp,
 212                       vlib_node_runtime_t * node,
 213                       u32 next0,
 214                       u32 thread_index)
 215 {
 216   snat_user_t *u;
 217   snat_user_key_t user_key;
 218   snat_session_t *s;
 219   clib_bihash_kv_8_8_t kv0, value0;
 220   u32 oldest_per_user_translation_list_index;
 221   dlist_elt_t * oldest_per_user_translation_list_elt;
 222   dlist_elt_t * per_user_translation_list_elt;
 223   dlist_elt_t * per_user_list_head_elt;
 224   u32 session_index;
 225   snat_session_key_t key1;
 226   u32 address_index = ~0;
 227   u32 outside_fib_index;
 228   uword * p;
 229   snat_worker_key_t worker_by_out_key;
 230
 231   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 232   if (! p)
 233     {
 234       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 235       return SNAT_IN2OUT_NEXT_DROP;
 236     }
 237   outside_fib_index = p[0];
 238
 239   key1.protocol = key0->protocol;
 240   user_key.addr = ip0->src_address;
 241   user_key.fib_index = rx_fib_index0;
 242   kv0.key = user_key.as_u64;
 243
 244   /* Ever heard of the "user" = src ip4 address before? */
 245   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 246     {
 247       /* no, make a new one */
 248       pool_get (sm->per_thread_data[thread_index].users, u);
 249       memset (u, 0, sizeof (*u));
 250       u->addr = ip0->src_address;
 251       u->fib_index = rx_fib_index0;
 252
 253       pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
 254
 255       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 256         sm->per_thread_data[thread_index].list_pool;
 257
 258       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 259                        u->sessions_per_user_list_head_index);
 260
 261       kv0.value = u - sm->per_thread_data[thread_index].users;
 262
 263       /* add user */
 264       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 265     }
 266   else
 267     {
 268       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 269                              value0.value);
 270     }
 271
 272   /* Over quota? Recycle the least recently used dynamic translation */
 273   if (u->nsessions >= sm->max_translations_per_user)
 274     {
 275       /* Remove the oldest dynamic translation */
 276       do {
 277           oldest_per_user_translation_list_index =
 278             clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
 279                                     u->sessions_per_user_list_head_index);
 280
 281           ASSERT (oldest_per_user_translation_list_index != ~0);
 282
 283           /* add it back to the end of the LRU list */
 284           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 285                               u->sessions_per_user_list_head_index,
 286                               oldest_per_user_translation_list_index);
 287           /* Get the list element */
 288           oldest_per_user_translation_list_elt =
 289             pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
 290                                oldest_per_user_translation_list_index);
 291
 292           /* Get the session index from the list element */
 293           session_index = oldest_per_user_translation_list_elt->value;
 294
 295           /* Get the session */
 296           s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 297                                  session_index);
 298       } while (snat_is_session_static (s));
 299
 300       if (snat_is_unk_proto_session (s))
 301         {
 302           clib_bihash_kv_16_8_t up_kv;
 303           snat_unk_proto_ses_key_t key;
 304
 305           /* Remove from lookup tables */
 306           key.l_addr = s->in2out.addr;
 307           key.r_addr = s->ext_host_addr;
 308           key.fib_index = s->in2out.fib_index;
 309           key.proto = s->in2out.port;
 310           up_kv.key[0] = key.as_u64[0];
 311           up_kv.key[1] = key.as_u64[1];
 312           if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &up_kv, 0))
 313             clib_warning ("in2out key del failed");
 314
 315           key.l_addr = s->out2in.addr;
 316           key.fib_index = s->out2in.fib_index;
 317           up_kv.key[0] = key.as_u64[0];
 318           up_kv.key[1] = key.as_u64[1];
 319           if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &up_kv, 0))
 320             clib_warning ("out2in key del failed");
 321         }
 322       else
 323         {
 324           /* Remove in2out, out2in keys */
 325           kv0.key = s->in2out.as_u64;
 326           if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 327               clib_warning ("in2out key delete failed");
 328           kv0.key = s->out2in.as_u64;
 329           if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 330               clib_warning ("out2in key delete failed");
 331
 332           /* log NAT event */
 333           snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 334                                               s->out2in.addr.as_u32,
 335                                               s->in2out.protocol,
 336                                               s->in2out.port,
 337                                               s->out2in.port,
 338                                               s->in2out.fib_index);
 339
 340           snat_free_outside_address_and_port
 341             (sm, &s->out2in, s->outside_address_index);
 342         }
 343       s->outside_address_index = ~0;
 344
 345       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 346                                                &address_index))
 347         {
 348           ASSERT(0);
 349
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353       s->outside_address_index = address_index;
 354     }
 355   else
 356     {
 357       u8 static_mapping = 1;
 358
 359       /* First try to match static mapping by local address and port */
 360       if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
 361         {
 362           static_mapping = 0;
 363           /* Try to create dynamic translation */
 364           if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
 365                                                    &address_index))
 366             {
 367               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 368               return SNAT_IN2OUT_NEXT_DROP;
 369             }
 370         }
 371
 372       /* Create a new session */
 373       pool_get (sm->per_thread_data[thread_index].sessions, s);
 374       memset (s, 0, sizeof (*s));
 375
 376       s->outside_address_index = address_index;
 377
 378       if (static_mapping)
 379         {
 380           u->nstaticsessions++;
 381           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 382         }
 383       else
 384         {
 385           u->nsessions++;
 386         }
 387
 388       /* Create list elts */
 389       pool_get (sm->per_thread_data[thread_index].list_pool,
 390                 per_user_translation_list_elt);
 391       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 392                        per_user_translation_list_elt -
 393                        sm->per_thread_data[thread_index].list_pool);
 394
 395       per_user_translation_list_elt->value =
 396         s - sm->per_thread_data[thread_index].sessions;
 397       s->per_user_index = per_user_translation_list_elt -
 398                           sm->per_thread_data[thread_index].list_pool;
 399       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 400
 401       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 402                           s->per_user_list_head_index,
 403                           per_user_translation_list_elt -
 404                           sm->per_thread_data[thread_index].list_pool);
 405    }
 406
 407   s->in2out = *key0;
 408   s->out2in = key1;
 409   s->out2in.protocol = key0->protocol;
 410   s->out2in.fib_index = outside_fib_index;
 411   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 412   *sessionp = s;
 413
 414   /* Add to translation hashes */
 415   kv0.key = s->in2out.as_u64;
 416   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 417   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 418       clib_warning ("in2out key add failed");
 419
 420   kv0.key = s->out2in.as_u64;
 421   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 422
 423   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 424       clib_warning ("out2in key add failed");
 425
 426   /* Add to translated packets worker lookup */
 427   worker_by_out_key.addr = s->out2in.addr;
 428   worker_by_out_key.port = s->out2in.port;
 429   worker_by_out_key.fib_index = s->out2in.fib_index;
 430   kv0.key = worker_by_out_key.as_u64;
 431   kv0.value = thread_index;
 432   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 433
 434   /* log NAT event */
 435   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 436                                       s->out2in.addr.as_u32,
 437                                       s->in2out.protocol,
 438                                       s->in2out.port,
 439                                       s->out2in.port,
 440                                       s->in2out.fib_index);
 441   return next0;
 442 }
 443
 444 static_always_inline
 445 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 446                                  snat_session_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   snat_session_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.protocol = SNAT_PROTOCOL_ICMP;
 461       key0.addr = ip0->src_address;
 462       key0.port = echo0->identifier;
 463     }
 464   else
 465     {
 466       inner_ip0 = (ip4_header_t *)(echo0+1);
 467       l4_header = ip4_next_header (inner_ip0);
 468       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 469       key0.addr = inner_ip0->dst_address;
 470       switch (key0.protocol)
 471         {
 472         case SNAT_PROTOCOL_ICMP:
 473           inner_icmp0 = (icmp46_header_t*)l4_header;
 474           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 475           key0.port = inner_echo0->identifier;
 476           break;
 477         case SNAT_PROTOCOL_UDP:
 478         case SNAT_PROTOCOL_TCP:
 479           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 480           break;
 481         default:
 482           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 483         }
 484     }
 485   *p_key0 = key0;
 486   return -1; /* success */
 487 }
 488
 489 /**
 490  * Get address and port values to be used for packet SNAT translation
 491  * and create session if needed
 492  *
 493  * @param[in,out] sm             SNAT main
 494  * @param[in,out] node           SNAT node runtime
 495  * @param[in] thread_index       thread index
 496  * @param[in,out] b0             buffer containing packet to be translated
 497  * @param[out] p_proto           protocol used for matching
 498  * @param[out] p_value           address and port after NAT translation
 499  * @param[out] p_dont_translate  if packet should not be translated
 500  * @param d                      optional parameter
 501  * @param e                      optional parameter
 502  */
 503 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 504                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 505                            snat_session_key_t *p_value,
 506                            u8 *p_dont_translate, void *d, void *e)
 507 {
 508   ip4_header_t *ip0;
 509   icmp46_header_t *icmp0;
 510   u32 sw_if_index0;
 511   u32 rx_fib_index0;
 512   snat_session_key_t key0;
 513   snat_session_t *s0 = 0;
 514   u8 dont_translate = 0;
 515   clib_bihash_kv_8_8_t kv0, value0;
 516   u32 next0 = ~0;
 517   int err;
 518
 519   ip0 = vlib_buffer_get_current (b0);
 520   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 521   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 522   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 523
 524   err = icmp_get_key (ip0, &key0);
 525   if (err != -1)
 526     {
 527       b0->error = node->errors[err];
 528       next0 = SNAT_IN2OUT_NEXT_DROP;
 529       goto out;
 530     }
 531   key0.fib_index = rx_fib_index0;
 532
 533   kv0.key = key0.as_u64;
 534
 535   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 536     {
 537       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
 538           IP_PROTOCOL_ICMP, rx_fib_index0)))
 539         {
 540           dont_translate = 1;
 541           goto out;
 542         }
 543
 544       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 545         {
 546           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 547           next0 = SNAT_IN2OUT_NEXT_DROP;
 548           goto out;
 549         }
 550
 551       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 552                          &s0, node, next0, thread_index);
 553
 554       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 555         goto out;
 556     }
 557   else
 558     {
 559       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 560                         icmp0->type != ICMP4_echo_reply &&
 561                         !icmp_is_error_message (icmp0)))
 562         {
 563           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 564           next0 = SNAT_IN2OUT_NEXT_DROP;
 565           goto out;
 566         }
 567
 568       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 569                               value0.value);
 570     }
 571
 572 out:
 573   *p_proto = key0.protocol;
 574   if (s0)
 575     *p_value = s0->out2in;
 576   *p_dont_translate = dont_translate;
 577   if (d)
 578     *(snat_session_t**)d = s0;
 579   return next0;
 580 }
 581
 582 /**
 583  * Get address and port values to be used for packet SNAT translation
 584  *
 585  * @param[in] sm                 SNAT main
 586  * @param[in,out] node           SNAT node runtime
 587  * @param[in] thread_index       thread index
 588  * @param[in,out] b0             buffer containing packet to be translated
 589  * @param[out] p_proto           protocol used for matching
 590  * @param[out] p_value           address and port after NAT translation
 591  * @param[out] p_dont_translate  if packet should not be translated
 592  * @param d                      optional parameter
 593  * @param e                      optional parameter
 594  */
 595 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 596                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 597                            snat_session_key_t *p_value,
 598                            u8 *p_dont_translate, void *d, void *e)
 599 {
 600   ip4_header_t *ip0;
 601   icmp46_header_t *icmp0;
 602   u32 sw_if_index0;
 603   u32 rx_fib_index0;
 604   snat_session_key_t key0;
 605   snat_session_key_t sm0;
 606   u8 dont_translate = 0;
 607   u8 is_addr_only;
 608   u32 next0 = ~0;
 609   int err;
 610
 611   ip0 = vlib_buffer_get_current (b0);
 612   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 613   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 614   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 615
 616   err = icmp_get_key (ip0, &key0);
 617   if (err != -1)
 618     {
 619       b0->error = node->errors[err];
 620       next0 = SNAT_IN2OUT_NEXT_DROP;
 621       goto out2;
 622     }
 623   key0.fib_index = rx_fib_index0;
 624
 625   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
 626     {
 627       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 628           IP_PROTOCOL_ICMP, rx_fib_index0)))
 629         {
 630           dont_translate = 1;
 631           goto out;
 632         }
 633
 634       if (icmp_is_error_message (icmp0))
 635         {
 636           next0 = SNAT_IN2OUT_NEXT_DROP;
 637           goto out;
 638         }
 639
 640       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 641       next0 = SNAT_IN2OUT_NEXT_DROP;
 642       goto out;
 643     }
 644
 645   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 646                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 647                     !icmp_is_error_message (icmp0)))
 648     {
 649       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 650       next0 = SNAT_IN2OUT_NEXT_DROP;
 651       goto out;
 652     }
 653
 654 out:
 655   *p_value = sm0;
 656 out2:
 657   *p_proto = key0.protocol;
 658   *p_dont_translate = dont_translate;
 659   return next0;
 660 }
 661
 662 static inline u32 icmp_in2out (snat_main_t *sm,
 663                                vlib_buffer_t * b0,
 664                                ip4_header_t * ip0,
 665                                icmp46_header_t * icmp0,
 666                                u32 sw_if_index0,
 667                                u32 rx_fib_index0,
 668                                vlib_node_runtime_t * node,
 669                                u32 next0,
 670                                u32 thread_index,
 671                                void *d,
 672                                void *e)
 673 {
 674   snat_session_key_t sm0;
 675   u8 protocol;
 676   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 677   ip4_header_t *inner_ip0;
 678   void *l4_header = 0;
 679   icmp46_header_t *inner_icmp0;
 680   u8 dont_translate;
 681   u32 new_addr0, old_addr0;
 682   u16 old_id0, new_id0;
 683   ip_csum_t sum0;
 684   u16 checksum0;
 685   u32 next0_tmp;
 686
 687   echo0 = (icmp_echo_header_t *)(icmp0+1);
 688
 689   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
 690                                        &protocol, &sm0, &dont_translate, d, e);
 691   if (next0_tmp != ~0)
 692     next0 = next0_tmp;
 693   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 694     goto out;
 695
 696   sum0 = ip_incremental_checksum (0, icmp0,
 697                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 698   checksum0 = ~ip_csum_fold (sum0);
 699   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 700     {
 701       next0 = SNAT_IN2OUT_NEXT_DROP;
 702       goto out;
 703     }
 704
 705   old_addr0 = ip0->src_address.as_u32;
 706   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 707   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 708
 709   sum0 = ip0->checksum;
 710   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 711                          src_address /* changed member */);
 712   ip0->checksum = ip_csum_fold (sum0);
 713
 714   if (!icmp_is_error_message (icmp0))
 715     {
 716       new_id0 = sm0.port;
 717       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 718         {
 719           old_id0 = echo0->identifier;
 720           new_id0 = sm0.port;
 721           echo0->identifier = new_id0;
 722
 723           sum0 = icmp0->checksum;
 724           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 725                                  identifier);
 726           icmp0->checksum = ip_csum_fold (sum0);
 727         }
 728     }
 729   else
 730     {
 731       inner_ip0 = (ip4_header_t *)(echo0+1);
 732       l4_header = ip4_next_header (inner_ip0);
 733
 734       if (!ip4_header_checksum_is_valid (inner_ip0))
 735         {
 736           next0 = SNAT_IN2OUT_NEXT_DROP;
 737           goto out;
 738         }
 739
 740       old_addr0 = inner_ip0->dst_address.as_u32;
 741       inner_ip0->dst_address = sm0.addr;
 742       new_addr0 = inner_ip0->dst_address.as_u32;
 743
 744       sum0 = icmp0->checksum;
 745       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 746                              dst_address /* changed member */);
 747       icmp0->checksum = ip_csum_fold (sum0);
 748
 749       switch (protocol)
 750         {
 751           case SNAT_PROTOCOL_ICMP:
 752             inner_icmp0 = (icmp46_header_t*)l4_header;
 753             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 754
 755             old_id0 = inner_echo0->identifier;
 756             new_id0 = sm0.port;
 757             inner_echo0->identifier = new_id0;
 758
 759             sum0 = icmp0->checksum;
 760             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 761                                    identifier);
 762             icmp0->checksum = ip_csum_fold (sum0);
 763             break;
 764           case SNAT_PROTOCOL_UDP:
 765           case SNAT_PROTOCOL_TCP:
 766             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 767             new_id0 = sm0.port;
 768             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 769
 770             sum0 = icmp0->checksum;
 771             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 772                                    dst_port);
 773             icmp0->checksum = ip_csum_fold (sum0);
 774             break;
 775           default:
 776             ASSERT(0);
 777         }
 778     }
 779
 780 out:
 781   return next0;
 782 }
 783
 784 /**
 785  * @brief Hairpinning
 786  *
 787  * Hairpinning allows two endpoints on the internal side of the NAT to
 788  * communicate even if they only use each other's external IP addresses
 789  * and ports.
 790  *
 791  * @param sm     SNAT main.
 792  * @param b0     Vlib buffer.
 793  * @param ip0    IP header.
 794  * @param udp0   UDP header.
 795  * @param tcp0   TCP header.
 796  * @param proto0 SNAT protocol.
 797  */
 798 static inline void
 799 snat_hairpinning (snat_main_t *sm,
 800                   vlib_buffer_t * b0,
 801                   ip4_header_t * ip0,
 802                   udp_header_t * udp0,
 803                   tcp_header_t * tcp0,
 804                   u32 proto0)
 805 {
 806   snat_session_key_t key0, sm0;
 807   snat_worker_key_t k0;
 808   snat_session_t * s0;
 809   clib_bihash_kv_8_8_t kv0, value0;
 810   ip_csum_t sum0;
 811   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 812   u16 new_dst_port0, old_dst_port0;
 813
 814   key0.addr = ip0->dst_address;
 815   key0.port = udp0->dst_port;
 816   key0.protocol = proto0;
 817   key0.fib_index = sm->outside_fib_index;
 818   kv0.key = key0.as_u64;
 819
 820   /* Check if destination is in active sessions */
 821   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 822     {
 823       /* or static mappings */
 824       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 825         {
 826           new_dst_addr0 = sm0.addr.as_u32;
 827           new_dst_port0 = sm0.port;
 828           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 829         }
 830     }
 831   else
 832     {
 833       si = value0.value;
 834       if (sm->num_workers > 1)
 835         {
 836           k0.addr = ip0->dst_address;
 837           k0.port = udp0->dst_port;
 838           k0.fib_index = sm->outside_fib_index;
 839           kv0.key = k0.as_u64;
 840           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 841             ASSERT(0);
 842           else
 843             ti = value0.value;
 844         }
 845       else
 846         ti = sm->num_workers;
 847
 848       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 849       new_dst_addr0 = s0->in2out.addr.as_u32;
 850       new_dst_port0 = s0->in2out.port;
 851       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 852     }
 853
 854   /* Destination is behind the same NAT, use internal address and port */
 855   if (new_dst_addr0)
 856     {
 857       old_dst_addr0 = ip0->dst_address.as_u32;
 858       ip0->dst_address.as_u32 = new_dst_addr0;
 859       sum0 = ip0->checksum;
 860       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 861                              ip4_header_t, dst_address);
 862       ip0->checksum = ip_csum_fold (sum0);
 863
 864       old_dst_port0 = tcp0->dst;
 865       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 866         {
 867           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 868             {
 869               tcp0->dst = new_dst_port0;
 870               sum0 = tcp0->checksum;
 871               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 872                                      ip4_header_t, dst_address);
 873               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 874                                      ip4_header_t /* cheat */, length);
 875               tcp0->checksum = ip_csum_fold(sum0);
 876             }
 877           else
 878             {
 879               udp0->dst_port = new_dst_port0;
 880               udp0->checksum = 0;
 881             }
 882         }
 883       else
 884         {
 885           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 886             {
 887               sum0 = tcp0->checksum;
 888               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 889                                      ip4_header_t, dst_address);
 890               tcp0->checksum = ip_csum_fold(sum0);
 891             }
 892         }
 893     }
 894 }
 895
 896 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 897                                          vlib_buffer_t * b0,
 898                                          ip4_header_t * ip0,
 899                                          icmp46_header_t * icmp0,
 900                                          u32 sw_if_index0,
 901                                          u32 rx_fib_index0,
 902                                          vlib_node_runtime_t * node,
 903                                          u32 next0,
 904                                          f64 now,
 905                                          u32 thread_index,
 906                                          snat_session_t ** p_s0)
 907 {
 908   snat_session_key_t key0, sm0;
 909   clib_bihash_kv_8_8_t kv0, value0;
 910   snat_worker_key_t k0;
 911   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 912   ip_csum_t sum0;
 913
 914   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 915                       next0, thread_index, p_s0, 0);
 916   snat_session_t * s0 = *p_s0;
 917   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 918     {
 919       /* Hairpinning */
 920       if (!icmp_is_error_message (icmp0))
 921         {
 922           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 923           u16 icmp_id0 = echo0->identifier;
 924           key0.addr = ip0->dst_address;
 925           key0.port = icmp_id0;
 926           key0.protocol = SNAT_PROTOCOL_ICMP;
 927           key0.fib_index = sm->outside_fib_index;
 928           kv0.key = key0.as_u64;
 929
 930           /* Check if destination is in active sessions */
 931           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 932             {
 933               /* or static mappings */
 934               if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 935                 {
 936                   new_dst_addr0 = sm0.addr.as_u32;
 937                   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 938                 }
 939             }
 940           else
 941             {
 942               si = value0.value;
 943               if (sm->num_workers > 1)
 944                 {
 945                   k0.addr = ip0->dst_address;
 946                   k0.port = icmp_id0;
 947                   k0.fib_index = sm->outside_fib_index;
 948                   kv0.key = k0.as_u64;
 949                   if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 950                     ASSERT(0);
 951                   else
 952                     ti = value0.value;
 953                 }
 954               else
 955                 ti = sm->num_workers;
 956
 957               s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 958               new_dst_addr0 = s0->in2out.addr.as_u32;
 959               vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 960               echo0->identifier = s0->in2out.port;
 961               sum0 = icmp0->checksum;
 962               sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 963                                      icmp_echo_header_t, identifier);
 964               icmp0->checksum = ip_csum_fold (sum0);
 965             }
 966
 967           /* Destination is behind the same NAT, use internal address and port */
 968           if (new_dst_addr0)
 969             {
 970               old_dst_addr0 = ip0->dst_address.as_u32;
 971               ip0->dst_address.as_u32 = new_dst_addr0;
 972               sum0 = ip0->checksum;
 973               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 974                                      ip4_header_t, dst_address);
 975               ip0->checksum = ip_csum_fold (sum0);
 976             }
 977         }
 978
 979       /* Accounting */
 980       s0->last_heard = now;
 981       s0->total_pkts++;
 982       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 983       /* Per-user LRU list maintenance for dynamic translations */
 984       if (!snat_is_session_static (s0))
 985         {
 986           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 987                              s0->per_user_index);
 988           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 989                               s0->per_user_list_head_index,
 990                               s0->per_user_index);
 991         }
 992     }
 993   return next0;
 994 }
 995
 996 static void
 997 snat_in2out_unknown_proto (snat_main_t *sm,
 998                            vlib_buffer_t * b,
 999                            ip4_header_t * ip,
1000                            u32 rx_fib_index,
1001                            u32 thread_index,
1002                            f64 now,
1003                            vlib_main_t * vm)
1004 {
1005   clib_bihash_kv_8_8_t kv, value;
1006   clib_bihash_kv_16_8_t s_kv, s_value;
1007   snat_static_mapping_t *m;
1008   snat_session_key_t m_key;
1009   u32 old_addr, new_addr = 0;
1010   ip_csum_t sum;
1011   snat_user_key_t u_key;
1012   snat_user_t *u;
1013   dlist_elt_t *head, *elt, *oldest;
1014   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1015   u32 elt_index, head_index, ses_index, oldest_index;
1016   snat_session_t * s;
1017   snat_unk_proto_ses_key_t key;
1018   u32 address_index = ~0;
1019   int i;
1020   u8 is_sm = 0;
1021
1022   old_addr = ip->src_address.as_u32;
1023
1024   key.l_addr = ip->src_address;
1025   key.r_addr = ip->dst_address;
1026   key.fib_index = rx_fib_index;
1027   key.proto = ip->protocol;
1028   key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
1029   s_kv.key[0] = key.as_u64[0];
1030   s_kv.key[1] = key.as_u64[1];
1031
1032   if (!clib_bihash_search_16_8 (&sm->in2out_unk_proto, &s_kv, &s_value))
1033     {
1034       s = pool_elt_at_index (tsm->sessions, s_value.value);
1035       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1036     }
1037   else
1038     {
1039       u_key.addr = ip->src_address;
1040       u_key.fib_index = rx_fib_index;
1041       kv.key = u_key.as_u64;
1042
1043       /* Ever heard of the "user" = src ip4 address before? */
1044       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1045         {
1046           /* no, make a new one */
1047           pool_get (tsm->users, u);
1048           memset (u, 0, sizeof (*u));
1049           u->addr = ip->src_address;
1050           u->fib_index = rx_fib_index;
1051
1052           pool_get (tsm->list_pool, head);
1053           u->sessions_per_user_list_head_index = head - tsm->list_pool;
1054
1055           clib_dlist_init (tsm->list_pool,
1056                            u->sessions_per_user_list_head_index);
1057
1058           kv.value = u - tsm->users;
1059
1060           /* add user */
1061           clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
1062         }
1063       else
1064         {
1065           u = pool_elt_at_index (tsm->users, value.value);
1066         }
1067
1068       m_key.addr = ip->src_address;
1069       m_key.port = 0;
1070       m_key.protocol = 0;
1071       m_key.fib_index = rx_fib_index;
1072       kv.key = m_key.as_u64;
1073
1074       /* Try to find static mapping first */
1075       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1076         {
1077           m = pool_elt_at_index (sm->static_mappings, value.value);
1078           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1079           is_sm = 1;
1080           goto create_ses;
1081         }
1082       /* Fallback to 3-tuple key */
1083       else
1084         {
1085           /* Choose same out address as for TCP/UDP session to same destination */
1086           if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1087             {
1088               head_index = u->sessions_per_user_list_head_index;
1089               head = pool_elt_at_index (tsm->list_pool, head_index);
1090               elt_index = head->next;
1091               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1092               ses_index = elt->value;
1093               while (ses_index != ~0)
1094                 {
1095                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1096                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1097                     {
1098                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1099                       address_index = s->outside_address_index;
1100
1101                       key.fib_index = sm->outside_fib_index;
1102                       key.l_addr.as_u32 = new_addr;
1103                       s_kv.key[0] = key.as_u64[0];
1104                       s_kv.key[1] = key.as_u64[1];
1105                       if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1106                         break;
1107
1108                       goto create_ses;
1109                     }
1110                 }
1111             }
1112           key.fib_index = sm->outside_fib_index;
1113           for (i = 0; i < vec_len (sm->addresses); i++)
1114             {
1115               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1116               s_kv.key[0] = key.as_u64[0];
1117               s_kv.key[1] = key.as_u64[1];
1118               if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1119                 {
1120                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1121                   address_index = i;
1122                   goto create_ses;
1123                 }
1124             }
1125           return;
1126         }
1127
1128 create_ses:
1129       /* Over quota? Recycle the least recently used dynamic translation */
1130       if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1131         {
1132           /* Remove the oldest dynamic translation */
1133           do {
1134               oldest_index = clib_dlist_remove_head (
1135                 tsm->list_pool, u->sessions_per_user_list_head_index);
1136
1137               ASSERT (oldest_index != ~0);
1138
1139               /* add it back to the end of the LRU list */
1140               clib_dlist_addtail (tsm->list_pool,
1141                                   u->sessions_per_user_list_head_index,
1142                                   oldest_index);
1143               /* Get the list element */
1144               oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1145
1146               /* Get the session index from the list element */
1147               ses_index = oldest->value;
1148
1149               /* Get the session */
1150               s = pool_elt_at_index (tsm->sessions, ses_index);
1151           } while (snat_is_session_static (s));
1152
1153           if (snat_is_unk_proto_session (s))
1154             {
1155               /* Remove from lookup tables */
1156               key.l_addr = s->in2out.addr;
1157               key.r_addr = s->ext_host_addr;
1158               key.fib_index = s->in2out.fib_index;
1159               key.proto = s->in2out.port;
1160               s_kv.key[0] = key.as_u64[0];
1161               s_kv.key[1] = key.as_u64[1];
1162               if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 0))
1163                 clib_warning ("in2out key del failed");
1164
1165               key.l_addr = s->out2in.addr;
1166               key.fib_index = s->out2in.fib_index;
1167               s_kv.key[0] = key.as_u64[0];
1168               s_kv.key[1] = key.as_u64[1];
1169               if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 0))
1170                 clib_warning ("out2in key del failed");
1171             }
1172           else
1173             {
1174               /* log NAT event */
1175               snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1176                                                   s->out2in.addr.as_u32,
1177                                                   s->in2out.protocol,
1178                                                   s->in2out.port,
1179                                                   s->out2in.port,
1180                                                   s->in2out.fib_index);
1181
1182               snat_free_outside_address_and_port (sm, &s->out2in,
1183                                                   s->outside_address_index);
1184
1185               /* Remove in2out, out2in keys */
1186               kv.key = s->in2out.as_u64;
1187               if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
1188                 clib_warning ("in2out key del failed");
1189               kv.key = s->out2in.as_u64;
1190               if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
1191                 clib_warning ("out2in key del failed");
1192             }
1193         }
1194       else
1195         {
1196           /* Create a new session */
1197           pool_get (tsm->sessions, s);
1198           memset (s, 0, sizeof (*s));
1199
1200           /* Create list elts */
1201           pool_get (tsm->list_pool, elt);
1202           clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1203           elt->value = s - tsm->sessions;
1204           s->per_user_index = elt - tsm->list_pool;
1205           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1206           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1207                               s->per_user_index);
1208         }
1209
1210       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1211       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1212       s->outside_address_index = address_index;
1213       s->out2in.addr.as_u32 = new_addr;
1214       s->out2in.fib_index = sm->outside_fib_index;
1215       s->in2out.addr.as_u32 = old_addr;
1216       s->in2out.fib_index = rx_fib_index;
1217       s->in2out.port = s->out2in.port = ip->protocol;
1218       if (is_sm)
1219         {
1220           u->nstaticsessions++;
1221           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1222         }
1223       else
1224         {
1225           u->nsessions++;
1226         }
1227
1228       /* Add to lookup tables */
1229       key.l_addr.as_u32 = old_addr;
1230       key.r_addr = ip->dst_address;
1231       key.proto = ip->protocol;
1232       key.fib_index = rx_fib_index;
1233       s_kv.key[0] = key.as_u64[0];
1234       s_kv.key[1] = key.as_u64[1];
1235       s_kv.value = s - tsm->sessions;
1236       if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
1237         clib_warning ("in2out key add failed");
1238
1239       key.l_addr.as_u32 = new_addr;
1240       key.fib_index = sm->outside_fib_index;
1241       s_kv.key[0] = key.as_u64[0];
1242       s_kv.key[1] = key.as_u64[1];
1243       if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
1244         clib_warning ("out2in key add failed");
1245   }
1246
1247   /* Update IP checksum */
1248   sum = ip->checksum;
1249   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1250   ip->checksum = ip_csum_fold (sum);
1251
1252   /* Accounting */
1253   s->last_heard = now;
1254   s->total_pkts++;
1255   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1256   /* Per-user LRU list maintenance */
1257   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1258   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1259                       s->per_user_index);
1260
1261   /* Hairpinning */
1262   old_addr = ip->dst_address.as_u32;
1263   key.l_addr.as_u32 = ip->dst_address.as_u32;
1264   key.r_addr.as_u32 = new_addr;
1265   key.fib_index = sm->outside_fib_index;
1266   s_kv.key[0] = key.as_u64[0];
1267   s_kv.key[1] = key.as_u64[1];
1268   if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1269     {
1270       m_key.addr = ip->dst_address;
1271       m_key.fib_index = sm->outside_fib_index;
1272       kv.key = m_key.as_u64;
1273       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1274         {
1275           vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1276           return;
1277         }
1278
1279       m = pool_elt_at_index (sm->static_mappings, value.value);
1280       vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1281       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1282     }
1283   else
1284     {
1285       s = pool_elt_at_index (tsm->sessions, s_value.value);
1286       vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1287       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1288     }
1289   sum = ip->checksum;
1290   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1291   ip->checksum = ip_csum_fold (sum);
1292 }
1293
1294 static inline uword
1295 snat_in2out_node_fn_inline (vlib_main_t * vm,
1296                             vlib_node_runtime_t * node,
1297                             vlib_frame_t * frame, int is_slow_path)
1298 {
1299   u32 n_left_from, * from, * to_next;
1300   snat_in2out_next_t next_index;
1301   u32 pkts_processed = 0;
1302   snat_main_t * sm = &snat_main;
1303   f64 now = vlib_time_now (vm);
1304   u32 stats_node_index;
1305   u32 thread_index = vlib_get_thread_index ();
1306
1307   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1308     snat_in2out_node.index;
1309
1310   from = vlib_frame_vector_args (frame);
1311   n_left_from = frame->n_vectors;
1312   next_index = node->cached_next_index;
1313
1314   while (n_left_from > 0)
1315     {
1316       u32 n_left_to_next;
1317
1318       vlib_get_next_frame (vm, node, next_index,
1319                            to_next, n_left_to_next);
1320
1321       while (n_left_from >= 4 && n_left_to_next >= 2)
1322         {
1323           u32 bi0, bi1;
1324           vlib_buffer_t * b0, * b1;
1325           u32 next0, next1;
1326           u32 sw_if_index0, sw_if_index1;
1327           ip4_header_t * ip0, * ip1;
1328           ip_csum_t sum0, sum1;
1329           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1330           u16 old_port0, new_port0, old_port1, new_port1;
1331           udp_header_t * udp0, * udp1;
1332           tcp_header_t * tcp0, * tcp1;
1333           icmp46_header_t * icmp0, * icmp1;
1334           snat_session_key_t key0, key1;
1335           u32 rx_fib_index0, rx_fib_index1;
1336           u32 proto0, proto1;
1337           snat_session_t * s0 = 0, * s1 = 0;
1338           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1339
1340           /* Prefetch next iteration. */
1341           {
1342             vlib_buffer_t * p2, * p3;
1343
1344             p2 = vlib_get_buffer (vm, from[2]);
1345             p3 = vlib_get_buffer (vm, from[3]);
1346
1347             vlib_prefetch_buffer_header (p2, LOAD);
1348             vlib_prefetch_buffer_header (p3, LOAD);
1349
1350             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1351             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1352           }
1353
1354           /* speculatively enqueue b0 and b1 to the current next frame */
1355           to_next[0] = bi0 = from[0];
1356           to_next[1] = bi1 = from[1];
1357           from += 2;
1358           to_next += 2;
1359           n_left_from -= 2;
1360           n_left_to_next -= 2;
1361
1362           b0 = vlib_get_buffer (vm, bi0);
1363           b1 = vlib_get_buffer (vm, bi1);
1364
1365           ip0 = vlib_buffer_get_current (b0);
1366           udp0 = ip4_next_header (ip0);
1367           tcp0 = (tcp_header_t *) udp0;
1368           icmp0 = (icmp46_header_t *) udp0;
1369
1370           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1371           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1372                                    sw_if_index0);
1373
1374           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1375
1376           if (PREDICT_FALSE(ip0->ttl == 1))
1377             {
1378               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1379               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1380                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1381                                            0);
1382               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1383               goto trace00;
1384             }
1385
1386           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1387
1388           /* Next configured feature, probably ip4-lookup */
1389           if (is_slow_path)
1390             {
1391               if (PREDICT_FALSE (proto0 == ~0))
1392                 {
1393                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1394                                              thread_index, now, vm);
1395                   goto trace00;
1396                 }
1397
1398               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1399                 {
1400                   next0 = icmp_in2out_slow_path
1401                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1402                      node, next0, now, thread_index, &s0);
1403                   goto trace00;
1404                 }
1405             }
1406           else
1407             {
1408               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1409                 {
1410                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1411                   goto trace00;
1412                 }
1413             }
1414
1415           key0.addr = ip0->src_address;
1416           key0.port = udp0->src_port;
1417           key0.protocol = proto0;
1418           key0.fib_index = rx_fib_index0;
1419
1420           kv0.key = key0.as_u64;
1421
1422           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
1423             {
1424               if (is_slow_path)
1425                 {
1426                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
1427                       proto0, rx_fib_index0)))
1428                     goto trace00;
1429
1430                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1431                                      &s0, node, next0, thread_index);
1432                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1433                     goto trace00;
1434                 }
1435               else
1436                 {
1437                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1438                   goto trace00;
1439                 }
1440             }
1441           else
1442             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1443                                     value0.value);
1444
1445           old_addr0 = ip0->src_address.as_u32;
1446           ip0->src_address = s0->out2in.addr;
1447           new_addr0 = ip0->src_address.as_u32;
1448           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1449
1450           sum0 = ip0->checksum;
1451           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1452                                  ip4_header_t,
1453                                  src_address /* changed member */);
1454           ip0->checksum = ip_csum_fold (sum0);
1455
1456           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1457             {
1458               old_port0 = tcp0->src_port;
1459               tcp0->src_port = s0->out2in.port;
1460               new_port0 = tcp0->src_port;
1461
1462               sum0 = tcp0->checksum;
1463               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1464                                      ip4_header_t,
1465                                      dst_address /* changed member */);
1466               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1467                                      ip4_header_t /* cheat */,
1468                                      length /* changed member */);
1469               tcp0->checksum = ip_csum_fold(sum0);
1470             }
1471           else
1472             {
1473               old_port0 = udp0->src_port;
1474               udp0->src_port = s0->out2in.port;
1475               udp0->checksum = 0;
1476             }
1477
1478           /* Hairpinning */
1479           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1480
1481           /* Accounting */
1482           s0->last_heard = now;
1483           s0->total_pkts++;
1484           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1485           /* Per-user LRU list maintenance for dynamic translation */
1486           if (!snat_is_session_static (s0))
1487             {
1488               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1489                                  s0->per_user_index);
1490               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1491                                   s0->per_user_list_head_index,
1492                                   s0->per_user_index);
1493             }
1494         trace00:
1495
1496           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1497                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1498             {
1499               snat_in2out_trace_t *t =
1500                  vlib_add_trace (vm, node, b0, sizeof (*t));
1501               t->is_slow_path = is_slow_path;
1502               t->sw_if_index = sw_if_index0;
1503               t->next_index = next0;
1504                   t->session_index = ~0;
1505               if (s0)
1506                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1507             }
1508
1509           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1510
1511           ip1 = vlib_buffer_get_current (b1);
1512           udp1 = ip4_next_header (ip1);
1513           tcp1 = (tcp_header_t *) udp1;
1514           icmp1 = (icmp46_header_t *) udp1;
1515
1516           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1517           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1518                                    sw_if_index1);
1519
1520           if (PREDICT_FALSE(ip1->ttl == 1))
1521             {
1522               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1523               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1524                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1525                                            0);
1526               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1527               goto trace01;
1528             }
1529
1530           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1531
1532           /* Next configured feature, probably ip4-lookup */
1533           if (is_slow_path)
1534             {
1535               if (PREDICT_FALSE (proto1 == ~0))
1536                 {
1537                   snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1538                                              thread_index, now, vm);
1539                   goto trace01;
1540                 }
1541
1542               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1543                 {
1544                   next1 = icmp_in2out_slow_path
1545                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1546                      next1, now, thread_index, &s1);
1547                   goto trace01;
1548                 }
1549             }
1550           else
1551             {
1552               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1553                 {
1554                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1555                   goto trace01;
1556                 }
1557             }
1558
1559           key1.addr = ip1->src_address;
1560           key1.port = udp1->src_port;
1561           key1.protocol = proto1;
1562           key1.fib_index = rx_fib_index1;
1563
1564           kv1.key = key1.as_u64;
1565
1566             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
1567             {
1568               if (is_slow_path)
1569                 {
1570                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1, ip1,
1571                       proto1, rx_fib_index1)))
1572                     goto trace01;
1573
1574                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1575                                      &s1, node, next1, thread_index);
1576                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1577                     goto trace01;
1578                 }
1579               else
1580                 {
1581                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1582                   goto trace01;
1583                 }
1584             }
1585           else
1586             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1587                                     value1.value);
1588
1589           old_addr1 = ip1->src_address.as_u32;
1590           ip1->src_address = s1->out2in.addr;
1591           new_addr1 = ip1->src_address.as_u32;
1592           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1593
1594           sum1 = ip1->checksum;
1595           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1596                                  ip4_header_t,
1597                                  src_address /* changed member */);
1598           ip1->checksum = ip_csum_fold (sum1);
1599
1600           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1601             {
1602               old_port1 = tcp1->src_port;
1603               tcp1->src_port = s1->out2in.port;
1604               new_port1 = tcp1->src_port;
1605
1606               sum1 = tcp1->checksum;
1607               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1608                                      ip4_header_t,
1609                                      dst_address /* changed member */);
1610               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1611                                      ip4_header_t /* cheat */,
1612                                      length /* changed member */);
1613               tcp1->checksum = ip_csum_fold(sum1);
1614             }
1615           else
1616             {
1617               old_port1 = udp1->src_port;
1618               udp1->src_port = s1->out2in.port;
1619               udp1->checksum = 0;
1620             }
1621
1622           /* Hairpinning */
1623           snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1624
1625           /* Accounting */
1626           s1->last_heard = now;
1627           s1->total_pkts++;
1628           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1629           /* Per-user LRU list maintenance for dynamic translation */
1630           if (!snat_is_session_static (s1))
1631             {
1632               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1633                                  s1->per_user_index);
1634               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1635                                   s1->per_user_list_head_index,
1636                                   s1->per_user_index);
1637             }
1638         trace01:
1639
1640           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1641                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1642             {
1643               snat_in2out_trace_t *t =
1644                  vlib_add_trace (vm, node, b1, sizeof (*t));
1645               t->sw_if_index = sw_if_index1;
1646               t->next_index = next1;
1647               t->session_index = ~0;
1648               if (s1)
1649                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1650             }
1651
1652           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1653
1654           /* verify speculative enqueues, maybe switch current next frame */
1655           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1656                                            to_next, n_left_to_next,
1657                                            bi0, bi1, next0, next1);
1658         }
1659
1660       while (n_left_from > 0 && n_left_to_next > 0)
1661         {
1662           u32 bi0;
1663           vlib_buffer_t * b0;
1664           u32 next0;
1665           u32 sw_if_index0;
1666           ip4_header_t * ip0;
1667           ip_csum_t sum0;
1668           u32 new_addr0, old_addr0;
1669           u16 old_port0, new_port0;
1670           udp_header_t * udp0;
1671           tcp_header_t * tcp0;
1672           icmp46_header_t * icmp0;
1673           snat_session_key_t key0;
1674           u32 rx_fib_index0;
1675           u32 proto0;
1676           snat_session_t * s0 = 0;
1677           clib_bihash_kv_8_8_t kv0, value0;
1678
1679           /* speculatively enqueue b0 to the current next frame */
1680           bi0 = from[0];
1681           to_next[0] = bi0;
1682           from += 1;
1683           to_next += 1;
1684           n_left_from -= 1;
1685           n_left_to_next -= 1;
1686
1687           b0 = vlib_get_buffer (vm, bi0);
1688           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1689
1690           ip0 = vlib_buffer_get_current (b0);
1691           udp0 = ip4_next_header (ip0);
1692           tcp0 = (tcp_header_t *) udp0;
1693           icmp0 = (icmp46_header_t *) udp0;
1694
1695           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1696           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1697                                    sw_if_index0);
1698
1699           if (PREDICT_FALSE(ip0->ttl == 1))
1700             {
1701               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1702               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1703                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1704                                            0);
1705               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1706               goto trace0;
1707             }
1708
1709           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1710
1711           /* Next configured feature, probably ip4-lookup */
1712           if (is_slow_path)
1713             {
1714               if (PREDICT_FALSE (proto0 == ~0))
1715                 {
1716                   snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1717                                              thread_index, now, vm);
1718                   goto trace0;
1719                 }
1720
1721               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1722                 {
1723                   next0 = icmp_in2out_slow_path
1724                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1725                      next0, now, thread_index, &s0);
1726                   goto trace0;
1727                 }
1728             }
1729           else
1730             {
1731               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1732                 {
1733                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1734                   goto trace0;
1735                 }
1736             }
1737
1738           key0.addr = ip0->src_address;
1739           key0.port = udp0->src_port;
1740           key0.protocol = proto0;
1741           key0.fib_index = rx_fib_index0;
1742
1743           kv0.key = key0.as_u64;
1744
1745           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
1746             {
1747               if (is_slow_path)
1748                 {
1749                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
1750                       proto0, rx_fib_index0)))
1751                     goto trace0;
1752
1753                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1754                                      &s0, node, next0, thread_index);
1755
1756                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1757                     goto trace0;
1758                 }
1759               else
1760                 {
1761                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1762                   goto trace0;
1763                 }
1764             }
1765           else
1766             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1767                                     value0.value);
1768
1769           old_addr0 = ip0->src_address.as_u32;
1770           ip0->src_address = s0->out2in.addr;
1771           new_addr0 = ip0->src_address.as_u32;
1772           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1773
1774           sum0 = ip0->checksum;
1775           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1776                                  ip4_header_t,
1777                                  src_address /* changed member */);
1778           ip0->checksum = ip_csum_fold (sum0);
1779
1780           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1781             {
1782               old_port0 = tcp0->src_port;
1783               tcp0->src_port = s0->out2in.port;
1784               new_port0 = tcp0->src_port;
1785
1786               sum0 = tcp0->checksum;
1787               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1788                                      ip4_header_t,
1789                                      dst_address /* changed member */);
1790               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1791                                      ip4_header_t /* cheat */,
1792                                      length /* changed member */);
1793               tcp0->checksum = ip_csum_fold(sum0);
1794             }
1795           else
1796             {
1797               old_port0 = udp0->src_port;
1798               udp0->src_port = s0->out2in.port;
1799               udp0->checksum = 0;
1800             }
1801
1802           /* Hairpinning */
1803           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1804
1805           /* Accounting */
1806           s0->last_heard = now;
1807           s0->total_pkts++;
1808           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1809           /* Per-user LRU list maintenance for dynamic translation */
1810           if (!snat_is_session_static (s0))
1811             {
1812               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1813                                  s0->per_user_index);
1814               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1815                                   s0->per_user_list_head_index,
1816                                   s0->per_user_index);
1817             }
1818
1819         trace0:
1820           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1821                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1822             {
1823               snat_in2out_trace_t *t =
1824                  vlib_add_trace (vm, node, b0, sizeof (*t));
1825               t->is_slow_path = is_slow_path;
1826               t->sw_if_index = sw_if_index0;
1827               t->next_index = next0;
1828                   t->session_index = ~0;
1829               if (s0)
1830                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1831             }
1832
1833           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1834
1835           /* verify speculative enqueue, maybe switch current next frame */
1836           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1837                                            to_next, n_left_to_next,
1838                                            bi0, next0);
1839         }
1840
1841       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1842     }
1843
1844   vlib_node_increment_counter (vm, stats_node_index,
1845                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1846                                pkts_processed);
1847   return frame->n_vectors;
1848 }
1849
1850 static uword
1851 snat_in2out_fast_path_fn (vlib_main_t * vm,
1852                           vlib_node_runtime_t * node,
1853                           vlib_frame_t * frame)
1854 {
1855   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
1856 }
1857
1858 VLIB_REGISTER_NODE (snat_in2out_node) = {
1859   .function = snat_in2out_fast_path_fn,
1860   .name = "snat-in2out",
1861   .vector_size = sizeof (u32),
1862   .format_trace = format_snat_in2out_trace,
1863   .type = VLIB_NODE_TYPE_INTERNAL,
1864
1865   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1866   .error_strings = snat_in2out_error_strings,
1867
1868   .runtime_data_bytes = sizeof (snat_runtime_t),
1869
1870   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1871
1872   /* edit / add dispositions here */
1873   .next_nodes = {
1874     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1875     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1876     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1877     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1878   },
1879 };
1880
1881 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1882
1883 static uword
1884 snat_in2out_slow_path_fn (vlib_main_t * vm,
1885                           vlib_node_runtime_t * node,
1886                           vlib_frame_t * frame)
1887 {
1888   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
1889 }
1890
1891 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1892   .function = snat_in2out_slow_path_fn,
1893   .name = "snat-in2out-slowpath",
1894   .vector_size = sizeof (u32),
1895   .format_trace = format_snat_in2out_trace,
1896   .type = VLIB_NODE_TYPE_INTERNAL,
1897
1898   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1899   .error_strings = snat_in2out_error_strings,
1900
1901   .runtime_data_bytes = sizeof (snat_runtime_t),
1902
1903   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1904
1905   /* edit / add dispositions here */
1906   .next_nodes = {
1907     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1908     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1909     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1910     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1911   },
1912 };
1913
1914 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1915
1916 /**************************/
1917 /*** deterministic mode ***/
1918 /**************************/
1919 static uword
1920 snat_det_in2out_node_fn (vlib_main_t * vm,
1921                          vlib_node_runtime_t * node,
1922                          vlib_frame_t * frame)
1923 {
1924   u32 n_left_from, * from, * to_next;
1925   snat_in2out_next_t next_index;
1926   u32 pkts_processed = 0;
1927   snat_main_t * sm = &snat_main;
1928   u32 now = (u32) vlib_time_now (vm);
1929   u32 thread_index = vlib_get_thread_index ();
1930
1931   from = vlib_frame_vector_args (frame);
1932   n_left_from = frame->n_vectors;
1933   next_index = node->cached_next_index;
1934
1935   while (n_left_from > 0)
1936     {
1937       u32 n_left_to_next;
1938
1939       vlib_get_next_frame (vm, node, next_index,
1940                            to_next, n_left_to_next);
1941
1942       while (n_left_from >= 4 && n_left_to_next >= 2)
1943         {
1944           u32 bi0, bi1;
1945           vlib_buffer_t * b0, * b1;
1946           u32 next0, next1;
1947           u32 sw_if_index0, sw_if_index1;
1948           ip4_header_t * ip0, * ip1;
1949           ip_csum_t sum0, sum1;
1950           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1951           u16 old_port0, new_port0, lo_port0, i0;
1952           u16 old_port1, new_port1, lo_port1, i1;
1953           udp_header_t * udp0, * udp1;
1954           tcp_header_t * tcp0, * tcp1;
1955           u32 proto0, proto1;
1956           snat_det_out_key_t key0, key1;
1957           snat_det_map_t * dm0, * dm1;
1958           snat_det_session_t * ses0 = 0, * ses1 = 0;
1959           u32 rx_fib_index0, rx_fib_index1;
1960           icmp46_header_t * icmp0, * icmp1;
1961
1962           /* Prefetch next iteration. */
1963           {
1964             vlib_buffer_t * p2, * p3;
1965
1966             p2 = vlib_get_buffer (vm, from[2]);
1967             p3 = vlib_get_buffer (vm, from[3]);
1968
1969             vlib_prefetch_buffer_header (p2, LOAD);
1970             vlib_prefetch_buffer_header (p3, LOAD);
1971
1972             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1973             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1974           }
1975
1976           /* speculatively enqueue b0 and b1 to the current next frame */
1977           to_next[0] = bi0 = from[0];
1978           to_next[1] = bi1 = from[1];
1979           from += 2;
1980           to_next += 2;
1981           n_left_from -= 2;
1982           n_left_to_next -= 2;
1983
1984           b0 = vlib_get_buffer (vm, bi0);
1985           b1 = vlib_get_buffer (vm, bi1);
1986
1987           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1988           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1989
1990           ip0 = vlib_buffer_get_current (b0);
1991           udp0 = ip4_next_header (ip0);
1992           tcp0 = (tcp_header_t *) udp0;
1993
1994           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1995
1996           if (PREDICT_FALSE(ip0->ttl == 1))
1997             {
1998               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1999               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2000                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2001                                            0);
2002               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2003               goto trace0;
2004             }
2005
2006           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2007
2008           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2009             {
2010               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2011               icmp0 = (icmp46_header_t *) udp0;
2012
2013               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2014                                   rx_fib_index0, node, next0, thread_index,
2015                                   &ses0, &dm0);
2016               goto trace0;
2017             }
2018
2019           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2020           if (PREDICT_FALSE(!dm0))
2021             {
2022               clib_warning("no match for internal host %U",
2023                            format_ip4_address, &ip0->src_address);
2024               next0 = SNAT_IN2OUT_NEXT_DROP;
2025               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2026               goto trace0;
2027             }
2028
2029           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2030
2031           key0.ext_host_addr = ip0->dst_address;
2032           key0.ext_host_port = tcp0->dst;
2033
2034           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2035           if (PREDICT_FALSE(!ses0))
2036             {
2037               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2038                 {
2039                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2040                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2041
2042                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2043                     continue;
2044
2045                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2046                   break;
2047                 }
2048               if (PREDICT_FALSE(!ses0))
2049                 {
2050                   /* too many sessions for user, send ICMP error packet */
2051
2052                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2053                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2054                                                ICMP4_destination_unreachable_destination_unreachable_host,
2055                                                0);
2056                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2057                   goto trace0;
2058                 }
2059             }
2060
2061           new_port0 = ses0->out.out_port;
2062
2063           old_addr0.as_u32 = ip0->src_address.as_u32;
2064           ip0->src_address.as_u32 = new_addr0.as_u32;
2065           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2066
2067           sum0 = ip0->checksum;
2068           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2069                                  ip4_header_t,
2070                                  src_address /* changed member */);
2071           ip0->checksum = ip_csum_fold (sum0);
2072
2073           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2074             {
2075               if (tcp0->flags & TCP_FLAG_SYN)
2076                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2077               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2078                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2079               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2080                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2081               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2082                 snat_det_ses_close(dm0, ses0);
2083               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2084                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2085               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2086                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2087
2088               old_port0 = tcp0->src;
2089               tcp0->src = new_port0;
2090
2091               sum0 = tcp0->checksum;
2092               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2093                                      ip4_header_t,
2094                                      dst_address /* changed member */);
2095               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2096                                      ip4_header_t /* cheat */,
2097                                      length /* changed member */);
2098               tcp0->checksum = ip_csum_fold(sum0);
2099             }
2100           else
2101             {
2102               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2103               old_port0 = udp0->src_port;
2104               udp0->src_port = new_port0;
2105               udp0->checksum = 0;
2106             }
2107
2108           switch(ses0->state)
2109             {
2110             case SNAT_SESSION_UDP_ACTIVE:
2111                 ses0->expire = now + sm->udp_timeout;
2112                 break;
2113             case SNAT_SESSION_TCP_SYN_SENT:
2114             case SNAT_SESSION_TCP_FIN_WAIT:
2115             case SNAT_SESSION_TCP_CLOSE_WAIT:
2116             case SNAT_SESSION_TCP_LAST_ACK:
2117                 ses0->expire = now + sm->tcp_transitory_timeout;
2118                 break;
2119             case SNAT_SESSION_TCP_ESTABLISHED:
2120                 ses0->expire = now + sm->tcp_established_timeout;
2121                 break;
2122             }
2123
2124         trace0:
2125           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2126                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2127             {
2128               snat_in2out_trace_t *t =
2129                  vlib_add_trace (vm, node, b0, sizeof (*t));
2130               t->is_slow_path = 0;
2131               t->sw_if_index = sw_if_index0;
2132               t->next_index = next0;
2133               t->session_index = ~0;
2134               if (ses0)
2135                 t->session_index = ses0 - dm0->sessions;
2136             }
2137
2138           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2139
2140           ip1 = vlib_buffer_get_current (b1);
2141           udp1 = ip4_next_header (ip1);
2142           tcp1 = (tcp_header_t *) udp1;
2143
2144           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2145
2146           if (PREDICT_FALSE(ip1->ttl == 1))
2147             {
2148               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2149               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2150                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2151                                            0);
2152               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2153               goto trace1;
2154             }
2155
2156           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2157
2158           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2159             {
2160               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2161               icmp1 = (icmp46_header_t *) udp1;
2162
2163               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2164                                   rx_fib_index1, node, next1, thread_index,
2165                                   &ses1, &dm1);
2166               goto trace1;
2167             }
2168
2169           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2170           if (PREDICT_FALSE(!dm1))
2171             {
2172               clib_warning("no match for internal host %U",
2173                            format_ip4_address, &ip0->src_address);
2174               next1 = SNAT_IN2OUT_NEXT_DROP;
2175               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2176               goto trace1;
2177             }
2178
2179           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2180
2181           key1.ext_host_addr = ip1->dst_address;
2182           key1.ext_host_port = tcp1->dst;
2183
2184           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2185           if (PREDICT_FALSE(!ses1))
2186             {
2187               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2188                 {
2189                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
2190                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2191
2192                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2193                     continue;
2194
2195                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2196                   break;
2197                 }
2198               if (PREDICT_FALSE(!ses1))
2199                 {
2200                   /* too many sessions for user, send ICMP error packet */
2201
2202                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2203                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2204                                                ICMP4_destination_unreachable_destination_unreachable_host,
2205                                                0);
2206                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2207                   goto trace1;
2208                 }
2209             }
2210
2211           new_port1 = ses1->out.out_port;
2212
2213           old_addr1.as_u32 = ip1->src_address.as_u32;
2214           ip1->src_address.as_u32 = new_addr1.as_u32;
2215           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2216
2217           sum1 = ip1->checksum;
2218           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2219                                  ip4_header_t,
2220                                  src_address /* changed member */);
2221           ip1->checksum = ip_csum_fold (sum1);
2222
2223           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2224             {
2225               if (tcp1->flags & TCP_FLAG_SYN)
2226                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2227               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2228                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2229               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2230                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2231               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2232                 snat_det_ses_close(dm1, ses1);
2233               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2234                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2235               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2236                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2237
2238               old_port1 = tcp1->src;
2239               tcp1->src = new_port1;
2240
2241               sum1 = tcp1->checksum;
2242               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2243                                      ip4_header_t,
2244                                      dst_address /* changed member */);
2245               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2246                                      ip4_header_t /* cheat */,
2247                                      length /* changed member */);
2248               tcp1->checksum = ip_csum_fold(sum1);
2249             }
2250           else
2251             {
2252               ses1->state = SNAT_SESSION_UDP_ACTIVE;
2253               old_port1 = udp1->src_port;
2254               udp1->src_port = new_port1;
2255               udp1->checksum = 0;
2256             }
2257
2258           switch(ses1->state)
2259             {
2260             case SNAT_SESSION_UDP_ACTIVE:
2261                 ses1->expire = now + sm->udp_timeout;
2262                 break;
2263             case SNAT_SESSION_TCP_SYN_SENT:
2264             case SNAT_SESSION_TCP_FIN_WAIT:
2265             case SNAT_SESSION_TCP_CLOSE_WAIT:
2266             case SNAT_SESSION_TCP_LAST_ACK:
2267                 ses1->expire = now + sm->tcp_transitory_timeout;
2268                 break;
2269             case SNAT_SESSION_TCP_ESTABLISHED:
2270                 ses1->expire = now + sm->tcp_established_timeout;
2271                 break;
2272             }
2273
2274         trace1:
2275           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2276                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2277             {
2278               snat_in2out_trace_t *t =
2279                  vlib_add_trace (vm, node, b1, sizeof (*t));
2280               t->is_slow_path = 0;
2281               t->sw_if_index = sw_if_index1;
2282               t->next_index = next1;
2283               t->session_index = ~0;
2284               if (ses1)
2285                 t->session_index = ses1 - dm1->sessions;
2286             }
2287
2288           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2289
2290           /* verify speculative enqueues, maybe switch current next frame */
2291           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2292                                            to_next, n_left_to_next,
2293                                            bi0, bi1, next0, next1);
2294          }
2295
2296       while (n_left_from > 0 && n_left_to_next > 0)
2297         {
2298           u32 bi0;
2299           vlib_buffer_t * b0;
2300           u32 next0;
2301           u32 sw_if_index0;
2302           ip4_header_t * ip0;
2303           ip_csum_t sum0;
2304           ip4_address_t new_addr0, old_addr0;
2305           u16 old_port0, new_port0, lo_port0, i0;
2306           udp_header_t * udp0;
2307           tcp_header_t * tcp0;
2308           u32 proto0;
2309           snat_det_out_key_t key0;
2310           snat_det_map_t * dm0;
2311           snat_det_session_t * ses0 = 0;
2312           u32 rx_fib_index0;
2313           icmp46_header_t * icmp0;
2314
2315           /* speculatively enqueue b0 to the current next frame */
2316           bi0 = from[0];
2317           to_next[0] = bi0;
2318           from += 1;
2319           to_next += 1;
2320           n_left_from -= 1;
2321           n_left_to_next -= 1;
2322
2323           b0 = vlib_get_buffer (vm, bi0);
2324           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2325
2326           ip0 = vlib_buffer_get_current (b0);
2327           udp0 = ip4_next_header (ip0);
2328           tcp0 = (tcp_header_t *) udp0;
2329
2330           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2331
2332           if (PREDICT_FALSE(ip0->ttl == 1))
2333             {
2334               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2335               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2336                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2337                                            0);
2338               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2339               goto trace00;
2340             }
2341
2342           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2343
2344           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2345             {
2346               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2347               icmp0 = (icmp46_header_t *) udp0;
2348
2349               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2350                                   rx_fib_index0, node, next0, thread_index,
2351                                   &ses0, &dm0);
2352               goto trace00;
2353             }
2354
2355           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2356           if (PREDICT_FALSE(!dm0))
2357             {
2358               clib_warning("no match for internal host %U",
2359                            format_ip4_address, &ip0->src_address);
2360               next0 = SNAT_IN2OUT_NEXT_DROP;
2361               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2362               goto trace00;
2363             }
2364
2365           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2366
2367           key0.ext_host_addr = ip0->dst_address;
2368           key0.ext_host_port = tcp0->dst;
2369
2370           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2371           if (PREDICT_FALSE(!ses0))
2372             {
2373               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2374                 {
2375                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2376                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2377
2378                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2379                     continue;
2380
2381                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2382                   break;
2383                 }
2384               if (PREDICT_FALSE(!ses0))
2385                 {
2386                   /* too many sessions for user, send ICMP error packet */
2387
2388                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2389                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2390                                                ICMP4_destination_unreachable_destination_unreachable_host,
2391                                                0);
2392                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2393                   goto trace00;
2394                 }
2395             }
2396
2397           new_port0 = ses0->out.out_port;
2398
2399           old_addr0.as_u32 = ip0->src_address.as_u32;
2400           ip0->src_address.as_u32 = new_addr0.as_u32;
2401           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2402
2403           sum0 = ip0->checksum;
2404           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2405                                  ip4_header_t,
2406                                  src_address /* changed member */);
2407           ip0->checksum = ip_csum_fold (sum0);
2408
2409           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2410             {
2411               if (tcp0->flags & TCP_FLAG_SYN)
2412                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2413               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2414                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2415               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2416                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2417               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2418                 snat_det_ses_close(dm0, ses0);
2419               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2420                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2421               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2422                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2423
2424               old_port0 = tcp0->src;
2425               tcp0->src = new_port0;
2426
2427               sum0 = tcp0->checksum;
2428               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2429                                      ip4_header_t,
2430                                      dst_address /* changed member */);
2431               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2432                                      ip4_header_t /* cheat */,
2433                                      length /* changed member */);
2434               tcp0->checksum = ip_csum_fold(sum0);
2435             }
2436           else
2437             {
2438               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2439               old_port0 = udp0->src_port;
2440               udp0->src_port = new_port0;
2441               udp0->checksum = 0;
2442             }
2443
2444           switch(ses0->state)
2445             {
2446             case SNAT_SESSION_UDP_ACTIVE:
2447                 ses0->expire = now + sm->udp_timeout;
2448                 break;
2449             case SNAT_SESSION_TCP_SYN_SENT:
2450             case SNAT_SESSION_TCP_FIN_WAIT:
2451             case SNAT_SESSION_TCP_CLOSE_WAIT:
2452             case SNAT_SESSION_TCP_LAST_ACK:
2453                 ses0->expire = now + sm->tcp_transitory_timeout;
2454                 break;
2455             case SNAT_SESSION_TCP_ESTABLISHED:
2456                 ses0->expire = now + sm->tcp_established_timeout;
2457                 break;
2458             }
2459
2460         trace00:
2461           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2462                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2463             {
2464               snat_in2out_trace_t *t =
2465                  vlib_add_trace (vm, node, b0, sizeof (*t));
2466               t->is_slow_path = 0;
2467               t->sw_if_index = sw_if_index0;
2468               t->next_index = next0;
2469               t->session_index = ~0;
2470               if (ses0)
2471                 t->session_index = ses0 - dm0->sessions;
2472             }
2473
2474           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2475
2476           /* verify speculative enqueue, maybe switch current next frame */
2477           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2478                                            to_next, n_left_to_next,
2479                                            bi0, next0);
2480         }
2481
2482       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2483     }
2484
2485   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2486                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2487                                pkts_processed);
2488   return frame->n_vectors;
2489 }
2490
2491 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2492   .function = snat_det_in2out_node_fn,
2493   .name = "snat-det-in2out",
2494   .vector_size = sizeof (u32),
2495   .format_trace = format_snat_in2out_trace,
2496   .type = VLIB_NODE_TYPE_INTERNAL,
2497
2498   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2499   .error_strings = snat_in2out_error_strings,
2500
2501   .runtime_data_bytes = sizeof (snat_runtime_t),
2502
2503   .n_next_nodes = 3,
2504
2505   /* edit / add dispositions here */
2506   .next_nodes = {
2507     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2508     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2509     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2510   },
2511 };
2512
2513 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2514
2515 /**
2516  * Get address and port values to be used for packet SNAT translation
2517  * and create session if needed
2518  *
2519  * @param[in,out] sm             SNAT main
2520  * @param[in,out] node           SNAT node runtime
2521  * @param[in] thread_index       thread index
2522  * @param[in,out] b0             buffer containing packet to be translated
2523  * @param[out] p_proto           protocol used for matching
2524  * @param[out] p_value           address and port after NAT translation
2525  * @param[out] p_dont_translate  if packet should not be translated
2526  * @param d                      optional parameter
2527  * @param e                      optional parameter
2528  */
2529 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
2530                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2531                           snat_session_key_t *p_value,
2532                           u8 *p_dont_translate, void *d, void *e)
2533 {
2534   ip4_header_t *ip0;
2535   icmp46_header_t *icmp0;
2536   u32 sw_if_index0;
2537   u32 rx_fib_index0;
2538   u8 protocol;
2539   snat_det_out_key_t key0;
2540   u8 dont_translate = 0;
2541   u32 next0 = ~0;
2542   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2543   ip4_header_t *inner_ip0;
2544   void *l4_header = 0;
2545   icmp46_header_t *inner_icmp0;
2546   snat_det_map_t * dm0 = 0;
2547   ip4_address_t new_addr0;
2548   u16 lo_port0, i0;
2549   snat_det_session_t * ses0 = 0;
2550   ip4_address_t in_addr;
2551   u16 in_port;
2552
2553   ip0 = vlib_buffer_get_current (b0);
2554   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2555   echo0 = (icmp_echo_header_t *)(icmp0+1);
2556   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2557   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2558
2559   if (!icmp_is_error_message (icmp0))
2560     {
2561       protocol = SNAT_PROTOCOL_ICMP;
2562       in_addr = ip0->src_address;
2563       in_port = echo0->identifier;
2564     }
2565   else
2566     {
2567       inner_ip0 = (ip4_header_t *)(echo0+1);
2568       l4_header = ip4_next_header (inner_ip0);
2569       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2570       in_addr = inner_ip0->dst_address;
2571       switch (protocol)
2572         {
2573         case SNAT_PROTOCOL_ICMP:
2574           inner_icmp0 = (icmp46_header_t*)l4_header;
2575           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2576           in_port = inner_echo0->identifier;
2577           break;
2578         case SNAT_PROTOCOL_UDP:
2579         case SNAT_PROTOCOL_TCP:
2580           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2581           break;
2582         default:
2583           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
2584           next0 = SNAT_IN2OUT_NEXT_DROP;
2585           goto out;
2586         }
2587     }
2588
2589   dm0 = snat_det_map_by_user(sm, &in_addr);
2590   if (PREDICT_FALSE(!dm0))
2591     {
2592       clib_warning("no match for internal host %U",
2593                    format_ip4_address, &in_addr);
2594       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2595           IP_PROTOCOL_ICMP, rx_fib_index0)))
2596         {
2597           dont_translate = 1;
2598           goto out;
2599         }
2600       next0 = SNAT_IN2OUT_NEXT_DROP;
2601       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2602       goto out;
2603     }
2604
2605   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
2606
2607   key0.ext_host_addr = ip0->dst_address;
2608   key0.ext_host_port = 0;
2609
2610   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
2611   if (PREDICT_FALSE(!ses0))
2612     {
2613       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2614           IP_PROTOCOL_ICMP, rx_fib_index0)))
2615         {
2616           dont_translate = 1;
2617           goto out;
2618         }
2619       if (icmp0->type != ICMP4_echo_request)
2620         {
2621           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2622           next0 = SNAT_IN2OUT_NEXT_DROP;
2623           goto out;
2624         }
2625       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2626         {
2627           key0.out_port = clib_host_to_net_u16 (lo_port0 +
2628             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
2629
2630           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
2631             continue;
2632
2633           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
2634           break;
2635         }
2636       if (PREDICT_FALSE(!ses0))
2637         {
2638           next0 = SNAT_IN2OUT_NEXT_DROP;
2639           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2640           goto out;
2641         }
2642     }
2643
2644   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
2645                     !icmp_is_error_message (icmp0)))
2646     {
2647       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2648       next0 = SNAT_IN2OUT_NEXT_DROP;
2649       goto out;
2650     }
2651
2652   u32 now = (u32) vlib_time_now (sm->vlib_main);
2653
2654   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
2655   ses0->expire = now + sm->icmp_timeout;
2656
2657 out:
2658   *p_proto = protocol;
2659   if (ses0)
2660     {
2661       p_value->addr = new_addr0;
2662       p_value->fib_index = sm->outside_fib_index;
2663       p_value->port = ses0->out.out_port;
2664     }
2665   *p_dont_translate = dont_translate;
2666   if (d)
2667     *(snat_det_session_t**)d = ses0;
2668   if (e)
2669     *(snat_det_map_t**)e = dm0;
2670   return next0;
2671 }
2672
2673 /**********************/
2674 /*** worker handoff ***/
2675 /**********************/
2676 static uword
2677 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
2678                                vlib_node_runtime_t * node,
2679                                vlib_frame_t * frame)
2680 {
2681   snat_main_t *sm = &snat_main;
2682   vlib_thread_main_t *tm = vlib_get_thread_main ();
2683   u32 n_left_from, *from, *to_next = 0;
2684   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2685   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2686     = 0;
2687   vlib_frame_queue_elt_t *hf = 0;
2688   vlib_frame_t *f = 0;
2689   int i;
2690   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2691   u32 next_worker_index = 0;
2692   u32 current_worker_index = ~0;
2693   u32 thread_index = vlib_get_thread_index ();
2694
2695   ASSERT (vec_len (sm->workers));
2696
2697   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2698     {
2699       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2700
2701       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2702                                sm->first_worker_index + sm->num_workers - 1,
2703                                (vlib_frame_queue_t *) (~0));
2704     }
2705
2706   from = vlib_frame_vector_args (frame);
2707   n_left_from = frame->n_vectors;
2708
2709   while (n_left_from > 0)
2710     {
2711       u32 bi0;
2712       vlib_buffer_t *b0;
2713       u32 sw_if_index0;
2714       u32 rx_fib_index0;
2715       ip4_header_t * ip0;
2716       u8 do_handoff;
2717
2718       bi0 = from[0];
2719       from += 1;
2720       n_left_from -= 1;
2721
2722       b0 = vlib_get_buffer (vm, bi0);
2723
2724       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2725       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2726
2727       ip0 = vlib_buffer_get_current (b0);
2728
2729       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
2730
2731       if (PREDICT_FALSE (next_worker_index != thread_index))
2732         {
2733           do_handoff = 1;
2734
2735           if (next_worker_index != current_worker_index)
2736             {
2737               if (hf)
2738                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2739
2740               hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index,
2741                                                       next_worker_index,
2742                                                       handoff_queue_elt_by_worker_index);
2743
2744               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2745               to_next_worker = &hf->buffer_index[hf->n_vectors];
2746               current_worker_index = next_worker_index;
2747             }
2748
2749           /* enqueue to correct worker thread */
2750           to_next_worker[0] = bi0;
2751           to_next_worker++;
2752           n_left_to_next_worker--;
2753
2754           if (n_left_to_next_worker == 0)
2755             {
2756               hf->n_vectors = VLIB_FRAME_SIZE;
2757               vlib_put_frame_queue_elt (hf);
2758               current_worker_index = ~0;
2759               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2760               hf = 0;
2761             }
2762         }
2763       else
2764         {
2765           do_handoff = 0;
2766           /* if this is 1st frame */
2767           if (!f)
2768             {
2769               f = vlib_get_frame_to_node (vm, sm->in2out_node_index);
2770               to_next = vlib_frame_vector_args (f);
2771             }
2772
2773           to_next[0] = bi0;
2774           to_next += 1;
2775           f->n_vectors++;
2776         }
2777
2778       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2779                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2780         {
2781           snat_in2out_worker_handoff_trace_t *t =
2782             vlib_add_trace (vm, node, b0, sizeof (*t));
2783           t->next_worker_index = next_worker_index;
2784           t->do_handoff = do_handoff;
2785         }
2786     }
2787
2788   if (f)
2789     vlib_put_frame_to_node (vm, sm->in2out_node_index, f);
2790
2791   if (hf)
2792     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2793
2794   /* Ship frames to the worker nodes */
2795   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2796     {
2797       if (handoff_queue_elt_by_worker_index[i])
2798         {
2799           hf = handoff_queue_elt_by_worker_index[i];
2800           /*
2801            * It works better to let the handoff node
2802            * rate-adapt, always ship the handoff queue element.
2803            */
2804           if (1 || hf->n_vectors == hf->last_n_vectors)
2805             {
2806               vlib_put_frame_queue_elt (hf);
2807               handoff_queue_elt_by_worker_index[i] = 0;
2808             }
2809           else
2810             hf->last_n_vectors = hf->n_vectors;
2811         }
2812       congested_handoff_queue_by_worker_index[i] =
2813         (vlib_frame_queue_t *) (~0);
2814     }
2815   hf = 0;
2816   current_worker_index = ~0;
2817   return frame->n_vectors;
2818 }
2819
2820 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
2821   .function = snat_in2out_worker_handoff_fn,
2822   .name = "snat-in2out-worker-handoff",
2823   .vector_size = sizeof (u32),
2824   .format_trace = format_snat_in2out_worker_handoff_trace,
2825   .type = VLIB_NODE_TYPE_INTERNAL,
2826
2827   .n_next_nodes = 1,
2828
2829   .next_nodes = {
2830     [0] = "error-drop",
2831   },
2832 };
2833
2834 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn);
2835
2836 static uword
2837 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
2838                                 vlib_node_runtime_t * node,
2839                                 vlib_frame_t * frame)
2840 {
2841   u32 n_left_from, * from, * to_next;
2842   snat_in2out_next_t next_index;
2843   u32 pkts_processed = 0;
2844   snat_main_t * sm = &snat_main;
2845   u32 stats_node_index;
2846
2847   stats_node_index = snat_in2out_fast_node.index;
2848
2849   from = vlib_frame_vector_args (frame);
2850   n_left_from = frame->n_vectors;
2851   next_index = node->cached_next_index;
2852
2853   while (n_left_from > 0)
2854     {
2855       u32 n_left_to_next;
2856
2857       vlib_get_next_frame (vm, node, next_index,
2858                            to_next, n_left_to_next);
2859
2860       while (n_left_from > 0 && n_left_to_next > 0)
2861         {
2862           u32 bi0;
2863           vlib_buffer_t * b0;
2864           u32 next0;
2865           u32 sw_if_index0;
2866           ip4_header_t * ip0;
2867           ip_csum_t sum0;
2868           u32 new_addr0, old_addr0;
2869           u16 old_port0, new_port0;
2870           udp_header_t * udp0;
2871           tcp_header_t * tcp0;
2872           icmp46_header_t * icmp0;
2873           snat_session_key_t key0, sm0;
2874           u32 proto0;
2875           u32 rx_fib_index0;
2876
2877           /* speculatively enqueue b0 to the current next frame */
2878           bi0 = from[0];
2879           to_next[0] = bi0;
2880           from += 1;
2881           to_next += 1;
2882           n_left_from -= 1;
2883           n_left_to_next -= 1;
2884
2885           b0 = vlib_get_buffer (vm, bi0);
2886           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2887
2888           ip0 = vlib_buffer_get_current (b0);
2889           udp0 = ip4_next_header (ip0);
2890           tcp0 = (tcp_header_t *) udp0;
2891           icmp0 = (icmp46_header_t *) udp0;
2892
2893           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2894           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2895
2896           if (PREDICT_FALSE(ip0->ttl == 1))
2897             {
2898               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2899               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2900                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2901                                            0);
2902               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2903               goto trace0;
2904             }
2905
2906           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2907
2908           if (PREDICT_FALSE (proto0 == ~0))
2909               goto trace0;
2910
2911           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2912             {
2913               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2914                                   rx_fib_index0, node, next0, ~0, 0, 0);
2915               goto trace0;
2916             }
2917
2918           key0.addr = ip0->src_address;
2919           key0.port = udp0->src_port;
2920           key0.fib_index = rx_fib_index0;
2921
2922           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
2923             {
2924               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2925               next0= SNAT_IN2OUT_NEXT_DROP;
2926               goto trace0;
2927             }
2928
2929           new_addr0 = sm0.addr.as_u32;
2930           new_port0 = sm0.port;
2931           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2932           old_addr0 = ip0->src_address.as_u32;
2933           ip0->src_address.as_u32 = new_addr0;
2934
2935           sum0 = ip0->checksum;
2936           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2937                                  ip4_header_t,
2938                                  src_address /* changed member */);
2939           ip0->checksum = ip_csum_fold (sum0);
2940
2941           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2942             {
2943               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2944                 {
2945                   old_port0 = tcp0->src_port;
2946                   tcp0->src_port = new_port0;
2947
2948                   sum0 = tcp0->checksum;
2949                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2950                                          ip4_header_t,
2951                                          dst_address /* changed member */);
2952                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2953                                          ip4_header_t /* cheat */,
2954                                          length /* changed member */);
2955                   tcp0->checksum = ip_csum_fold(sum0);
2956                 }
2957               else
2958                 {
2959                   old_port0 = udp0->src_port;
2960                   udp0->src_port = new_port0;
2961                   udp0->checksum = 0;
2962                 }
2963             }
2964           else
2965             {
2966               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2967                 {
2968                   sum0 = tcp0->checksum;
2969                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2970                                          ip4_header_t,
2971                                          dst_address /* changed member */);
2972                   tcp0->checksum = ip_csum_fold(sum0);
2973                 }
2974             }
2975
2976           /* Hairpinning */
2977           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
2978
2979         trace0:
2980           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2981                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2982             {
2983               snat_in2out_trace_t *t =
2984                  vlib_add_trace (vm, node, b0, sizeof (*t));
2985               t->sw_if_index = sw_if_index0;
2986               t->next_index = next0;
2987             }
2988
2989           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2990
2991           /* verify speculative enqueue, maybe switch current next frame */
2992           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2993                                            to_next, n_left_to_next,
2994                                            bi0, next0);
2995         }
2996
2997       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2998     }
2999
3000   vlib_node_increment_counter (vm, stats_node_index,
3001                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3002                                pkts_processed);
3003   return frame->n_vectors;
3004 }
3005
3006
3007 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3008   .function = snat_in2out_fast_static_map_fn,
3009   .name = "snat-in2out-fast",
3010   .vector_size = sizeof (u32),
3011   .format_trace = format_snat_in2out_fast_trace,
3012   .type = VLIB_NODE_TYPE_INTERNAL,
3013
3014   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3015   .error_strings = snat_in2out_error_strings,
3016
3017   .runtime_data_bytes = sizeof (snat_runtime_t),
3018
3019   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3020
3021   /* edit / add dispositions here */
3022   .next_nodes = {
3023     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3024     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3025     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
3026     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3027   },
3028 };
3029
3030 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);