src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     NAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out NAT44 session key.
 121  * @param out2in Out2in NAT44 session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140   ip4_header_t *ip0;
 141
 142   ip0 = vlib_buffer_get_current (b0);
 143
 144   user_key.addr = in2out.addr;
 145   user_key.fib_index = in2out.fib_index;
 146   kv0.key = user_key.as_u64;
 147
 148   /* Ever heard of the "user" = inside ip4 address before? */
 149   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 150                               &kv0, &value0))
 151     {
 152       /* no, make a new one */
 153       pool_get (sm->per_thread_data[thread_index].users, u);
 154       memset (u, 0, sizeof (*u));
 155       u->addr = in2out.addr;
 156       u->fib_index = in2out.fib_index;
 157
 158       pool_get (sm->per_thread_data[thread_index].list_pool,
 159                 per_user_list_head_elt);
 160
 161       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 162         sm->per_thread_data[thread_index].list_pool;
 163
 164       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 165                        u->sessions_per_user_list_head_index);
 166
 167       kv0.value = u - sm->per_thread_data[thread_index].users;
 168
 169       /* add user */
 170       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 171                                &kv0, 1 /* is_add */);
 172
 173       /* add non-traslated packets worker lookup */
 174       kv0.value = thread_index;
 175       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 176     }
 177   else
 178     {
 179       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 180                              value0.value);
 181     }
 182
 183   pool_get (sm->per_thread_data[thread_index].sessions, s);
 184   memset (s, 0, sizeof (*s));
 185
 186   s->outside_address_index = ~0;
 187   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 188   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 189   u->nstaticsessions++;
 190
 191   /* Create list elts */
 192   pool_get (sm->per_thread_data[thread_index].list_pool,
 193             per_user_translation_list_elt);
 194   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 195                    per_user_translation_list_elt -
 196                    sm->per_thread_data[thread_index].list_pool);
 197
 198   per_user_translation_list_elt->value =
 199     s - sm->per_thread_data[thread_index].sessions;
 200   s->per_user_index =
 201     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 202   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 203
 204   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 205                       s->per_user_list_head_index,
 206                       per_user_translation_list_elt -
 207                       sm->per_thread_data[thread_index].list_pool);
 208
 209   s->in2out = in2out;
 210   s->out2in = out2in;
 211   s->in2out.protocol = out2in.protocol;
 212
 213   /* Add to translation hashes */
 214   kv0.key = s->in2out.as_u64;
 215   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 216   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 217                                1 /* is_add */))
 218       clib_warning ("in2out key add failed");
 219
 220   kv0.key = s->out2in.as_u64;
 221   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 222
 223   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 224                                1 /* is_add */))
 225       clib_warning ("out2in key add failed");
 226
 227   /* log NAT event */
 228   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 229                                       s->out2in.addr.as_u32,
 230                                       s->in2out.protocol,
 231                                       s->in2out.port,
 232                                       s->out2in.port,
 233                                       s->in2out.fib_index);
 234    return s;
 235 }
 236
 237 static_always_inline
 238 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 239                                  snat_session_key_t *p_key0)
 240 {
 241   icmp46_header_t *icmp0;
 242   snat_session_key_t key0;
 243   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 244   ip4_header_t *inner_ip0;
 245   void *l4_header = 0;
 246   icmp46_header_t *inner_icmp0;
 247
 248   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 249   echo0 = (icmp_echo_header_t *)(icmp0+1);
 250
 251   if (!icmp_is_error_message (icmp0))
 252     {
 253       key0.protocol = SNAT_PROTOCOL_ICMP;
 254       key0.addr = ip0->dst_address;
 255       key0.port = echo0->identifier;
 256     }
 257   else
 258     {
 259       inner_ip0 = (ip4_header_t *)(echo0+1);
 260       l4_header = ip4_next_header (inner_ip0);
 261       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 262       key0.addr = inner_ip0->src_address;
 263       switch (key0.protocol)
 264         {
 265         case SNAT_PROTOCOL_ICMP:
 266           inner_icmp0 = (icmp46_header_t*)l4_header;
 267           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 268           key0.port = inner_echo0->identifier;
 269           break;
 270         case SNAT_PROTOCOL_UDP:
 271         case SNAT_PROTOCOL_TCP:
 272           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 273           break;
 274         default:
 275           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 276         }
 277     }
 278   *p_key0 = key0;
 279   return -1; /* success */
 280 }
 281
 282 /**
 283  * Get address and port values to be used for ICMP packet translation
 284  * and create session if needed
 285  *
 286  * @param[in,out] sm             NAT main
 287  * @param[in,out] node           NAT node runtime
 288  * @param[in] thread_index       thread index
 289  * @param[in,out] b0             buffer containing packet to be translated
 290  * @param[out] p_proto           protocol used for matching
 291  * @param[out] p_value           address and port after NAT translation
 292  * @param[out] p_dont_translate  if packet should not be translated
 293  * @param d                      optional parameter
 294  * @param e                      optional parameter
 295  */
 296 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 297                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 298                            snat_session_key_t *p_value,
 299                            u8 *p_dont_translate, void *d, void *e)
 300 {
 301   ip4_header_t *ip0;
 302   icmp46_header_t *icmp0;
 303   u32 sw_if_index0;
 304   u32 rx_fib_index0;
 305   snat_session_key_t key0;
 306   snat_session_key_t sm0;
 307   snat_session_t *s0 = 0;
 308   u8 dont_translate = 0;
 309   clib_bihash_kv_8_8_t kv0, value0;
 310   u8 is_addr_only;
 311   u32 next0 = ~0;
 312   int err;
 313
 314   ip0 = vlib_buffer_get_current (b0);
 315   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 316   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 317   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 318
 319   key0.protocol = 0;
 320
 321   err = icmp_get_key (ip0, &key0);
 322   if (err != -1)
 323     {
 324       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 325       next0 = SNAT_OUT2IN_NEXT_DROP;
 326       goto out;
 327     }
 328   key0.fib_index = rx_fib_index0;
 329
 330   kv0.key = key0.as_u64;
 331
 332   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 333                               &value0))
 334     {
 335       /* Try to match static mapping by external address and port,
 336          destination address and port in packet */
 337       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 338         {
 339           /* Don't NAT packet aimed at the intfc address */
 340           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 341                                               ip0->dst_address.as_u32)))
 342             {
 343               dont_translate = 1;
 344               goto out;
 345             }
 346           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 347           next0 = SNAT_OUT2IN_NEXT_DROP;
 348           goto out;
 349         }
 350
 351       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 352                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 353         {
 354           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 355           next0 = SNAT_OUT2IN_NEXT_DROP;
 356           goto out;
 357         }
 358
 359       /* Create session initiated by host from external network */
 360       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 361                                              node, thread_index);
 362
 363       if (!s0)
 364         {
 365           next0 = SNAT_OUT2IN_NEXT_DROP;
 366           goto out;
 367         }
 368     }
 369   else
 370     {
 371       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 372                         icmp0->type != ICMP4_echo_request &&
 373                         !icmp_is_error_message (icmp0)))
 374         {
 375           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 376           next0 = SNAT_OUT2IN_NEXT_DROP;
 377           goto out;
 378         }
 379
 380       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 381                               value0.value);
 382     }
 383
 384 out:
 385   *p_proto = key0.protocol;
 386   if (s0)
 387     *p_value = s0->in2out;
 388   *p_dont_translate = dont_translate;
 389   if (d)
 390     *(snat_session_t**)d = s0;
 391   return next0;
 392 }
 393
 394 /**
 395  * Get address and port values to be used for ICMP packet translation
 396  *
 397  * @param[in] sm                 NAT main
 398  * @param[in,out] node           NAT node runtime
 399  * @param[in] thread_index       thread index
 400  * @param[in,out] b0             buffer containing packet to be translated
 401  * @param[out] p_proto           protocol used for matching
 402  * @param[out] p_value           address and port after NAT translation
 403  * @param[out] p_dont_translate  if packet should not be translated
 404  * @param d                      optional parameter
 405  * @param e                      optional parameter
 406  */
 407 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 408                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 409                            snat_session_key_t *p_value,
 410                            u8 *p_dont_translate, void *d, void *e)
 411 {
 412   ip4_header_t *ip0;
 413   icmp46_header_t *icmp0;
 414   u32 sw_if_index0;
 415   u32 rx_fib_index0;
 416   snat_session_key_t key0;
 417   snat_session_key_t sm0;
 418   u8 dont_translate = 0;
 419   u8 is_addr_only;
 420   u32 next0 = ~0;
 421   int err;
 422
 423   ip0 = vlib_buffer_get_current (b0);
 424   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 425   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 426   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 427
 428   err = icmp_get_key (ip0, &key0);
 429   if (err != -1)
 430     {
 431       b0->error = node->errors[err];
 432       next0 = SNAT_OUT2IN_NEXT_DROP;
 433       goto out2;
 434     }
 435   key0.fib_index = rx_fib_index0;
 436
 437   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 438     {
 439       /* Don't NAT packet aimed at the intfc address */
 440       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 441         {
 442           dont_translate = 1;
 443           goto out;
 444         }
 445       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 446       next0 = SNAT_OUT2IN_NEXT_DROP;
 447       goto out;
 448     }
 449
 450   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 451                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 452                     !icmp_is_error_message (icmp0)))
 453     {
 454       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 455       next0 = SNAT_OUT2IN_NEXT_DROP;
 456       goto out;
 457     }
 458
 459 out:
 460   *p_value = sm0;
 461 out2:
 462   *p_proto = key0.protocol;
 463   *p_dont_translate = dont_translate;
 464   return next0;
 465 }
 466
 467 static inline u32 icmp_out2in (snat_main_t *sm,
 468                                vlib_buffer_t * b0,
 469                                ip4_header_t * ip0,
 470                                icmp46_header_t * icmp0,
 471                                u32 sw_if_index0,
 472                                u32 rx_fib_index0,
 473                                vlib_node_runtime_t * node,
 474                                u32 next0,
 475                                u32 thread_index,
 476                                void *d,
 477                                void *e)
 478 {
 479   snat_session_key_t sm0;
 480   u8 protocol;
 481   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 482   ip4_header_t *inner_ip0 = 0;
 483   void *l4_header = 0;
 484   icmp46_header_t *inner_icmp0;
 485   u8 dont_translate;
 486   u32 new_addr0, old_addr0;
 487   u16 old_id0, new_id0;
 488   ip_csum_t sum0;
 489   u16 checksum0;
 490   u32 next0_tmp;
 491
 492   echo0 = (icmp_echo_header_t *)(icmp0+1);
 493
 494   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 495                                        &protocol, &sm0, &dont_translate, d, e);
 496   if (next0_tmp != ~0)
 497     next0 = next0_tmp;
 498   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 499     goto out;
 500
 501   sum0 = ip_incremental_checksum (0, icmp0,
 502                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 503   checksum0 = ~ip_csum_fold (sum0);
 504   if (checksum0 != 0 && checksum0 != 0xffff)
 505     {
 506       next0 = SNAT_OUT2IN_NEXT_DROP;
 507       goto out;
 508     }
 509
 510   old_addr0 = ip0->dst_address.as_u32;
 511   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 512   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 513
 514   sum0 = ip0->checksum;
 515   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 516                          dst_address /* changed member */);
 517   ip0->checksum = ip_csum_fold (sum0);
 518
 519   if (!icmp_is_error_message (icmp0))
 520     {
 521       new_id0 = sm0.port;
 522       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 523         {
 524           old_id0 = echo0->identifier;
 525           new_id0 = sm0.port;
 526           echo0->identifier = new_id0;
 527
 528           sum0 = icmp0->checksum;
 529           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 530                                  identifier /* changed member */);
 531           icmp0->checksum = ip_csum_fold (sum0);
 532         }
 533     }
 534   else
 535     {
 536       inner_ip0 = (ip4_header_t *)(echo0+1);
 537       l4_header = ip4_next_header (inner_ip0);
 538
 539       if (!ip4_header_checksum_is_valid (inner_ip0))
 540         {
 541           next0 = SNAT_OUT2IN_NEXT_DROP;
 542           goto out;
 543         }
 544
 545       old_addr0 = inner_ip0->src_address.as_u32;
 546       inner_ip0->src_address = sm0.addr;
 547       new_addr0 = inner_ip0->src_address.as_u32;
 548
 549       sum0 = icmp0->checksum;
 550       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 551                              src_address /* changed member */);
 552       icmp0->checksum = ip_csum_fold (sum0);
 553
 554       switch (protocol)
 555         {
 556         case SNAT_PROTOCOL_ICMP:
 557           inner_icmp0 = (icmp46_header_t*)l4_header;
 558           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 559
 560           old_id0 = inner_echo0->identifier;
 561           new_id0 = sm0.port;
 562           inner_echo0->identifier = new_id0;
 563
 564           sum0 = icmp0->checksum;
 565           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 566                                  identifier);
 567           icmp0->checksum = ip_csum_fold (sum0);
 568           break;
 569         case SNAT_PROTOCOL_UDP:
 570         case SNAT_PROTOCOL_TCP:
 571           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 572           new_id0 = sm0.port;
 573           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 574
 575           sum0 = icmp0->checksum;
 576           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 577                                  src_port);
 578           icmp0->checksum = ip_csum_fold (sum0);
 579           break;
 580         default:
 581           ASSERT(0);
 582         }
 583     }
 584
 585 out:
 586   return next0;
 587 }
 588
 589
 590 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 591                                          vlib_buffer_t * b0,
 592                                          ip4_header_t * ip0,
 593                                          icmp46_header_t * icmp0,
 594                                          u32 sw_if_index0,
 595                                          u32 rx_fib_index0,
 596                                          vlib_node_runtime_t * node,
 597                                          u32 next0, f64 now,
 598                                          u32 thread_index,
 599                                          snat_session_t ** p_s0)
 600 {
 601   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 602                       next0, thread_index, p_s0, 0);
 603   snat_session_t * s0 = *p_s0;
 604   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 605     {
 606       /* Accounting */
 607       s0->last_heard = now;
 608       s0->total_pkts++;
 609       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 610       /* Per-user LRU list maintenance for dynamic translation */
 611       if (!snat_is_session_static (s0))
 612         {
 613           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 614                              s0->per_user_index);
 615           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 616                               s0->per_user_list_head_index,
 617                               s0->per_user_index);
 618         }
 619     }
 620   return next0;
 621 }
 622
 623 static void
 624 snat_out2in_unknown_proto (snat_main_t *sm,
 625                            vlib_buffer_t * b,
 626                            ip4_header_t * ip,
 627                            u32 rx_fib_index,
 628                            u32 thread_index,
 629                            f64 now,
 630                            vlib_main_t * vm)
 631 {
 632   clib_bihash_kv_8_8_t kv, value;
 633   clib_bihash_kv_16_8_t s_kv, s_value;
 634   snat_static_mapping_t *m;
 635   snat_session_key_t m_key;
 636   u32 old_addr, new_addr;
 637   ip_csum_t sum;
 638   nat_ed_ses_key_t key;
 639   snat_session_t * s;
 640   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 641   snat_user_key_t u_key;
 642   snat_user_t *u;
 643   dlist_elt_t *head, *elt;
 644
 645   old_addr = ip->dst_address.as_u32;
 646
 647   key.l_addr = ip->dst_address;
 648   key.r_addr = ip->src_address;
 649   key.fib_index = rx_fib_index;
 650   key.proto = ip->protocol;
 651   key.rsvd = 0;
 652   key.l_port = 0;
 653   s_kv.key[0] = key.as_u64[0];
 654   s_kv.key[1] = key.as_u64[1];
 655
 656   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 657     {
 658       s = pool_elt_at_index (tsm->sessions, s_value.value);
 659       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 660     }
 661   else
 662     {
 663       m_key.addr = ip->dst_address;
 664       m_key.port = 0;
 665       m_key.protocol = 0;
 666       m_key.fib_index = rx_fib_index;
 667       kv.key = m_key.as_u64;
 668       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 669         return;
 670
 671       m = pool_elt_at_index (sm->static_mappings, value.value);
 672
 673       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 674
 675       u_key.addr = ip->src_address;
 676       u_key.fib_index = m->fib_index;
 677       kv.key = u_key.as_u64;
 678
 679       /* Ever heard of the "user" = src ip4 address before? */
 680       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 681         {
 682           /* no, make a new one */
 683           pool_get (tsm->users, u);
 684           memset (u, 0, sizeof (*u));
 685           u->addr = ip->src_address;
 686           u->fib_index = rx_fib_index;
 687
 688           pool_get (tsm->list_pool, head);
 689           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 690
 691           clib_dlist_init (tsm->list_pool,
 692                            u->sessions_per_user_list_head_index);
 693
 694           kv.value = u - tsm->users;
 695
 696           /* add user */
 697           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
 698         }
 699       else
 700         {
 701           u = pool_elt_at_index (tsm->users, value.value);
 702         }
 703
 704       /* Create a new session */
 705       pool_get (tsm->sessions, s);
 706       memset (s, 0, sizeof (*s));
 707
 708       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 709       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 710       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 711       s->outside_address_index = ~0;
 712       s->out2in.addr.as_u32 = old_addr;
 713       s->out2in.fib_index = rx_fib_index;
 714       s->in2out.addr.as_u32 = new_addr;
 715       s->in2out.fib_index = m->fib_index;
 716       s->in2out.port = s->out2in.port = ip->protocol;
 717       u->nstaticsessions++;
 718
 719       /* Create list elts */
 720       pool_get (tsm->list_pool, elt);
 721       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 722       elt->value = s - tsm->sessions;
 723       s->per_user_index = elt - tsm->list_pool;
 724       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 725       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 726                           s->per_user_index);
 727
 728       /* Add to lookup tables */
 729       s_kv.value = s - tsm->sessions;
 730       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 731         clib_warning ("out2in key add failed");
 732
 733       key.l_addr = ip->dst_address;
 734       key.fib_index = m->fib_index;
 735       s_kv.key[0] = key.as_u64[0];
 736       s_kv.key[1] = key.as_u64[1];
 737       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 738         clib_warning ("in2out key add failed");
 739    }
 740
 741   /* Update IP checksum */
 742   sum = ip->checksum;
 743   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 744   ip->checksum = ip_csum_fold (sum);
 745
 746   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 747
 748   /* Accounting */
 749   s->last_heard = now;
 750   s->total_pkts++;
 751   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 752   /* Per-user LRU list maintenance */
 753   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 754   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 755                       s->per_user_index);
 756 }
 757
 758 static snat_session_t *
 759 snat_out2in_lb (snat_main_t *sm,
 760                 vlib_buffer_t * b,
 761                 ip4_header_t * ip,
 762                 u32 rx_fib_index,
 763                 u32 thread_index,
 764                 f64 now,
 765                 vlib_main_t * vm)
 766 {
 767   nat_ed_ses_key_t key;
 768   clib_bihash_kv_16_8_t s_kv, s_value;
 769   udp_header_t *udp = ip4_next_header (ip);
 770   tcp_header_t *tcp = (tcp_header_t *) udp;
 771   snat_session_t *s = 0;
 772   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 773   snat_session_key_t e_key, l_key;
 774   clib_bihash_kv_8_8_t kv, value;
 775   u32 old_addr, new_addr;
 776   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 777   u16 new_port, old_port;
 778   ip_csum_t sum;
 779   snat_user_key_t u_key;
 780   snat_user_t *u;
 781   dlist_elt_t *head, *elt;
 782
 783   old_addr = ip->dst_address.as_u32;
 784
 785   key.l_addr = ip->dst_address;
 786   key.r_addr = ip->src_address;
 787   key.fib_index = rx_fib_index;
 788   key.proto = ip->protocol;
 789   key.rsvd = 0;
 790   key.l_port = udp->dst_port;
 791   s_kv.key[0] = key.as_u64[0];
 792   s_kv.key[1] = key.as_u64[1];
 793
 794   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 795     {
 796       s = pool_elt_at_index (tsm->sessions, s_value.value);
 797     }
 798   else
 799     {
 800       e_key.addr = ip->dst_address;
 801       e_key.port = udp->dst_port;
 802       e_key.protocol = proto;
 803       e_key.fib_index = rx_fib_index;
 804       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
 805         return 0;
 806
 807       u_key.addr = l_key.addr;
 808       u_key.fib_index = l_key.fib_index;
 809       kv.key = u_key.as_u64;
 810
 811       /* Ever heard of the "user" = src ip4 address before? */
 812       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 813         {
 814           /* no, make a new one */
 815           pool_get (tsm->users, u);
 816           memset (u, 0, sizeof (*u));
 817           u->addr = l_key.addr;
 818           u->fib_index = l_key.fib_index;
 819
 820           pool_get (tsm->list_pool, head);
 821           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 822
 823           clib_dlist_init (tsm->list_pool,
 824                            u->sessions_per_user_list_head_index);
 825
 826           kv.value = u - tsm->users;
 827
 828           /* add user */
 829           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 830             clib_warning ("user key add failed");
 831         }
 832       else
 833         {
 834           u = pool_elt_at_index (tsm->users, value.value);
 835         }
 836
 837       /* Create a new session */
 838       pool_get (tsm->sessions, s);
 839       memset (s, 0, sizeof (*s));
 840
 841       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 842       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 843       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 844       s->outside_address_index = ~0;
 845       s->out2in = e_key;
 846       s->in2out = l_key;
 847       u->nstaticsessions++;
 848
 849       /* Create list elts */
 850       pool_get (tsm->list_pool, elt);
 851       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 852       elt->value = s - tsm->sessions;
 853       s->per_user_index = elt - tsm->list_pool;
 854       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 855       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 856                           s->per_user_index);
 857
 858       /* Add to lookup tables */
 859       s_kv.value = s - tsm->sessions;
 860       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 861         clib_warning ("out2in-ed key add failed");
 862
 863       key.l_addr = l_key.addr;
 864       key.fib_index = l_key.fib_index;
 865       key.l_port = l_key.port;
 866       s_kv.key[0] = key.as_u64[0];
 867       s_kv.key[1] = key.as_u64[1];
 868       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 869         clib_warning ("in2out-ed key add failed");
 870     }
 871
 872   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 873
 874   /* Update IP checksum */
 875   sum = ip->checksum;
 876   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 877   ip->checksum = ip_csum_fold (sum);
 878
 879   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 880     {
 881       old_port = tcp->dst_port;
 882       tcp->dst_port = s->in2out.port;
 883       new_port = tcp->dst_port;
 884
 885       sum = tcp->checksum;
 886       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 887       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 888       tcp->checksum = ip_csum_fold(sum);
 889     }
 890   else
 891     {
 892       udp->dst_port = s->in2out.port;
 893       udp->checksum = 0;
 894     }
 895
 896   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 897
 898   /* Accounting */
 899   s->last_heard = now;
 900   s->total_pkts++;
 901   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 902   return s;
 903 }
 904
 905 static uword
 906 snat_out2in_node_fn (vlib_main_t * vm,
 907                   vlib_node_runtime_t * node,
 908                   vlib_frame_t * frame)
 909 {
 910   u32 n_left_from, * from, * to_next;
 911   snat_out2in_next_t next_index;
 912   u32 pkts_processed = 0;
 913   snat_main_t * sm = &snat_main;
 914   f64 now = vlib_time_now (vm);
 915   u32 thread_index = vlib_get_thread_index ();
 916
 917   from = vlib_frame_vector_args (frame);
 918   n_left_from = frame->n_vectors;
 919   next_index = node->cached_next_index;
 920
 921   while (n_left_from > 0)
 922     {
 923       u32 n_left_to_next;
 924
 925       vlib_get_next_frame (vm, node, next_index,
 926                            to_next, n_left_to_next);
 927
 928       while (n_left_from >= 4 && n_left_to_next >= 2)
 929         {
 930           u32 bi0, bi1;
 931           vlib_buffer_t * b0, * b1;
 932           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 933           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 934           u32 sw_if_index0, sw_if_index1;
 935           ip4_header_t * ip0, *ip1;
 936           ip_csum_t sum0, sum1;
 937           u32 new_addr0, old_addr0;
 938           u16 new_port0, old_port0;
 939           u32 new_addr1, old_addr1;
 940           u16 new_port1, old_port1;
 941           udp_header_t * udp0, * udp1;
 942           tcp_header_t * tcp0, * tcp1;
 943           icmp46_header_t * icmp0, * icmp1;
 944           snat_session_key_t key0, key1, sm0, sm1;
 945           u32 rx_fib_index0, rx_fib_index1;
 946           u32 proto0, proto1;
 947           snat_session_t * s0 = 0, * s1 = 0;
 948           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 949
 950           /* Prefetch next iteration. */
 951           {
 952             vlib_buffer_t * p2, * p3;
 953
 954             p2 = vlib_get_buffer (vm, from[2]);
 955             p3 = vlib_get_buffer (vm, from[3]);
 956
 957             vlib_prefetch_buffer_header (p2, LOAD);
 958             vlib_prefetch_buffer_header (p3, LOAD);
 959
 960             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 961             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 962           }
 963
 964           /* speculatively enqueue b0 and b1 to the current next frame */
 965           to_next[0] = bi0 = from[0];
 966           to_next[1] = bi1 = from[1];
 967           from += 2;
 968           to_next += 2;
 969           n_left_from -= 2;
 970           n_left_to_next -= 2;
 971
 972           b0 = vlib_get_buffer (vm, bi0);
 973           b1 = vlib_get_buffer (vm, bi1);
 974
 975           vnet_buffer (b0)->snat.flags = 0;
 976           vnet_buffer (b1)->snat.flags = 0;
 977
 978           ip0 = vlib_buffer_get_current (b0);
 979           udp0 = ip4_next_header (ip0);
 980           tcp0 = (tcp_header_t *) udp0;
 981           icmp0 = (icmp46_header_t *) udp0;
 982
 983           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 984           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 985                                    sw_if_index0);
 986
 987           if (PREDICT_FALSE(ip0->ttl == 1))
 988             {
 989               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 990               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 991                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 992                                            0);
 993               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 994               goto trace0;
 995             }
 996
 997           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 998
 999           if (PREDICT_FALSE (proto0 == ~0))
1000             {
1001               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1002                                         thread_index, now, vm);
1003               goto trace0;
1004             }
1005
1006           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1007             {
1008               next0 = icmp_out2in_slow_path
1009                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1010                  next0, now, thread_index, &s0);
1011               goto trace0;
1012             }
1013
1014           key0.addr = ip0->dst_address;
1015           key0.port = udp0->dst_port;
1016           key0.protocol = proto0;
1017           key0.fib_index = rx_fib_index0;
1018
1019           kv0.key = key0.as_u64;
1020
1021           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1022                                       &kv0, &value0))
1023             {
1024               /* Try to match static mapping by external address and port,
1025                  destination address and port in packet */
1026               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1027                 {
1028                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1029                   /*
1030                    * Send DHCP packets to the ipv4 stack, or we won't
1031                    * be able to use dhcp client on the outside interface
1032                    */
1033                   if (proto0 != SNAT_PROTOCOL_UDP
1034                       || (udp0->dst_port
1035                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1036                     next0 = SNAT_OUT2IN_NEXT_DROP;
1037                   goto trace0;
1038                 }
1039
1040               /* Create session initiated by host from external network */
1041               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1042                                                      thread_index);
1043               if (!s0)
1044                 {
1045                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1046                   next0 = SNAT_OUT2IN_NEXT_DROP;
1047                   goto trace0;
1048                 }
1049             }
1050           else
1051             {
1052               if (PREDICT_FALSE (value0.value == ~0ULL))
1053                 {
1054                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index, now,
1055                                  vm);
1056                   goto trace0;
1057                 }
1058               else
1059                 {
1060                   s0 = pool_elt_at_index (
1061                     sm->per_thread_data[thread_index].sessions,
1062                     value0.value);
1063                 }
1064             }
1065
1066           old_addr0 = ip0->dst_address.as_u32;
1067           ip0->dst_address = s0->in2out.addr;
1068           new_addr0 = ip0->dst_address.as_u32;
1069           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1070
1071           sum0 = ip0->checksum;
1072           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1073                                  ip4_header_t,
1074                                  dst_address /* changed member */);
1075           ip0->checksum = ip_csum_fold (sum0);
1076
1077           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1078             {
1079               old_port0 = tcp0->dst_port;
1080               tcp0->dst_port = s0->in2out.port;
1081               new_port0 = tcp0->dst_port;
1082
1083               sum0 = tcp0->checksum;
1084               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1085                                      ip4_header_t,
1086                                      dst_address /* changed member */);
1087
1088               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1089                                      ip4_header_t /* cheat */,
1090                                      length /* changed member */);
1091               tcp0->checksum = ip_csum_fold(sum0);
1092             }
1093           else
1094             {
1095               old_port0 = udp0->dst_port;
1096               udp0->dst_port = s0->in2out.port;
1097               udp0->checksum = 0;
1098             }
1099
1100           /* Accounting */
1101           s0->last_heard = now;
1102           s0->total_pkts++;
1103           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1104           /* Per-user LRU list maintenance for dynamic translation */
1105           if (!snat_is_session_static (s0))
1106             {
1107               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1108                                  s0->per_user_index);
1109               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1110                                   s0->per_user_list_head_index,
1111                                   s0->per_user_index);
1112             }
1113         trace0:
1114
1115           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1116                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1117             {
1118               snat_out2in_trace_t *t =
1119                  vlib_add_trace (vm, node, b0, sizeof (*t));
1120               t->sw_if_index = sw_if_index0;
1121               t->next_index = next0;
1122               t->session_index = ~0;
1123               if (s0)
1124                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1125             }
1126
1127           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1128
1129
1130           ip1 = vlib_buffer_get_current (b1);
1131           udp1 = ip4_next_header (ip1);
1132           tcp1 = (tcp_header_t *) udp1;
1133           icmp1 = (icmp46_header_t *) udp1;
1134
1135           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1136           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1137                                    sw_if_index1);
1138
1139           if (PREDICT_FALSE(ip1->ttl == 1))
1140             {
1141               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1142               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1143                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1144                                            0);
1145               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1146               goto trace1;
1147             }
1148
1149           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1150
1151           if (PREDICT_FALSE (proto1 == ~0))
1152             {
1153               snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1154                                         thread_index, now, vm);
1155               goto trace1;
1156             }
1157
1158           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1159             {
1160               next1 = icmp_out2in_slow_path
1161                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1162                  next1, now, thread_index, &s1);
1163               goto trace1;
1164             }
1165
1166           key1.addr = ip1->dst_address;
1167           key1.port = udp1->dst_port;
1168           key1.protocol = proto1;
1169           key1.fib_index = rx_fib_index1;
1170
1171           kv1.key = key1.as_u64;
1172
1173           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1174                                       &kv1, &value1))
1175             {
1176               /* Try to match static mapping by external address and port,
1177                  destination address and port in packet */
1178               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1179                 {
1180                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1181                   /*
1182                    * Send DHCP packets to the ipv4 stack, or we won't
1183                    * be able to use dhcp client on the outside interface
1184                    */
1185                   if (proto1 != SNAT_PROTOCOL_UDP
1186                       || (udp1->dst_port
1187                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1188                     next1 = SNAT_OUT2IN_NEXT_DROP;
1189                   goto trace1;
1190                 }
1191
1192               /* Create session initiated by host from external network */
1193               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1194                                                      thread_index);
1195               if (!s1)
1196                 {
1197                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1198                   next1 = SNAT_OUT2IN_NEXT_DROP;
1199                   goto trace1;
1200                 }
1201             }
1202           else
1203             {
1204               if (PREDICT_FALSE (value1.value == ~0ULL))
1205                 {
1206                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index, now,
1207                                  vm);
1208                   goto trace1;
1209                 }
1210               else
1211                 {
1212                   s1 = pool_elt_at_index (
1213                     sm->per_thread_data[thread_index].sessions,
1214                     value1.value);
1215                 }
1216             }
1217
1218           old_addr1 = ip1->dst_address.as_u32;
1219           ip1->dst_address = s1->in2out.addr;
1220           new_addr1 = ip1->dst_address.as_u32;
1221           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1222
1223           sum1 = ip1->checksum;
1224           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1225                                  ip4_header_t,
1226                                  dst_address /* changed member */);
1227           ip1->checksum = ip_csum_fold (sum1);
1228
1229           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1230             {
1231               old_port1 = tcp1->dst_port;
1232               tcp1->dst_port = s1->in2out.port;
1233               new_port1 = tcp1->dst_port;
1234
1235               sum1 = tcp1->checksum;
1236               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1237                                      ip4_header_t,
1238                                      dst_address /* changed member */);
1239
1240               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1241                                      ip4_header_t /* cheat */,
1242                                      length /* changed member */);
1243               tcp1->checksum = ip_csum_fold(sum1);
1244             }
1245           else
1246             {
1247               old_port1 = udp1->dst_port;
1248               udp1->dst_port = s1->in2out.port;
1249               udp1->checksum = 0;
1250             }
1251
1252           /* Accounting */
1253           s1->last_heard = now;
1254           s1->total_pkts++;
1255           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1256           /* Per-user LRU list maintenance for dynamic translation */
1257           if (!snat_is_session_static (s1))
1258             {
1259               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1260                                  s1->per_user_index);
1261               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1262                                   s1->per_user_list_head_index,
1263                                   s1->per_user_index);
1264             }
1265         trace1:
1266
1267           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1268                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1269             {
1270               snat_out2in_trace_t *t =
1271                  vlib_add_trace (vm, node, b1, sizeof (*t));
1272               t->sw_if_index = sw_if_index1;
1273               t->next_index = next1;
1274               t->session_index = ~0;
1275               if (s1)
1276                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1277             }
1278
1279           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1280
1281           /* verify speculative enqueues, maybe switch current next frame */
1282           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1283                                            to_next, n_left_to_next,
1284                                            bi0, bi1, next0, next1);
1285         }
1286
1287       while (n_left_from > 0 && n_left_to_next > 0)
1288         {
1289           u32 bi0;
1290           vlib_buffer_t * b0;
1291           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1292           u32 sw_if_index0;
1293           ip4_header_t * ip0;
1294           ip_csum_t sum0;
1295           u32 new_addr0, old_addr0;
1296           u16 new_port0, old_port0;
1297           udp_header_t * udp0;
1298           tcp_header_t * tcp0;
1299           icmp46_header_t * icmp0;
1300           snat_session_key_t key0, sm0;
1301           u32 rx_fib_index0;
1302           u32 proto0;
1303           snat_session_t * s0 = 0;
1304           clib_bihash_kv_8_8_t kv0, value0;
1305
1306           /* speculatively enqueue b0 to the current next frame */
1307           bi0 = from[0];
1308           to_next[0] = bi0;
1309           from += 1;
1310           to_next += 1;
1311           n_left_from -= 1;
1312           n_left_to_next -= 1;
1313
1314           b0 = vlib_get_buffer (vm, bi0);
1315
1316           vnet_buffer (b0)->snat.flags = 0;
1317
1318           ip0 = vlib_buffer_get_current (b0);
1319           udp0 = ip4_next_header (ip0);
1320           tcp0 = (tcp_header_t *) udp0;
1321           icmp0 = (icmp46_header_t *) udp0;
1322
1323           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1324           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1325                                    sw_if_index0);
1326
1327           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1328
1329           if (PREDICT_FALSE (proto0 == ~0))
1330             {
1331               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1332                                         thread_index, now, vm);
1333               goto trace00;
1334             }
1335
1336           if (PREDICT_FALSE(ip0->ttl == 1))
1337             {
1338               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1339               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1340                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1341                                            0);
1342               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1343               goto trace00;
1344             }
1345
1346           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1347             {
1348               next0 = icmp_out2in_slow_path
1349                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1350                  next0, now, thread_index, &s0);
1351               goto trace00;
1352             }
1353
1354           key0.addr = ip0->dst_address;
1355           key0.port = udp0->dst_port;
1356           key0.protocol = proto0;
1357           key0.fib_index = rx_fib_index0;
1358
1359           kv0.key = key0.as_u64;
1360
1361           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1362                                       &kv0, &value0))
1363             {
1364               /* Try to match static mapping by external address and port,
1365                  destination address and port in packet */
1366               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1367                 {
1368                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1369                   /*
1370                    * Send DHCP packets to the ipv4 stack, or we won't
1371                    * be able to use dhcp client on the outside interface
1372                    */
1373                   if (proto0 != SNAT_PROTOCOL_UDP
1374                       || (udp0->dst_port
1375                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1376
1377                     next0 = SNAT_OUT2IN_NEXT_DROP;
1378                   goto trace00;
1379                 }
1380
1381               /* Create session initiated by host from external network */
1382               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1383                                                      thread_index);
1384               if (!s0)
1385                 {
1386                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1387                     next0 = SNAT_OUT2IN_NEXT_DROP;
1388                   goto trace00;
1389                 }
1390             }
1391           else
1392             {
1393               if (PREDICT_FALSE (value0.value == ~0ULL))
1394                 {
1395                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index, now,
1396                                  vm);
1397                   goto trace00;
1398                 }
1399               else
1400                 {
1401                   s0 = pool_elt_at_index (
1402                     sm->per_thread_data[thread_index].sessions,
1403                     value0.value);
1404                 }
1405             }
1406
1407           old_addr0 = ip0->dst_address.as_u32;
1408           ip0->dst_address = s0->in2out.addr;
1409           new_addr0 = ip0->dst_address.as_u32;
1410           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1411
1412           sum0 = ip0->checksum;
1413           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1414                                  ip4_header_t,
1415                                  dst_address /* changed member */);
1416           ip0->checksum = ip_csum_fold (sum0);
1417
1418           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1419             {
1420               old_port0 = tcp0->dst_port;
1421               tcp0->dst_port = s0->in2out.port;
1422               new_port0 = tcp0->dst_port;
1423
1424               sum0 = tcp0->checksum;
1425               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1426                                      ip4_header_t,
1427                                      dst_address /* changed member */);
1428
1429               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1430                                      ip4_header_t /* cheat */,
1431                                      length /* changed member */);
1432               tcp0->checksum = ip_csum_fold(sum0);
1433             }
1434           else
1435             {
1436               old_port0 = udp0->dst_port;
1437               udp0->dst_port = s0->in2out.port;
1438               udp0->checksum = 0;
1439             }
1440
1441           /* Accounting */
1442           s0->last_heard = now;
1443           s0->total_pkts++;
1444           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1445           /* Per-user LRU list maintenance for dynamic translation */
1446           if (!snat_is_session_static (s0))
1447             {
1448               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1449                                  s0->per_user_index);
1450               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1451                                   s0->per_user_list_head_index,
1452                                   s0->per_user_index);
1453             }
1454         trace00:
1455
1456           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1457                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1458             {
1459               snat_out2in_trace_t *t =
1460                  vlib_add_trace (vm, node, b0, sizeof (*t));
1461               t->sw_if_index = sw_if_index0;
1462               t->next_index = next0;
1463               t->session_index = ~0;
1464               if (s0)
1465                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1466             }
1467
1468           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1469
1470           /* verify speculative enqueue, maybe switch current next frame */
1471           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1472                                            to_next, n_left_to_next,
1473                                            bi0, next0);
1474         }
1475
1476       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1477     }
1478
1479   vlib_node_increment_counter (vm, snat_out2in_node.index,
1480                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1481                                pkts_processed);
1482   return frame->n_vectors;
1483 }
1484
1485 VLIB_REGISTER_NODE (snat_out2in_node) = {
1486   .function = snat_out2in_node_fn,
1487   .name = "nat44-out2in",
1488   .vector_size = sizeof (u32),
1489   .format_trace = format_snat_out2in_trace,
1490   .type = VLIB_NODE_TYPE_INTERNAL,
1491
1492   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1493   .error_strings = snat_out2in_error_strings,
1494
1495   .runtime_data_bytes = sizeof (snat_runtime_t),
1496
1497   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1498
1499   /* edit / add dispositions here */
1500   .next_nodes = {
1501     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1502     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1503     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1504   },
1505 };
1506 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1507
1508 /**************************/
1509 /*** deterministic mode ***/
1510 /**************************/
1511 static uword
1512 snat_det_out2in_node_fn (vlib_main_t * vm,
1513                          vlib_node_runtime_t * node,
1514                          vlib_frame_t * frame)
1515 {
1516   u32 n_left_from, * from, * to_next;
1517   snat_out2in_next_t next_index;
1518   u32 pkts_processed = 0;
1519   snat_main_t * sm = &snat_main;
1520   u32 thread_index = vlib_get_thread_index ();
1521
1522   from = vlib_frame_vector_args (frame);
1523   n_left_from = frame->n_vectors;
1524   next_index = node->cached_next_index;
1525
1526   while (n_left_from > 0)
1527     {
1528       u32 n_left_to_next;
1529
1530       vlib_get_next_frame (vm, node, next_index,
1531                            to_next, n_left_to_next);
1532
1533       while (n_left_from >= 4 && n_left_to_next >= 2)
1534         {
1535           u32 bi0, bi1;
1536           vlib_buffer_t * b0, * b1;
1537           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1538           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1539           u32 sw_if_index0, sw_if_index1;
1540           ip4_header_t * ip0, * ip1;
1541           ip_csum_t sum0, sum1;
1542           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1543           u16 new_port0, old_port0, old_port1, new_port1;
1544           udp_header_t * udp0, * udp1;
1545           tcp_header_t * tcp0, * tcp1;
1546           u32 proto0, proto1;
1547           snat_det_out_key_t key0, key1;
1548           snat_det_map_t * dm0, * dm1;
1549           snat_det_session_t * ses0 = 0, * ses1 = 0;
1550           u32 rx_fib_index0, rx_fib_index1;
1551           icmp46_header_t * icmp0, * icmp1;
1552
1553           /* Prefetch next iteration. */
1554           {
1555             vlib_buffer_t * p2, * p3;
1556
1557             p2 = vlib_get_buffer (vm, from[2]);
1558             p3 = vlib_get_buffer (vm, from[3]);
1559
1560             vlib_prefetch_buffer_header (p2, LOAD);
1561             vlib_prefetch_buffer_header (p3, LOAD);
1562
1563             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1564             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1565           }
1566
1567           /* speculatively enqueue b0 and b1 to the current next frame */
1568           to_next[0] = bi0 = from[0];
1569           to_next[1] = bi1 = from[1];
1570           from += 2;
1571           to_next += 2;
1572           n_left_from -= 2;
1573           n_left_to_next -= 2;
1574
1575           b0 = vlib_get_buffer (vm, bi0);
1576           b1 = vlib_get_buffer (vm, bi1);
1577
1578           ip0 = vlib_buffer_get_current (b0);
1579           udp0 = ip4_next_header (ip0);
1580           tcp0 = (tcp_header_t *) udp0;
1581
1582           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1583
1584           if (PREDICT_FALSE(ip0->ttl == 1))
1585             {
1586               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1587               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1588                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1589                                            0);
1590               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1591               goto trace0;
1592             }
1593
1594           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1595
1596           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1597             {
1598               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1599               icmp0 = (icmp46_header_t *) udp0;
1600
1601               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1602                                   rx_fib_index0, node, next0, thread_index,
1603                                   &ses0, &dm0);
1604               goto trace0;
1605             }
1606
1607           key0.ext_host_addr = ip0->src_address;
1608           key0.ext_host_port = tcp0->src;
1609           key0.out_port = tcp0->dst;
1610
1611           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1612           if (PREDICT_FALSE(!dm0))
1613             {
1614               clib_warning("unknown dst address:  %U",
1615                            format_ip4_address, &ip0->dst_address);
1616               next0 = SNAT_OUT2IN_NEXT_DROP;
1617               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1618               goto trace0;
1619             }
1620
1621           snat_det_reverse(dm0, &ip0->dst_address,
1622                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1623
1624           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1625           if (PREDICT_FALSE(!ses0))
1626             {
1627               clib_warning("no match src %U:%d dst %U:%d for user %U",
1628                            format_ip4_address, &ip0->src_address,
1629                            clib_net_to_host_u16 (tcp0->src),
1630                            format_ip4_address, &ip0->dst_address,
1631                            clib_net_to_host_u16 (tcp0->dst),
1632                            format_ip4_address, &new_addr0);
1633               next0 = SNAT_OUT2IN_NEXT_DROP;
1634               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1635               goto trace0;
1636             }
1637           new_port0 = ses0->in_port;
1638
1639           old_addr0 = ip0->dst_address;
1640           ip0->dst_address = new_addr0;
1641           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1642
1643           sum0 = ip0->checksum;
1644           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1645                                  ip4_header_t,
1646                                  dst_address /* changed member */);
1647           ip0->checksum = ip_csum_fold (sum0);
1648
1649           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1650             {
1651               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1652                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1653               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1654                 snat_det_ses_close(dm0, ses0);
1655
1656               old_port0 = tcp0->dst;
1657               tcp0->dst = new_port0;
1658
1659               sum0 = tcp0->checksum;
1660               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1661                                      ip4_header_t,
1662                                      dst_address /* changed member */);
1663
1664               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1665                                      ip4_header_t /* cheat */,
1666                                      length /* changed member */);
1667               tcp0->checksum = ip_csum_fold(sum0);
1668             }
1669           else
1670             {
1671               old_port0 = udp0->dst_port;
1672               udp0->dst_port = new_port0;
1673               udp0->checksum = 0;
1674             }
1675
1676         trace0:
1677
1678           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1679                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1680             {
1681               snat_out2in_trace_t *t =
1682                  vlib_add_trace (vm, node, b0, sizeof (*t));
1683               t->sw_if_index = sw_if_index0;
1684               t->next_index = next0;
1685               t->session_index = ~0;
1686               if (ses0)
1687                 t->session_index = ses0 - dm0->sessions;
1688             }
1689
1690           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1691
1692           b1 = vlib_get_buffer (vm, bi1);
1693
1694           ip1 = vlib_buffer_get_current (b1);
1695           udp1 = ip4_next_header (ip1);
1696           tcp1 = (tcp_header_t *) udp1;
1697
1698           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1699
1700           if (PREDICT_FALSE(ip1->ttl == 1))
1701             {
1702               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1703               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1704                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1705                                            0);
1706               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1707               goto trace1;
1708             }
1709
1710           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1711
1712           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1713             {
1714               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1715               icmp1 = (icmp46_header_t *) udp1;
1716
1717               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1718                                   rx_fib_index1, node, next1, thread_index,
1719                                   &ses1, &dm1);
1720               goto trace1;
1721             }
1722
1723           key1.ext_host_addr = ip1->src_address;
1724           key1.ext_host_port = tcp1->src;
1725           key1.out_port = tcp1->dst;
1726
1727           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1728           if (PREDICT_FALSE(!dm1))
1729             {
1730               clib_warning("unknown dst address:  %U",
1731                            format_ip4_address, &ip1->dst_address);
1732               next1 = SNAT_OUT2IN_NEXT_DROP;
1733               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1734               goto trace1;
1735             }
1736
1737           snat_det_reverse(dm1, &ip1->dst_address,
1738                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1739
1740           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1741           if (PREDICT_FALSE(!ses1))
1742             {
1743               clib_warning("no match src %U:%d dst %U:%d for user %U",
1744                            format_ip4_address, &ip1->src_address,
1745                            clib_net_to_host_u16 (tcp1->src),
1746                            format_ip4_address, &ip1->dst_address,
1747                            clib_net_to_host_u16 (tcp1->dst),
1748                            format_ip4_address, &new_addr1);
1749               next1 = SNAT_OUT2IN_NEXT_DROP;
1750               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1751               goto trace1;
1752             }
1753           new_port1 = ses1->in_port;
1754
1755           old_addr1 = ip1->dst_address;
1756           ip1->dst_address = new_addr1;
1757           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1758
1759           sum1 = ip1->checksum;
1760           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1761                                  ip4_header_t,
1762                                  dst_address /* changed member */);
1763           ip1->checksum = ip_csum_fold (sum1);
1764
1765           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1766             {
1767               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1768                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1769               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1770                 snat_det_ses_close(dm1, ses1);
1771
1772               old_port1 = tcp1->dst;
1773               tcp1->dst = new_port1;
1774
1775               sum1 = tcp1->checksum;
1776               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1777                                      ip4_header_t,
1778                                      dst_address /* changed member */);
1779
1780               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1781                                      ip4_header_t /* cheat */,
1782                                      length /* changed member */);
1783               tcp1->checksum = ip_csum_fold(sum1);
1784             }
1785           else
1786             {
1787               old_port1 = udp1->dst_port;
1788               udp1->dst_port = new_port1;
1789               udp1->checksum = 0;
1790             }
1791
1792         trace1:
1793
1794           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1795                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1796             {
1797               snat_out2in_trace_t *t =
1798                  vlib_add_trace (vm, node, b1, sizeof (*t));
1799               t->sw_if_index = sw_if_index1;
1800               t->next_index = next1;
1801               t->session_index = ~0;
1802               if (ses1)
1803                 t->session_index = ses1 - dm1->sessions;
1804             }
1805
1806           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1807
1808           /* verify speculative enqueues, maybe switch current next frame */
1809           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1810                                            to_next, n_left_to_next,
1811                                            bi0, bi1, next0, next1);
1812          }
1813
1814       while (n_left_from > 0 && n_left_to_next > 0)
1815         {
1816           u32 bi0;
1817           vlib_buffer_t * b0;
1818           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1819           u32 sw_if_index0;
1820           ip4_header_t * ip0;
1821           ip_csum_t sum0;
1822           ip4_address_t new_addr0, old_addr0;
1823           u16 new_port0, old_port0;
1824           udp_header_t * udp0;
1825           tcp_header_t * tcp0;
1826           u32 proto0;
1827           snat_det_out_key_t key0;
1828           snat_det_map_t * dm0;
1829           snat_det_session_t * ses0 = 0;
1830           u32 rx_fib_index0;
1831           icmp46_header_t * icmp0;
1832
1833           /* speculatively enqueue b0 to the current next frame */
1834           bi0 = from[0];
1835           to_next[0] = bi0;
1836           from += 1;
1837           to_next += 1;
1838           n_left_from -= 1;
1839           n_left_to_next -= 1;
1840
1841           b0 = vlib_get_buffer (vm, bi0);
1842
1843           ip0 = vlib_buffer_get_current (b0);
1844           udp0 = ip4_next_header (ip0);
1845           tcp0 = (tcp_header_t *) udp0;
1846
1847           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1848
1849           if (PREDICT_FALSE(ip0->ttl == 1))
1850             {
1851               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1852               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1853                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1854                                            0);
1855               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1856               goto trace00;
1857             }
1858
1859           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1860
1861           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1862             {
1863               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1864               icmp0 = (icmp46_header_t *) udp0;
1865
1866               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1867                                   rx_fib_index0, node, next0, thread_index,
1868                                   &ses0, &dm0);
1869               goto trace00;
1870             }
1871
1872           key0.ext_host_addr = ip0->src_address;
1873           key0.ext_host_port = tcp0->src;
1874           key0.out_port = tcp0->dst;
1875
1876           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1877           if (PREDICT_FALSE(!dm0))
1878             {
1879               clib_warning("unknown dst address:  %U",
1880                            format_ip4_address, &ip0->dst_address);
1881               next0 = SNAT_OUT2IN_NEXT_DROP;
1882               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1883               goto trace00;
1884             }
1885
1886           snat_det_reverse(dm0, &ip0->dst_address,
1887                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1888
1889           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1890           if (PREDICT_FALSE(!ses0))
1891             {
1892               clib_warning("no match src %U:%d dst %U:%d for user %U",
1893                            format_ip4_address, &ip0->src_address,
1894                            clib_net_to_host_u16 (tcp0->src),
1895                            format_ip4_address, &ip0->dst_address,
1896                            clib_net_to_host_u16 (tcp0->dst),
1897                            format_ip4_address, &new_addr0);
1898               next0 = SNAT_OUT2IN_NEXT_DROP;
1899               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1900               goto trace00;
1901             }
1902           new_port0 = ses0->in_port;
1903
1904           old_addr0 = ip0->dst_address;
1905           ip0->dst_address = new_addr0;
1906           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1907
1908           sum0 = ip0->checksum;
1909           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1910                                  ip4_header_t,
1911                                  dst_address /* changed member */);
1912           ip0->checksum = ip_csum_fold (sum0);
1913
1914           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1915             {
1916               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1917                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1918               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1919                 snat_det_ses_close(dm0, ses0);
1920
1921               old_port0 = tcp0->dst;
1922               tcp0->dst = new_port0;
1923
1924               sum0 = tcp0->checksum;
1925               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1926                                      ip4_header_t,
1927                                      dst_address /* changed member */);
1928
1929               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1930                                      ip4_header_t /* cheat */,
1931                                      length /* changed member */);
1932               tcp0->checksum = ip_csum_fold(sum0);
1933             }
1934           else
1935             {
1936               old_port0 = udp0->dst_port;
1937               udp0->dst_port = new_port0;
1938               udp0->checksum = 0;
1939             }
1940
1941         trace00:
1942
1943           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1944                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1945             {
1946               snat_out2in_trace_t *t =
1947                  vlib_add_trace (vm, node, b0, sizeof (*t));
1948               t->sw_if_index = sw_if_index0;
1949               t->next_index = next0;
1950               t->session_index = ~0;
1951               if (ses0)
1952                 t->session_index = ses0 - dm0->sessions;
1953             }
1954
1955           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1956
1957           /* verify speculative enqueue, maybe switch current next frame */
1958           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1959                                            to_next, n_left_to_next,
1960                                            bi0, next0);
1961         }
1962
1963       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1964     }
1965
1966   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1967                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1968                                pkts_processed);
1969   return frame->n_vectors;
1970 }
1971
1972 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1973   .function = snat_det_out2in_node_fn,
1974   .name = "nat44-det-out2in",
1975   .vector_size = sizeof (u32),
1976   .format_trace = format_snat_out2in_trace,
1977   .type = VLIB_NODE_TYPE_INTERNAL,
1978
1979   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1980   .error_strings = snat_out2in_error_strings,
1981
1982   .runtime_data_bytes = sizeof (snat_runtime_t),
1983
1984   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1985
1986   /* edit / add dispositions here */
1987   .next_nodes = {
1988     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1989     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1990     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1991   },
1992 };
1993 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1994
1995 /**
1996  * Get address and port values to be used for ICMP packet translation
1997  * and create session if needed
1998  *
1999  * @param[in,out] sm             NAT main
2000  * @param[in,out] node           NAT node runtime
2001  * @param[in] thread_index       thread index
2002  * @param[in,out] b0             buffer containing packet to be translated
2003  * @param[out] p_proto           protocol used for matching
2004  * @param[out] p_value           address and port after NAT translation
2005  * @param[out] p_dont_translate  if packet should not be translated
2006  * @param d                      optional parameter
2007  * @param e                      optional parameter
2008  */
2009 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2010                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2011                           snat_session_key_t *p_value,
2012                           u8 *p_dont_translate, void *d, void *e)
2013 {
2014   ip4_header_t *ip0;
2015   icmp46_header_t *icmp0;
2016   u32 sw_if_index0;
2017   u8 protocol;
2018   snat_det_out_key_t key0;
2019   u8 dont_translate = 0;
2020   u32 next0 = ~0;
2021   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2022   ip4_header_t *inner_ip0;
2023   void *l4_header = 0;
2024   icmp46_header_t *inner_icmp0;
2025   snat_det_map_t * dm0 = 0;
2026   ip4_address_t new_addr0 = {{0}};
2027   snat_det_session_t * ses0 = 0;
2028   ip4_address_t out_addr;
2029
2030   ip0 = vlib_buffer_get_current (b0);
2031   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2032   echo0 = (icmp_echo_header_t *)(icmp0+1);
2033   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2034
2035   if (!icmp_is_error_message (icmp0))
2036     {
2037       protocol = SNAT_PROTOCOL_ICMP;
2038       key0.ext_host_addr = ip0->src_address;
2039       key0.ext_host_port = 0;
2040       key0.out_port = echo0->identifier;
2041       out_addr = ip0->dst_address;
2042     }
2043   else
2044     {
2045       inner_ip0 = (ip4_header_t *)(echo0+1);
2046       l4_header = ip4_next_header (inner_ip0);
2047       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2048       key0.ext_host_addr = inner_ip0->dst_address;
2049       out_addr = inner_ip0->src_address;
2050       switch (protocol)
2051         {
2052         case SNAT_PROTOCOL_ICMP:
2053           inner_icmp0 = (icmp46_header_t*)l4_header;
2054           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2055           key0.ext_host_port = 0;
2056           key0.out_port = inner_echo0->identifier;
2057           break;
2058         case SNAT_PROTOCOL_UDP:
2059         case SNAT_PROTOCOL_TCP:
2060           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2061           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2062           break;
2063         default:
2064           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2065           next0 = SNAT_OUT2IN_NEXT_DROP;
2066           goto out;
2067         }
2068     }
2069
2070   dm0 = snat_det_map_by_out(sm, &out_addr);
2071   if (PREDICT_FALSE(!dm0))
2072     {
2073       /* Don't NAT packet aimed at the intfc address */
2074       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2075                                           ip0->dst_address.as_u32)))
2076         {
2077           dont_translate = 1;
2078           goto out;
2079         }
2080       clib_warning("unknown dst address:  %U",
2081                    format_ip4_address, &ip0->dst_address);
2082       goto out;
2083     }
2084
2085   snat_det_reverse(dm0, &ip0->dst_address,
2086                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2087
2088   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2089   if (PREDICT_FALSE(!ses0))
2090     {
2091       /* Don't NAT packet aimed at the intfc address */
2092       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2093                                           ip0->dst_address.as_u32)))
2094         {
2095           dont_translate = 1;
2096           goto out;
2097         }
2098       clib_warning("no match src %U:%d dst %U:%d for user %U",
2099                    format_ip4_address, &key0.ext_host_addr,
2100                    clib_net_to_host_u16 (key0.ext_host_port),
2101                    format_ip4_address, &out_addr,
2102                    clib_net_to_host_u16 (key0.out_port),
2103                    format_ip4_address, &new_addr0);
2104       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2105       next0 = SNAT_OUT2IN_NEXT_DROP;
2106       goto out;
2107     }
2108
2109   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2110                     !icmp_is_error_message (icmp0)))
2111     {
2112       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2113       next0 = SNAT_OUT2IN_NEXT_DROP;
2114       goto out;
2115     }
2116
2117   goto out;
2118
2119 out:
2120   *p_proto = protocol;
2121   if (ses0)
2122     {
2123       p_value->addr = new_addr0;
2124       p_value->fib_index = sm->inside_fib_index;
2125       p_value->port = ses0->in_port;
2126     }
2127   *p_dont_translate = dont_translate;
2128   if (d)
2129     *(snat_det_session_t**)d = ses0;
2130   if (e)
2131     *(snat_det_map_t**)e = dm0;
2132   return next0;
2133 }
2134
2135 /**********************/
2136 /*** worker handoff ***/
2137 /**********************/
2138 static uword
2139 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2140                                vlib_node_runtime_t * node,
2141                                vlib_frame_t * frame)
2142 {
2143   snat_main_t *sm = &snat_main;
2144   vlib_thread_main_t *tm = vlib_get_thread_main ();
2145   u32 n_left_from, *from, *to_next = 0;
2146   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2147   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2148     = 0;
2149   vlib_frame_queue_elt_t *hf = 0;
2150   vlib_frame_t *f = 0;
2151   int i;
2152   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2153   u32 next_worker_index = 0;
2154   u32 current_worker_index = ~0;
2155   u32 thread_index = vlib_get_thread_index ();
2156
2157   ASSERT (vec_len (sm->workers));
2158
2159   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2160     {
2161       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2162
2163       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2164                                sm->first_worker_index + sm->num_workers - 1,
2165                                (vlib_frame_queue_t *) (~0));
2166     }
2167
2168   from = vlib_frame_vector_args (frame);
2169   n_left_from = frame->n_vectors;
2170
2171   while (n_left_from > 0)
2172     {
2173       u32 bi0;
2174       vlib_buffer_t *b0;
2175       u32 sw_if_index0;
2176       u32 rx_fib_index0;
2177       ip4_header_t * ip0;
2178       u8 do_handoff;
2179
2180       bi0 = from[0];
2181       from += 1;
2182       n_left_from -= 1;
2183
2184       b0 = vlib_get_buffer (vm, bi0);
2185
2186       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2187       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2188
2189       ip0 = vlib_buffer_get_current (b0);
2190
2191       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2192
2193       if (PREDICT_FALSE (next_worker_index != thread_index))
2194         {
2195           do_handoff = 1;
2196
2197           if (next_worker_index != current_worker_index)
2198             {
2199               if (hf)
2200                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2201
2202               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2203                                                       next_worker_index,
2204                                                       handoff_queue_elt_by_worker_index);
2205
2206               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2207               to_next_worker = &hf->buffer_index[hf->n_vectors];
2208               current_worker_index = next_worker_index;
2209             }
2210
2211           /* enqueue to correct worker thread */
2212           to_next_worker[0] = bi0;
2213           to_next_worker++;
2214           n_left_to_next_worker--;
2215
2216           if (n_left_to_next_worker == 0)
2217             {
2218               hf->n_vectors = VLIB_FRAME_SIZE;
2219               vlib_put_frame_queue_elt (hf);
2220               current_worker_index = ~0;
2221               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2222               hf = 0;
2223             }
2224         }
2225       else
2226         {
2227           do_handoff = 0;
2228           /* if this is 1st frame */
2229           if (!f)
2230             {
2231               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2232               to_next = vlib_frame_vector_args (f);
2233             }
2234
2235           to_next[0] = bi0;
2236           to_next += 1;
2237           f->n_vectors++;
2238         }
2239
2240       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2241                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2242         {
2243           snat_out2in_worker_handoff_trace_t *t =
2244             vlib_add_trace (vm, node, b0, sizeof (*t));
2245           t->next_worker_index = next_worker_index;
2246           t->do_handoff = do_handoff;
2247         }
2248     }
2249
2250   if (f)
2251     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2252
2253   if (hf)
2254     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2255
2256   /* Ship frames to the worker nodes */
2257   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2258     {
2259       if (handoff_queue_elt_by_worker_index[i])
2260         {
2261           hf = handoff_queue_elt_by_worker_index[i];
2262           /*
2263            * It works better to let the handoff node
2264            * rate-adapt, always ship the handoff queue element.
2265            */
2266           if (1 || hf->n_vectors == hf->last_n_vectors)
2267             {
2268               vlib_put_frame_queue_elt (hf);
2269               handoff_queue_elt_by_worker_index[i] = 0;
2270             }
2271           else
2272             hf->last_n_vectors = hf->n_vectors;
2273         }
2274       congested_handoff_queue_by_worker_index[i] =
2275         (vlib_frame_queue_t *) (~0);
2276     }
2277   hf = 0;
2278   current_worker_index = ~0;
2279   return frame->n_vectors;
2280 }
2281
2282 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2283   .function = snat_out2in_worker_handoff_fn,
2284   .name = "nat44-out2in-worker-handoff",
2285   .vector_size = sizeof (u32),
2286   .format_trace = format_snat_out2in_worker_handoff_trace,
2287   .type = VLIB_NODE_TYPE_INTERNAL,
2288
2289   .n_next_nodes = 1,
2290
2291   .next_nodes = {
2292     [0] = "error-drop",
2293   },
2294 };
2295
2296 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2297
2298 static uword
2299 snat_out2in_fast_node_fn (vlib_main_t * vm,
2300                           vlib_node_runtime_t * node,
2301                           vlib_frame_t * frame)
2302 {
2303   u32 n_left_from, * from, * to_next;
2304   snat_out2in_next_t next_index;
2305   u32 pkts_processed = 0;
2306   snat_main_t * sm = &snat_main;
2307
2308   from = vlib_frame_vector_args (frame);
2309   n_left_from = frame->n_vectors;
2310   next_index = node->cached_next_index;
2311
2312   while (n_left_from > 0)
2313     {
2314       u32 n_left_to_next;
2315
2316       vlib_get_next_frame (vm, node, next_index,
2317                            to_next, n_left_to_next);
2318
2319       while (n_left_from > 0 && n_left_to_next > 0)
2320         {
2321           u32 bi0;
2322           vlib_buffer_t * b0;
2323           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2324           u32 sw_if_index0;
2325           ip4_header_t * ip0;
2326           ip_csum_t sum0;
2327           u32 new_addr0, old_addr0;
2328           u16 new_port0, old_port0;
2329           udp_header_t * udp0;
2330           tcp_header_t * tcp0;
2331           icmp46_header_t * icmp0;
2332           snat_session_key_t key0, sm0;
2333           u32 proto0;
2334           u32 rx_fib_index0;
2335
2336           /* speculatively enqueue b0 to the current next frame */
2337           bi0 = from[0];
2338           to_next[0] = bi0;
2339           from += 1;
2340           to_next += 1;
2341           n_left_from -= 1;
2342           n_left_to_next -= 1;
2343
2344           b0 = vlib_get_buffer (vm, bi0);
2345
2346           ip0 = vlib_buffer_get_current (b0);
2347           udp0 = ip4_next_header (ip0);
2348           tcp0 = (tcp_header_t *) udp0;
2349           icmp0 = (icmp46_header_t *) udp0;
2350
2351           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2352           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2353
2354           vnet_feature_next (sw_if_index0, &next0, b0);
2355
2356           if (PREDICT_FALSE(ip0->ttl == 1))
2357             {
2358               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2359               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2360                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2361                                            0);
2362               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2363               goto trace00;
2364             }
2365
2366           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2367
2368           if (PREDICT_FALSE (proto0 == ~0))
2369               goto trace00;
2370
2371           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2372             {
2373               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2374                                   rx_fib_index0, node, next0, ~0, 0, 0);
2375               goto trace00;
2376             }
2377
2378           key0.addr = ip0->dst_address;
2379           key0.port = udp0->dst_port;
2380           key0.fib_index = rx_fib_index0;
2381
2382           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2383             {
2384               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2385               goto trace00;
2386             }
2387
2388           new_addr0 = sm0.addr.as_u32;
2389           new_port0 = sm0.port;
2390           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2391           old_addr0 = ip0->dst_address.as_u32;
2392           ip0->dst_address.as_u32 = new_addr0;
2393
2394           sum0 = ip0->checksum;
2395           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2396                                  ip4_header_t,
2397                                  dst_address /* changed member */);
2398           ip0->checksum = ip_csum_fold (sum0);
2399
2400           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2401             {
2402                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2403                 {
2404                   old_port0 = tcp0->dst_port;
2405                   tcp0->dst_port = new_port0;
2406
2407                   sum0 = tcp0->checksum;
2408                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2409                                          ip4_header_t,
2410                                          dst_address /* changed member */);
2411
2412                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2413                                          ip4_header_t /* cheat */,
2414                                          length /* changed member */);
2415                   tcp0->checksum = ip_csum_fold(sum0);
2416                 }
2417               else
2418                 {
2419                   old_port0 = udp0->dst_port;
2420                   udp0->dst_port = new_port0;
2421                   udp0->checksum = 0;
2422                 }
2423             }
2424           else
2425             {
2426               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2427                 {
2428                   sum0 = tcp0->checksum;
2429                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2430                                          ip4_header_t,
2431                                          dst_address /* changed member */);
2432
2433                   tcp0->checksum = ip_csum_fold(sum0);
2434                 }
2435             }
2436
2437         trace00:
2438
2439           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2440                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2441             {
2442               snat_out2in_trace_t *t =
2443                  vlib_add_trace (vm, node, b0, sizeof (*t));
2444               t->sw_if_index = sw_if_index0;
2445               t->next_index = next0;
2446             }
2447
2448           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2449
2450           /* verify speculative enqueue, maybe switch current next frame */
2451           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2452                                            to_next, n_left_to_next,
2453                                            bi0, next0);
2454         }
2455
2456       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2457     }
2458
2459   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2460                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2461                                pkts_processed);
2462   return frame->n_vectors;
2463 }
2464
2465 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2466   .function = snat_out2in_fast_node_fn,
2467   .name = "nat44-out2in-fast",
2468   .vector_size = sizeof (u32),
2469   .format_trace = format_snat_out2in_fast_trace,
2470   .type = VLIB_NODE_TYPE_INTERNAL,
2471
2472   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2473   .error_strings = snat_out2in_error_strings,
2474
2475   .runtime_data_bytes = sizeof (snat_runtime_t),
2476
2477   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2478
2479   /* edit / add dispositions here */
2480   .next_nodes = {
2481     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2482     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2483     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2484   },
2485 };
2486 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);