src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   user_session_increment (sm, u, 1 /* static */);
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 337                       u32 thread_index)
 338 {
 339   nat_ed_ses_key_t key;
 340   clib_bihash_kv_16_8_t kv, value;
 341   udp_header_t *udp;
 342   snat_user_t *u;
 343   snat_session_t *s = 0;
 344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 345
 346   if (ip->protocol == IP_PROTOCOL_ICMP)
 347     {
 348       if (icmp_get_ed_key (ip, &key))
 349         return;
 350     }
 351   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 352     {
 353       udp = ip4_next_header(ip);
 354       key.r_addr = ip->src_address;
 355       key.l_addr = ip->dst_address;
 356       key.proto = ip->protocol;
 357       key.l_port = udp->dst_port;
 358       key.r_port = udp->src_port;
 359     }
 360   else
 361     {
 362       key.r_addr = ip->src_address;
 363       key.l_addr = ip->dst_address;
 364       key.proto = ip->protocol;
 365       key.l_port = key.r_port = 0;
 366     }
 367   key.fib_index = 0;
 368   kv.key[0] = key.as_u64[0];
 369   kv.key[1] = key.as_u64[1];
 370
 371   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 372     {
 373       s = pool_elt_at_index (tsm->sessions, value.value);
 374     }
 375   else
 376     {
 377       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 378         return;
 379
 380       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
 381       if (!u)
 382         {
 383           clib_warning ("create NAT user failed");
 384           return;
 385         }
 386
 387       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 388       if (!s)
 389         {
 390           clib_warning ("create NAT session failed");
 391           return;
 392         }
 393
 394       s->ext_host_addr = key.r_addr;
 395       s->ext_host_port = key.r_port;
 396       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 397       s->outside_address_index = ~0;
 398       s->out2in.addr = key.l_addr;
 399       s->out2in.port = key.l_port;
 400       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 401       s->out2in.fib_index = 0;
 402       s->in2out = s->out2in;
 403       user_session_increment (sm, u, 0);
 404
 405       kv.value = s - tsm->sessions;
 406       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 407         clib_warning ("in2out_ed key add failed");
 408     }
 409
 410   /* Per-user LRU list maintenance */
 411   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 412   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 413                       s->per_user_index);
 414 }
 415
 416 /**
 417  * Get address and port values to be used for ICMP packet translation
 418  * and create session if needed
 419  *
 420  * @param[in,out] sm             NAT main
 421  * @param[in,out] node           NAT node runtime
 422  * @param[in] thread_index       thread index
 423  * @param[in,out] b0             buffer containing packet to be translated
 424  * @param[out] p_proto           protocol used for matching
 425  * @param[out] p_value           address and port after NAT translation
 426  * @param[out] p_dont_translate  if packet should not be translated
 427  * @param d                      optional parameter
 428  * @param e                      optional parameter
 429  */
 430 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 431                            u32 thread_index, vlib_buffer_t *b0,
 432                            ip4_header_t *ip0, u8 *p_proto,
 433                            snat_session_key_t *p_value,
 434                            u8 *p_dont_translate, void *d, void *e)
 435 {
 436   icmp46_header_t *icmp0;
 437   u32 sw_if_index0;
 438   u32 rx_fib_index0;
 439   snat_session_key_t key0;
 440   snat_session_key_t sm0;
 441   snat_session_t *s0 = 0;
 442   u8 dont_translate = 0;
 443   clib_bihash_kv_8_8_t kv0, value0;
 444   u8 is_addr_only;
 445   u32 next0 = ~0;
 446   int err;
 447
 448   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 449   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 450   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 451
 452   key0.protocol = 0;
 453
 454   err = icmp_get_key (ip0, &key0);
 455   if (err != -1)
 456     {
 457       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 458       next0 = SNAT_OUT2IN_NEXT_DROP;
 459       goto out;
 460     }
 461   key0.fib_index = rx_fib_index0;
 462
 463   kv0.key = key0.as_u64;
 464
 465   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 466                               &value0))
 467     {
 468       /* Try to match static mapping by external address and port,
 469          destination address and port in packet */
 470       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 471         {
 472           if (!sm->forwarding_enabled)
 473             {
 474               /* Don't NAT packet aimed at the intfc address */
 475               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 476                                                   ip0->dst_address.as_u32)))
 477                 {
 478                   dont_translate = 1;
 479                   goto out;
 480                 }
 481               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 482               next0 = SNAT_OUT2IN_NEXT_DROP;
 483               goto out;
 484             }
 485           else
 486             {
 487               dont_translate = 1;
 488               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 489                 {
 490                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 491                   goto out;
 492                 }
 493               create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
 494               goto out;
 495             }
 496         }
 497
 498       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 499                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 500         {
 501           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 502           next0 = SNAT_OUT2IN_NEXT_DROP;
 503           goto out;
 504         }
 505
 506       /* Create session initiated by host from external network */
 507       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 508                                              node, thread_index);
 509
 510       if (!s0)
 511         {
 512           next0 = SNAT_OUT2IN_NEXT_DROP;
 513           goto out;
 514         }
 515     }
 516   else
 517     {
 518       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 519                         icmp0->type != ICMP4_echo_request &&
 520                         !icmp_is_error_message (icmp0)))
 521         {
 522           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 523           next0 = SNAT_OUT2IN_NEXT_DROP;
 524           goto out;
 525         }
 526
 527       if (PREDICT_FALSE (value0.value == ~0ULL))
 528         {
 529           nat_ed_ses_key_t key;
 530           clib_bihash_kv_16_8_t s_kv, s_value;
 531
 532           key.as_u64[0] = 0;
 533           key.as_u64[1] = 0;
 534           if (icmp_get_ed_key (ip0, &key))
 535             {
 536               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 537               next0 = SNAT_OUT2IN_NEXT_DROP;
 538               goto out;
 539             }
 540           key.fib_index = rx_fib_index0;
 541           s_kv.key[0] = key.as_u64[0];
 542           s_kv.key[1] = key.as_u64[1];
 543           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 544             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 545                                     s_value.value);
 546           else
 547            {
 548               next0 = SNAT_OUT2IN_NEXT_DROP;
 549               goto out;
 550            }
 551         }
 552       else
 553         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 554                                 value0.value);
 555     }
 556
 557 out:
 558   *p_proto = key0.protocol;
 559   if (s0)
 560     *p_value = s0->in2out;
 561   *p_dont_translate = dont_translate;
 562   if (d)
 563     *(snat_session_t**)d = s0;
 564   return next0;
 565 }
 566
 567 /**
 568  * Get address and port values to be used for ICMP packet translation
 569  *
 570  * @param[in] sm                 NAT main
 571  * @param[in,out] node           NAT node runtime
 572  * @param[in] thread_index       thread index
 573  * @param[in,out] b0             buffer containing packet to be translated
 574  * @param[out] p_proto           protocol used for matching
 575  * @param[out] p_value           address and port after NAT translation
 576  * @param[out] p_dont_translate  if packet should not be translated
 577  * @param d                      optional parameter
 578  * @param e                      optional parameter
 579  */
 580 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 581                            u32 thread_index, vlib_buffer_t *b0,
 582                            ip4_header_t *ip0, u8 *p_proto,
 583                            snat_session_key_t *p_value,
 584                            u8 *p_dont_translate, void *d, void *e)
 585 {
 586   icmp46_header_t *icmp0;
 587   u32 sw_if_index0;
 588   u32 rx_fib_index0;
 589   snat_session_key_t key0;
 590   snat_session_key_t sm0;
 591   u8 dont_translate = 0;
 592   u8 is_addr_only;
 593   u32 next0 = ~0;
 594   int err;
 595
 596   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 597   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 598   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 599
 600   err = icmp_get_key (ip0, &key0);
 601   if (err != -1)
 602     {
 603       b0->error = node->errors[err];
 604       next0 = SNAT_OUT2IN_NEXT_DROP;
 605       goto out2;
 606     }
 607   key0.fib_index = rx_fib_index0;
 608
 609   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 610     {
 611       /* Don't NAT packet aimed at the intfc address */
 612       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 613         {
 614           dont_translate = 1;
 615           goto out;
 616         }
 617       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 618       next0 = SNAT_OUT2IN_NEXT_DROP;
 619       goto out;
 620     }
 621
 622   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 623                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 624                     !icmp_is_error_message (icmp0)))
 625     {
 626       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 627       next0 = SNAT_OUT2IN_NEXT_DROP;
 628       goto out;
 629     }
 630
 631 out:
 632   *p_value = sm0;
 633 out2:
 634   *p_proto = key0.protocol;
 635   *p_dont_translate = dont_translate;
 636   return next0;
 637 }
 638
 639 static inline u32 icmp_out2in (snat_main_t *sm,
 640                                vlib_buffer_t * b0,
 641                                ip4_header_t * ip0,
 642                                icmp46_header_t * icmp0,
 643                                u32 sw_if_index0,
 644                                u32 rx_fib_index0,
 645                                vlib_node_runtime_t * node,
 646                                u32 next0,
 647                                u32 thread_index,
 648                                void *d,
 649                                void *e)
 650 {
 651   snat_session_key_t sm0;
 652   u8 protocol;
 653   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 654   ip4_header_t *inner_ip0 = 0;
 655   void *l4_header = 0;
 656   icmp46_header_t *inner_icmp0;
 657   u8 dont_translate;
 658   u32 new_addr0, old_addr0;
 659   u16 old_id0, new_id0;
 660   ip_csum_t sum0;
 661   u16 checksum0;
 662   u32 next0_tmp;
 663
 664   echo0 = (icmp_echo_header_t *)(icmp0+1);
 665
 666   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 667                                        &protocol, &sm0, &dont_translate, d, e);
 668   if (next0_tmp != ~0)
 669     next0 = next0_tmp;
 670   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 671     goto out;
 672
 673   sum0 = ip_incremental_checksum (0, icmp0,
 674                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 675   checksum0 = ~ip_csum_fold (sum0);
 676   if (checksum0 != 0 && checksum0 != 0xffff)
 677     {
 678       next0 = SNAT_OUT2IN_NEXT_DROP;
 679       goto out;
 680     }
 681
 682   old_addr0 = ip0->dst_address.as_u32;
 683   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 684   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 685
 686   sum0 = ip0->checksum;
 687   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 688                          dst_address /* changed member */);
 689   ip0->checksum = ip_csum_fold (sum0);
 690
 691   if (icmp0->checksum == 0)
 692     icmp0->checksum = 0xffff;
 693
 694   if (!icmp_is_error_message (icmp0))
 695     {
 696       new_id0 = sm0.port;
 697       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 698         {
 699           old_id0 = echo0->identifier;
 700           new_id0 = sm0.port;
 701           echo0->identifier = new_id0;
 702
 703           sum0 = icmp0->checksum;
 704           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 705                                  identifier /* changed member */);
 706           icmp0->checksum = ip_csum_fold (sum0);
 707         }
 708     }
 709   else
 710     {
 711       inner_ip0 = (ip4_header_t *)(echo0+1);
 712       l4_header = ip4_next_header (inner_ip0);
 713
 714       if (!ip4_header_checksum_is_valid (inner_ip0))
 715         {
 716           next0 = SNAT_OUT2IN_NEXT_DROP;
 717           goto out;
 718         }
 719
 720       old_addr0 = inner_ip0->src_address.as_u32;
 721       inner_ip0->src_address = sm0.addr;
 722       new_addr0 = inner_ip0->src_address.as_u32;
 723
 724       sum0 = icmp0->checksum;
 725       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 726                              src_address /* changed member */);
 727       icmp0->checksum = ip_csum_fold (sum0);
 728
 729       switch (protocol)
 730         {
 731         case SNAT_PROTOCOL_ICMP:
 732           inner_icmp0 = (icmp46_header_t*)l4_header;
 733           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 734
 735           old_id0 = inner_echo0->identifier;
 736           new_id0 = sm0.port;
 737           inner_echo0->identifier = new_id0;
 738
 739           sum0 = icmp0->checksum;
 740           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 741                                  identifier);
 742           icmp0->checksum = ip_csum_fold (sum0);
 743           break;
 744         case SNAT_PROTOCOL_UDP:
 745         case SNAT_PROTOCOL_TCP:
 746           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 747           new_id0 = sm0.port;
 748           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 749
 750           sum0 = icmp0->checksum;
 751           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 752                                  src_port);
 753           icmp0->checksum = ip_csum_fold (sum0);
 754           break;
 755         default:
 756           ASSERT(0);
 757         }
 758     }
 759
 760 out:
 761   return next0;
 762 }
 763
 764
 765 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 766                                          vlib_buffer_t * b0,
 767                                          ip4_header_t * ip0,
 768                                          icmp46_header_t * icmp0,
 769                                          u32 sw_if_index0,
 770                                          u32 rx_fib_index0,
 771                                          vlib_node_runtime_t * node,
 772                                          u32 next0, f64 now,
 773                                          u32 thread_index,
 774                                          snat_session_t ** p_s0)
 775 {
 776   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 777                       next0, thread_index, p_s0, 0);
 778   snat_session_t * s0 = *p_s0;
 779   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 780     {
 781       /* Accounting */
 782       s0->last_heard = now;
 783       s0->total_pkts++;
 784       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 785       /* Per-user LRU list maintenance */
 786       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 787                          s0->per_user_index);
 788       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 789                           s0->per_user_list_head_index,
 790                           s0->per_user_index);
 791     }
 792   return next0;
 793 }
 794
 795 static snat_session_t *
 796 snat_out2in_unknown_proto (snat_main_t *sm,
 797                            vlib_buffer_t * b,
 798                            ip4_header_t * ip,
 799                            u32 rx_fib_index,
 800                            u32 thread_index,
 801                            f64 now,
 802                            vlib_main_t * vm,
 803                            vlib_node_runtime_t * node)
 804 {
 805   clib_bihash_kv_8_8_t kv, value;
 806   clib_bihash_kv_16_8_t s_kv, s_value;
 807   snat_static_mapping_t *m;
 808   snat_session_key_t m_key;
 809   u32 old_addr, new_addr;
 810   ip_csum_t sum;
 811   nat_ed_ses_key_t key;
 812   snat_session_t * s;
 813   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 814   snat_user_t *u;
 815
 816   old_addr = ip->dst_address.as_u32;
 817
 818   key.l_addr = ip->dst_address;
 819   key.r_addr = ip->src_address;
 820   key.fib_index = rx_fib_index;
 821   key.proto = ip->protocol;
 822   key.r_port = 0;
 823   key.l_port = 0;
 824   s_kv.key[0] = key.as_u64[0];
 825   s_kv.key[1] = key.as_u64[1];
 826
 827   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 828     {
 829       s = pool_elt_at_index (tsm->sessions, s_value.value);
 830       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 831     }
 832   else
 833     {
 834       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 835         {
 836           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 837           return 0;
 838         }
 839
 840       m_key.addr = ip->dst_address;
 841       m_key.port = 0;
 842       m_key.protocol = 0;
 843       m_key.fib_index = rx_fib_index;
 844       kv.key = m_key.as_u64;
 845       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 846         {
 847           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 848           return 0;
 849         }
 850
 851       m = pool_elt_at_index (sm->static_mappings, value.value);
 852
 853       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 854
 855       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 856                                   thread_index);
 857       if (!u)
 858         {
 859           clib_warning ("create NAT user failed");
 860           return 0;
 861         }
 862
 863       /* Create a new session */
 864       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 865       if (!s)
 866         {
 867           clib_warning ("create NAT session failed");
 868           return 0;
 869         }
 870
 871       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 872       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 873       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 874       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 875       s->outside_address_index = ~0;
 876       s->out2in.addr.as_u32 = old_addr;
 877       s->out2in.fib_index = rx_fib_index;
 878       s->in2out.addr.as_u32 = new_addr;
 879       s->in2out.fib_index = m->fib_index;
 880       s->in2out.port = s->out2in.port = ip->protocol;
 881       user_session_increment (sm, u, 1 /* static */);
 882
 883       /* Add to lookup tables */
 884       s_kv.value = s - tsm->sessions;
 885       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 886         clib_warning ("out2in key add failed");
 887
 888       key.l_addr = ip->dst_address;
 889       key.fib_index = m->fib_index;
 890       s_kv.key[0] = key.as_u64[0];
 891       s_kv.key[1] = key.as_u64[1];
 892       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 893         clib_warning ("in2out key add failed");
 894    }
 895
 896   /* Update IP checksum */
 897   sum = ip->checksum;
 898   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 899   ip->checksum = ip_csum_fold (sum);
 900
 901   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 902
 903   /* Accounting */
 904   s->last_heard = now;
 905   s->total_pkts++;
 906   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 907   /* Per-user LRU list maintenance */
 908   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 909   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 910                       s->per_user_index);
 911
 912   return s;
 913 }
 914
 915 static snat_session_t *
 916 snat_out2in_lb (snat_main_t *sm,
 917                 vlib_buffer_t * b,
 918                 ip4_header_t * ip,
 919                 u32 rx_fib_index,
 920                 u32 thread_index,
 921                 f64 now,
 922                 vlib_main_t * vm,
 923                 vlib_node_runtime_t * node)
 924 {
 925   nat_ed_ses_key_t key;
 926   clib_bihash_kv_16_8_t s_kv, s_value;
 927   udp_header_t *udp = ip4_next_header (ip);
 928   tcp_header_t *tcp = (tcp_header_t *) udp;
 929   snat_session_t *s = 0;
 930   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 931   snat_session_key_t e_key, l_key;
 932   u32 old_addr, new_addr;
 933   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 934   u16 new_port, old_port;
 935   ip_csum_t sum;
 936   snat_user_t *u;
 937   u32 address_index;
 938   snat_session_key_t eh_key;
 939   twice_nat_type_t twice_nat;
 940   u8 lb;
 941
 942   old_addr = ip->dst_address.as_u32;
 943
 944   key.l_addr = ip->dst_address;
 945   key.r_addr = ip->src_address;
 946   key.fib_index = rx_fib_index;
 947   key.proto = ip->protocol;
 948   key.r_port = udp->src_port;
 949   key.l_port = udp->dst_port;
 950   s_kv.key[0] = key.as_u64[0];
 951   s_kv.key[1] = key.as_u64[1];
 952
 953   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 954     {
 955       s = pool_elt_at_index (tsm->sessions, s_value.value);
 956     }
 957   else
 958     {
 959       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 960         {
 961           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 962           return 0;
 963         }
 964
 965       e_key.addr = ip->dst_address;
 966       e_key.port = udp->dst_port;
 967       e_key.protocol = proto;
 968       e_key.fib_index = rx_fib_index;
 969       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 970         return 0;
 971
 972       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 973                                   thread_index);
 974       if (!u)
 975       {
 976         clib_warning ("create NAT user failed");
 977         return 0;
 978       }
 979
 980       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 981       if (!s)
 982         {
 983           clib_warning ("create NAT session failed");
 984           return 0;
 985         }
 986
 987       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 988       s->ext_host_port = udp->src_port;
 989       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 990       if (lb)
 991         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 992       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 993       s->outside_address_index = ~0;
 994       s->out2in = e_key;
 995       s->in2out = l_key;
 996       user_session_increment (sm, u, 1 /* static */);
 997
 998       /* Add to lookup tables */
 999       s_kv.value = s - tsm->sessions;
1000       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1001         clib_warning ("out2in-ed key add failed");
1002
1003       if (twice_nat == TWICE_NAT ||
1004           (twice_nat == TWICE_NAT_SELF &&
1005            ip->src_address.as_u32 == l_key.addr.as_u32))
1006         {
1007           eh_key.protocol = proto;
1008           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1009                                                    thread_index, &eh_key,
1010                                                    &address_index,
1011                                                    sm->port_per_thread,
1012                                                    sm->per_thread_data[thread_index].snat_thread_index))
1013             {
1014               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1015               return 0;
1016             }
1017           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1018           key.r_port = s->ext_host_nat_port = eh_key.port;
1019           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1020         }
1021       key.l_addr = l_key.addr;
1022       key.fib_index = l_key.fib_index;
1023       key.l_port = l_key.port;
1024       s_kv.key[0] = key.as_u64[0];
1025       s_kv.key[1] = key.as_u64[1];
1026       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1027         clib_warning ("in2out-ed key add failed");
1028     }
1029
1030   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1031
1032   /* Update IP checksum */
1033   sum = ip->checksum;
1034   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1035   if (is_twice_nat_session (s))
1036     sum = ip_csum_update (sum, ip->src_address.as_u32,
1037                           s->ext_host_nat_addr.as_u32, ip4_header_t,
1038                           src_address);
1039   ip->checksum = ip_csum_fold (sum);
1040
1041   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1042     {
1043       old_port = tcp->dst_port;
1044       tcp->dst_port = s->in2out.port;
1045       new_port = tcp->dst_port;
1046
1047       sum = tcp->checksum;
1048       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1049       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1050       if (is_twice_nat_session (s))
1051         {
1052           sum = ip_csum_update (sum, ip->src_address.as_u32,
1053                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1054                                 dst_address);
1055           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1056                                 ip4_header_t, length);
1057           tcp->src_port = s->ext_host_nat_port;
1058           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1059         }
1060       tcp->checksum = ip_csum_fold(sum);
1061     }
1062   else
1063     {
1064       udp->dst_port = s->in2out.port;
1065       if (is_twice_nat_session (s))
1066         {
1067           udp->src_port = s->ext_host_nat_port;
1068           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1069         }
1070       udp->checksum = 0;
1071     }
1072
1073   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1074
1075   /* Accounting */
1076   s->last_heard = now;
1077   s->total_pkts++;
1078   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1079   /* Per-user LRU list maintenance */
1080   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1081   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1082                       s->per_user_index);
1083
1084   return s;
1085 }
1086
1087 static uword
1088 snat_out2in_node_fn (vlib_main_t * vm,
1089                   vlib_node_runtime_t * node,
1090                   vlib_frame_t * frame)
1091 {
1092   u32 n_left_from, * from, * to_next;
1093   snat_out2in_next_t next_index;
1094   u32 pkts_processed = 0;
1095   snat_main_t * sm = &snat_main;
1096   f64 now = vlib_time_now (vm);
1097   u32 thread_index = vlib_get_thread_index ();
1098
1099   from = vlib_frame_vector_args (frame);
1100   n_left_from = frame->n_vectors;
1101   next_index = node->cached_next_index;
1102
1103   while (n_left_from > 0)
1104     {
1105       u32 n_left_to_next;
1106
1107       vlib_get_next_frame (vm, node, next_index,
1108                            to_next, n_left_to_next);
1109
1110       while (n_left_from >= 4 && n_left_to_next >= 2)
1111         {
1112           u32 bi0, bi1;
1113           vlib_buffer_t * b0, * b1;
1114           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1115           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1116           u32 sw_if_index0, sw_if_index1;
1117           ip4_header_t * ip0, *ip1;
1118           ip_csum_t sum0, sum1;
1119           u32 new_addr0, old_addr0;
1120           u16 new_port0, old_port0;
1121           u32 new_addr1, old_addr1;
1122           u16 new_port1, old_port1;
1123           udp_header_t * udp0, * udp1;
1124           tcp_header_t * tcp0, * tcp1;
1125           icmp46_header_t * icmp0, * icmp1;
1126           snat_session_key_t key0, key1, sm0, sm1;
1127           u32 rx_fib_index0, rx_fib_index1;
1128           u32 proto0, proto1;
1129           snat_session_t * s0 = 0, * s1 = 0;
1130           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1131
1132           /* Prefetch next iteration. */
1133           {
1134             vlib_buffer_t * p2, * p3;
1135
1136             p2 = vlib_get_buffer (vm, from[2]);
1137             p3 = vlib_get_buffer (vm, from[3]);
1138
1139             vlib_prefetch_buffer_header (p2, LOAD);
1140             vlib_prefetch_buffer_header (p3, LOAD);
1141
1142             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1143             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1144           }
1145
1146           /* speculatively enqueue b0 and b1 to the current next frame */
1147           to_next[0] = bi0 = from[0];
1148           to_next[1] = bi1 = from[1];
1149           from += 2;
1150           to_next += 2;
1151           n_left_from -= 2;
1152           n_left_to_next -= 2;
1153
1154           b0 = vlib_get_buffer (vm, bi0);
1155           b1 = vlib_get_buffer (vm, bi1);
1156
1157           vnet_buffer (b0)->snat.flags = 0;
1158           vnet_buffer (b1)->snat.flags = 0;
1159
1160           ip0 = vlib_buffer_get_current (b0);
1161           udp0 = ip4_next_header (ip0);
1162           tcp0 = (tcp_header_t *) udp0;
1163           icmp0 = (icmp46_header_t *) udp0;
1164
1165           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1166           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1167                                    sw_if_index0);
1168
1169           if (PREDICT_FALSE(ip0->ttl == 1))
1170             {
1171               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1172               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1173                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1174                                            0);
1175               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1176               goto trace0;
1177             }
1178
1179           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1180
1181           if (PREDICT_FALSE (proto0 == ~0))
1182             {
1183               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1184                                              thread_index, now, vm, node);
1185               if (!sm->forwarding_enabled)
1186                 if (!s0)
1187                   next0 = SNAT_OUT2IN_NEXT_DROP;
1188               goto trace0;
1189             }
1190
1191           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1192             {
1193               next0 = icmp_out2in_slow_path
1194                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1195                  next0, now, thread_index, &s0);
1196               goto trace0;
1197             }
1198
1199           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1200             {
1201               next0 = SNAT_OUT2IN_NEXT_REASS;
1202               goto trace0;
1203             }
1204
1205           key0.addr = ip0->dst_address;
1206           key0.port = udp0->dst_port;
1207           key0.protocol = proto0;
1208           key0.fib_index = rx_fib_index0;
1209
1210           kv0.key = key0.as_u64;
1211
1212           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1213                                       &kv0, &value0))
1214             {
1215               /* Try to match static mapping by external address and port,
1216                  destination address and port in packet */
1217               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1218                 {
1219                   /*
1220                    * Send DHCP packets to the ipv4 stack, or we won't
1221                    * be able to use dhcp client on the outside interface
1222                    */
1223                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1224                       && (udp0->dst_port ==
1225                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1226                     {
1227                       vnet_feature_next
1228                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1229                       goto trace0;
1230                     }
1231
1232                   if (!sm->forwarding_enabled)
1233                     {
1234                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1235                       next0 = SNAT_OUT2IN_NEXT_DROP;
1236                       goto trace0;
1237                     }
1238                   else
1239                     {
1240                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1241                         {
1242                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1243                           goto trace0;
1244                         }
1245                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1246                       goto trace0;
1247                     }
1248                 }
1249
1250               /* Create session initiated by host from external network */
1251               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1252                                                      thread_index);
1253               if (!s0)
1254                 {
1255                   next0 = SNAT_OUT2IN_NEXT_DROP;
1256                   goto trace0;
1257                 }
1258             }
1259           else
1260             {
1261               if (PREDICT_FALSE (value0.value == ~0ULL))
1262                 {
1263                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1264                                       now, vm, node);
1265                   if (!s0)
1266                     next0 = SNAT_OUT2IN_NEXT_DROP;
1267                   goto trace0;
1268                 }
1269               else
1270                 {
1271                   s0 = pool_elt_at_index (
1272                     sm->per_thread_data[thread_index].sessions,
1273                     value0.value);
1274                 }
1275             }
1276
1277           old_addr0 = ip0->dst_address.as_u32;
1278           ip0->dst_address = s0->in2out.addr;
1279           new_addr0 = ip0->dst_address.as_u32;
1280           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1281
1282           sum0 = ip0->checksum;
1283           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1284                                  ip4_header_t,
1285                                  dst_address /* changed member */);
1286           ip0->checksum = ip_csum_fold (sum0);
1287
1288           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1289             {
1290               old_port0 = tcp0->dst_port;
1291               tcp0->dst_port = s0->in2out.port;
1292               new_port0 = tcp0->dst_port;
1293
1294               sum0 = tcp0->checksum;
1295               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1296                                      ip4_header_t,
1297                                      dst_address /* changed member */);
1298
1299               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1300                                      ip4_header_t /* cheat */,
1301                                      length /* changed member */);
1302               tcp0->checksum = ip_csum_fold(sum0);
1303             }
1304           else
1305             {
1306               old_port0 = udp0->dst_port;
1307               udp0->dst_port = s0->in2out.port;
1308               udp0->checksum = 0;
1309             }
1310
1311           /* Accounting */
1312           s0->last_heard = now;
1313           s0->total_pkts++;
1314           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1315           /* Per-user LRU list maintenance */
1316           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1317                              s0->per_user_index);
1318           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1319                               s0->per_user_list_head_index,
1320                               s0->per_user_index);
1321         trace0:
1322
1323           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1324                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1325             {
1326               snat_out2in_trace_t *t =
1327                  vlib_add_trace (vm, node, b0, sizeof (*t));
1328               t->sw_if_index = sw_if_index0;
1329               t->next_index = next0;
1330               t->session_index = ~0;
1331               if (s0)
1332                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1333             }
1334
1335           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1336
1337
1338           ip1 = vlib_buffer_get_current (b1);
1339           udp1 = ip4_next_header (ip1);
1340           tcp1 = (tcp_header_t *) udp1;
1341           icmp1 = (icmp46_header_t *) udp1;
1342
1343           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1344           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1345                                    sw_if_index1);
1346
1347           if (PREDICT_FALSE(ip1->ttl == 1))
1348             {
1349               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1350               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1351                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1352                                            0);
1353               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1354               goto trace1;
1355             }
1356
1357           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1358
1359           if (PREDICT_FALSE (proto1 == ~0))
1360             {
1361               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1362                                              thread_index, now, vm, node);
1363               if (!sm->forwarding_enabled)
1364                 if (!s1)
1365                   next1 = SNAT_OUT2IN_NEXT_DROP;
1366               goto trace1;
1367             }
1368
1369           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1370             {
1371               next1 = icmp_out2in_slow_path
1372                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1373                  next1, now, thread_index, &s1);
1374               goto trace1;
1375             }
1376
1377           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1378             {
1379               next1 = SNAT_OUT2IN_NEXT_REASS;
1380               goto trace1;
1381             }
1382
1383           key1.addr = ip1->dst_address;
1384           key1.port = udp1->dst_port;
1385           key1.protocol = proto1;
1386           key1.fib_index = rx_fib_index1;
1387
1388           kv1.key = key1.as_u64;
1389
1390           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1391                                       &kv1, &value1))
1392             {
1393               /* Try to match static mapping by external address and port,
1394                  destination address and port in packet */
1395               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1396                 {
1397                   /*
1398                    * Send DHCP packets to the ipv4 stack, or we won't
1399                    * be able to use dhcp client on the outside interface
1400                    */
1401                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1402                       && (udp1->dst_port ==
1403                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1404                     {
1405                       vnet_feature_next
1406                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1407                       goto trace1;
1408                     }
1409
1410                   if (!sm->forwarding_enabled)
1411                     {
1412                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1413                       next1 = SNAT_OUT2IN_NEXT_DROP;
1414                       goto trace1;
1415                     }
1416                   else
1417                     {
1418                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1419                         {
1420                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1421                           goto trace1;
1422                         }
1423                       create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1424                       goto trace1;
1425                     }
1426                 }
1427
1428               /* Create session initiated by host from external network */
1429               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1430                                                      thread_index);
1431               if (!s1)
1432                 {
1433                   next1 = SNAT_OUT2IN_NEXT_DROP;
1434                   goto trace1;
1435                 }
1436             }
1437           else
1438             {
1439               if (PREDICT_FALSE (value1.value == ~0ULL))
1440                 {
1441                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1442                                       now, vm, node);
1443                   if (!s1)
1444                     next1 = SNAT_OUT2IN_NEXT_DROP;
1445                   goto trace1;
1446                 }
1447               else
1448                 {
1449                   s1 = pool_elt_at_index (
1450                     sm->per_thread_data[thread_index].sessions,
1451                     value1.value);
1452                 }
1453             }
1454
1455           old_addr1 = ip1->dst_address.as_u32;
1456           ip1->dst_address = s1->in2out.addr;
1457           new_addr1 = ip1->dst_address.as_u32;
1458           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1459
1460           sum1 = ip1->checksum;
1461           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1462                                  ip4_header_t,
1463                                  dst_address /* changed member */);
1464           ip1->checksum = ip_csum_fold (sum1);
1465
1466           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1467             {
1468               old_port1 = tcp1->dst_port;
1469               tcp1->dst_port = s1->in2out.port;
1470               new_port1 = tcp1->dst_port;
1471
1472               sum1 = tcp1->checksum;
1473               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1474                                      ip4_header_t,
1475                                      dst_address /* changed member */);
1476
1477               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1478                                      ip4_header_t /* cheat */,
1479                                      length /* changed member */);
1480               tcp1->checksum = ip_csum_fold(sum1);
1481             }
1482           else
1483             {
1484               old_port1 = udp1->dst_port;
1485               udp1->dst_port = s1->in2out.port;
1486               udp1->checksum = 0;
1487             }
1488
1489           /* Accounting */
1490           s1->last_heard = now;
1491           s1->total_pkts++;
1492           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1493           /* Per-user LRU list maintenance */
1494           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1495                              s1->per_user_index);
1496           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1497                               s1->per_user_list_head_index,
1498                               s1->per_user_index);
1499         trace1:
1500
1501           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1502                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1503             {
1504               snat_out2in_trace_t *t =
1505                  vlib_add_trace (vm, node, b1, sizeof (*t));
1506               t->sw_if_index = sw_if_index1;
1507               t->next_index = next1;
1508               t->session_index = ~0;
1509               if (s1)
1510                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1511             }
1512
1513           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1514
1515           /* verify speculative enqueues, maybe switch current next frame */
1516           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1517                                            to_next, n_left_to_next,
1518                                            bi0, bi1, next0, next1);
1519         }
1520
1521       while (n_left_from > 0 && n_left_to_next > 0)
1522         {
1523           u32 bi0;
1524           vlib_buffer_t * b0;
1525           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1526           u32 sw_if_index0;
1527           ip4_header_t * ip0;
1528           ip_csum_t sum0;
1529           u32 new_addr0, old_addr0;
1530           u16 new_port0, old_port0;
1531           udp_header_t * udp0;
1532           tcp_header_t * tcp0;
1533           icmp46_header_t * icmp0;
1534           snat_session_key_t key0, sm0;
1535           u32 rx_fib_index0;
1536           u32 proto0;
1537           snat_session_t * s0 = 0;
1538           clib_bihash_kv_8_8_t kv0, value0;
1539
1540           /* speculatively enqueue b0 to the current next frame */
1541           bi0 = from[0];
1542           to_next[0] = bi0;
1543           from += 1;
1544           to_next += 1;
1545           n_left_from -= 1;
1546           n_left_to_next -= 1;
1547
1548           b0 = vlib_get_buffer (vm, bi0);
1549
1550           vnet_buffer (b0)->snat.flags = 0;
1551
1552           ip0 = vlib_buffer_get_current (b0);
1553           udp0 = ip4_next_header (ip0);
1554           tcp0 = (tcp_header_t *) udp0;
1555           icmp0 = (icmp46_header_t *) udp0;
1556
1557           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1558           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1559                                    sw_if_index0);
1560
1561           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1562
1563           if (PREDICT_FALSE (proto0 == ~0))
1564             {
1565               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1566                                              thread_index, now, vm, node);
1567               if (!sm->forwarding_enabled)
1568                 if (!s0)
1569                   next0 = SNAT_OUT2IN_NEXT_DROP;
1570               goto trace00;
1571             }
1572
1573           if (PREDICT_FALSE(ip0->ttl == 1))
1574             {
1575               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1576               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1577                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1578                                            0);
1579               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1580               goto trace00;
1581             }
1582
1583           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1584             {
1585               next0 = icmp_out2in_slow_path
1586                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1587                  next0, now, thread_index, &s0);
1588               goto trace00;
1589             }
1590
1591           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1592             {
1593               next0 = SNAT_OUT2IN_NEXT_REASS;
1594               goto trace00;
1595             }
1596
1597           key0.addr = ip0->dst_address;
1598           key0.port = udp0->dst_port;
1599           key0.protocol = proto0;
1600           key0.fib_index = rx_fib_index0;
1601
1602           kv0.key = key0.as_u64;
1603
1604           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1605                                       &kv0, &value0))
1606             {
1607               /* Try to match static mapping by external address and port,
1608                  destination address and port in packet */
1609               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1610                 {
1611                   /*
1612                    * Send DHCP packets to the ipv4 stack, or we won't
1613                    * be able to use dhcp client on the outside interface
1614                    */
1615                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1616                       && (udp0->dst_port ==
1617                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1618                     {
1619                       vnet_feature_next
1620                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1621                       goto trace00;
1622                     }
1623
1624                   if (!sm->forwarding_enabled)
1625                     {
1626                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1627                       next0 = SNAT_OUT2IN_NEXT_DROP;
1628                       goto trace00;
1629                     }
1630                   else
1631                     {
1632                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1633                         {
1634                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1635                           goto trace00;
1636                         }
1637                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1638                       goto trace00;
1639                     }
1640                 }
1641
1642               /* Create session initiated by host from external network */
1643               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1644                                                      thread_index);
1645               if (!s0)
1646                 {
1647                   next0 = SNAT_OUT2IN_NEXT_DROP;
1648                   goto trace00;
1649                 }
1650             }
1651           else
1652             {
1653               if (PREDICT_FALSE (value0.value == ~0ULL))
1654                 {
1655                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1656                                       now, vm, node);
1657                   if (!s0)
1658                     next0 = SNAT_OUT2IN_NEXT_DROP;
1659                   goto trace00;
1660                 }
1661               else
1662                 {
1663                   s0 = pool_elt_at_index (
1664                     sm->per_thread_data[thread_index].sessions,
1665                     value0.value);
1666                 }
1667             }
1668
1669           old_addr0 = ip0->dst_address.as_u32;
1670           ip0->dst_address = s0->in2out.addr;
1671           new_addr0 = ip0->dst_address.as_u32;
1672           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1673
1674           sum0 = ip0->checksum;
1675           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1676                                  ip4_header_t,
1677                                  dst_address /* changed member */);
1678           ip0->checksum = ip_csum_fold (sum0);
1679
1680           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1681             {
1682               old_port0 = tcp0->dst_port;
1683               tcp0->dst_port = s0->in2out.port;
1684               new_port0 = tcp0->dst_port;
1685
1686               sum0 = tcp0->checksum;
1687               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1688                                      ip4_header_t,
1689                                      dst_address /* changed member */);
1690
1691               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1692                                      ip4_header_t /* cheat */,
1693                                      length /* changed member */);
1694               tcp0->checksum = ip_csum_fold(sum0);
1695             }
1696           else
1697             {
1698               old_port0 = udp0->dst_port;
1699               udp0->dst_port = s0->in2out.port;
1700               udp0->checksum = 0;
1701             }
1702
1703           /* Accounting */
1704           s0->last_heard = now;
1705           s0->total_pkts++;
1706           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1707           /* Per-user LRU list maintenance */
1708           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1709                              s0->per_user_index);
1710           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1711                               s0->per_user_list_head_index,
1712                               s0->per_user_index);
1713         trace00:
1714
1715           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1716                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1717             {
1718               snat_out2in_trace_t *t =
1719                  vlib_add_trace (vm, node, b0, sizeof (*t));
1720               t->sw_if_index = sw_if_index0;
1721               t->next_index = next0;
1722               t->session_index = ~0;
1723               if (s0)
1724                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1725             }
1726
1727           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1728
1729           /* verify speculative enqueue, maybe switch current next frame */
1730           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1731                                            to_next, n_left_to_next,
1732                                            bi0, next0);
1733         }
1734
1735       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1736     }
1737
1738   vlib_node_increment_counter (vm, snat_out2in_node.index,
1739                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1740                                pkts_processed);
1741   return frame->n_vectors;
1742 }
1743
1744 VLIB_REGISTER_NODE (snat_out2in_node) = {
1745   .function = snat_out2in_node_fn,
1746   .name = "nat44-out2in",
1747   .vector_size = sizeof (u32),
1748   .format_trace = format_snat_out2in_trace,
1749   .type = VLIB_NODE_TYPE_INTERNAL,
1750
1751   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1752   .error_strings = snat_out2in_error_strings,
1753
1754   .runtime_data_bytes = sizeof (snat_runtime_t),
1755
1756   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1757
1758   /* edit / add dispositions here */
1759   .next_nodes = {
1760     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1761     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1762     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1763     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1764     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1765   },
1766 };
1767 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1768
1769 static uword
1770 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1771                             vlib_node_runtime_t * node,
1772                             vlib_frame_t * frame)
1773 {
1774   u32 n_left_from, *from, *to_next;
1775   snat_out2in_next_t next_index;
1776   u32 pkts_processed = 0;
1777   snat_main_t *sm = &snat_main;
1778   f64 now = vlib_time_now (vm);
1779   u32 thread_index = vlib_get_thread_index ();
1780   snat_main_per_thread_data_t *per_thread_data =
1781     &sm->per_thread_data[thread_index];
1782   u32 *fragments_to_drop = 0;
1783   u32 *fragments_to_loopback = 0;
1784
1785   from = vlib_frame_vector_args (frame);
1786   n_left_from = frame->n_vectors;
1787   next_index = node->cached_next_index;
1788
1789   while (n_left_from > 0)
1790     {
1791       u32 n_left_to_next;
1792
1793       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1794
1795       while (n_left_from > 0 && n_left_to_next > 0)
1796        {
1797           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1798           vlib_buffer_t *b0;
1799           u32 next0;
1800           u8 cached0 = 0;
1801           ip4_header_t *ip0;
1802           nat_reass_ip4_t *reass0;
1803           udp_header_t * udp0;
1804           tcp_header_t * tcp0;
1805           snat_session_key_t key0, sm0;
1806           clib_bihash_kv_8_8_t kv0, value0;
1807           snat_session_t * s0 = 0;
1808           u16 old_port0, new_port0;
1809           ip_csum_t sum0;
1810
1811           /* speculatively enqueue b0 to the current next frame */
1812           bi0 = from[0];
1813           to_next[0] = bi0;
1814           from += 1;
1815           to_next += 1;
1816           n_left_from -= 1;
1817           n_left_to_next -= 1;
1818
1819           b0 = vlib_get_buffer (vm, bi0);
1820           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1821
1822           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1823           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1824                                                                sw_if_index0);
1825
1826           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1827             {
1828               next0 = SNAT_OUT2IN_NEXT_DROP;
1829               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1830               goto trace0;
1831             }
1832
1833           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1834           udp0 = ip4_next_header (ip0);
1835           tcp0 = (tcp_header_t *) udp0;
1836           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1837
1838           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1839                                                  ip0->dst_address,
1840                                                  ip0->fragment_id,
1841                                                  ip0->protocol,
1842                                                  1,
1843                                                  &fragments_to_drop);
1844
1845           if (PREDICT_FALSE (!reass0))
1846             {
1847               next0 = SNAT_OUT2IN_NEXT_DROP;
1848               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1849               goto trace0;
1850             }
1851
1852           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1853             {
1854               key0.addr = ip0->dst_address;
1855               key0.port = udp0->dst_port;
1856               key0.protocol = proto0;
1857               key0.fib_index = rx_fib_index0;
1858               kv0.key = key0.as_u64;
1859
1860               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1861                 {
1862                   /* Try to match static mapping by external address and port,
1863                      destination address and port in packet */
1864                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1865                     {
1866                       /*
1867                        * Send DHCP packets to the ipv4 stack, or we won't
1868                        * be able to use dhcp client on the outside interface
1869                        */
1870                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1871                           && (udp0->dst_port
1872                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1873                         {
1874                           vnet_feature_next
1875                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1876                              &next0, b0);
1877                           goto trace0;
1878                         }
1879
1880                       if (!sm->forwarding_enabled)
1881                         {
1882                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1883                           next0 = SNAT_OUT2IN_NEXT_DROP;
1884                           goto trace0;
1885                         }
1886                       else
1887                         {
1888                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1889                             {
1890                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1891                               goto trace0;
1892                             }
1893                           create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1894                           goto trace0;
1895                         }
1896                     }
1897
1898                   /* Create session initiated by host from external network */
1899                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1900                                                          thread_index);
1901                   if (!s0)
1902                     {
1903                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1904                       next0 = SNAT_OUT2IN_NEXT_DROP;
1905                       goto trace0;
1906                     }
1907                   reass0->sess_index = s0 - per_thread_data->sessions;
1908                   reass0->thread_index = thread_index;
1909                 }
1910               else
1911                 {
1912                   s0 = pool_elt_at_index (per_thread_data->sessions,
1913                                           value0.value);
1914                   reass0->sess_index = value0.value;
1915                 }
1916               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1917             }
1918           else
1919             {
1920               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1921                 {
1922                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1923                     {
1924                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1925                       next0 = SNAT_OUT2IN_NEXT_DROP;
1926                       goto trace0;
1927                     }
1928                   cached0 = 1;
1929                   goto trace0;
1930                 }
1931               s0 = pool_elt_at_index (per_thread_data->sessions,
1932                                       reass0->sess_index);
1933             }
1934
1935           old_addr0 = ip0->dst_address.as_u32;
1936           ip0->dst_address = s0->in2out.addr;
1937           new_addr0 = ip0->dst_address.as_u32;
1938           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1939
1940           sum0 = ip0->checksum;
1941           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1942                                  ip4_header_t,
1943                                  dst_address /* changed member */);
1944           ip0->checksum = ip_csum_fold (sum0);
1945
1946           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1947             {
1948               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1949                 {
1950                   old_port0 = tcp0->dst_port;
1951                   tcp0->dst_port = s0->in2out.port;
1952                   new_port0 = tcp0->dst_port;
1953
1954                   sum0 = tcp0->checksum;
1955                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1956                                          ip4_header_t,
1957                                          dst_address /* changed member */);
1958
1959                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1960                                          ip4_header_t /* cheat */,
1961                                          length /* changed member */);
1962                   tcp0->checksum = ip_csum_fold(sum0);
1963                 }
1964               else
1965                 {
1966                   old_port0 = udp0->dst_port;
1967                   udp0->dst_port = s0->in2out.port;
1968                   udp0->checksum = 0;
1969                 }
1970             }
1971
1972           /* Accounting */
1973           s0->last_heard = now;
1974           s0->total_pkts++;
1975           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1976           /* Per-user LRU list maintenance */
1977           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1978                              s0->per_user_index);
1979           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1980                               s0->per_user_list_head_index,
1981                               s0->per_user_index);
1982
1983         trace0:
1984           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1985                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1986             {
1987               nat44_out2in_reass_trace_t *t =
1988                  vlib_add_trace (vm, node, b0, sizeof (*t));
1989               t->cached = cached0;
1990               t->sw_if_index = sw_if_index0;
1991               t->next_index = next0;
1992             }
1993
1994           if (cached0)
1995             {
1996               n_left_to_next++;
1997               to_next--;
1998             }
1999           else
2000             {
2001               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2002
2003               /* verify speculative enqueue, maybe switch current next frame */
2004               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2005                                                to_next, n_left_to_next,
2006                                                bi0, next0);
2007             }
2008
2009           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2010             {
2011               from = vlib_frame_vector_args (frame);
2012               u32 len = vec_len (fragments_to_loopback);
2013               if (len <= VLIB_FRAME_SIZE)
2014                 {
2015                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2016                   n_left_from = len;
2017                   vec_reset_length (fragments_to_loopback);
2018                 }
2019               else
2020                 {
2021                   clib_memcpy (from,
2022                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2023                                sizeof (u32) * VLIB_FRAME_SIZE);
2024                   n_left_from = VLIB_FRAME_SIZE;
2025                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2026                 }
2027             }
2028        }
2029
2030       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2031     }
2032
2033   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2034                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2035                                pkts_processed);
2036
2037   nat_send_all_to_node (vm, fragments_to_drop, node,
2038                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2039                         SNAT_OUT2IN_NEXT_DROP);
2040
2041   vec_free (fragments_to_drop);
2042   vec_free (fragments_to_loopback);
2043   return frame->n_vectors;
2044 }
2045
2046 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2047   .function = nat44_out2in_reass_node_fn,
2048   .name = "nat44-out2in-reass",
2049   .vector_size = sizeof (u32),
2050   .format_trace = format_nat44_out2in_reass_trace,
2051   .type = VLIB_NODE_TYPE_INTERNAL,
2052
2053   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2054   .error_strings = snat_out2in_error_strings,
2055
2056   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2057
2058   /* edit / add dispositions here */
2059   .next_nodes = {
2060     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2061     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2062     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2063     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2064     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2065   },
2066 };
2067 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2068                               nat44_out2in_reass_node_fn);
2069
2070 /**************************/
2071 /*** deterministic mode ***/
2072 /**************************/
2073 static uword
2074 snat_det_out2in_node_fn (vlib_main_t * vm,
2075                          vlib_node_runtime_t * node,
2076                          vlib_frame_t * frame)
2077 {
2078   u32 n_left_from, * from, * to_next;
2079   snat_out2in_next_t next_index;
2080   u32 pkts_processed = 0;
2081   snat_main_t * sm = &snat_main;
2082   u32 thread_index = vlib_get_thread_index ();
2083
2084   from = vlib_frame_vector_args (frame);
2085   n_left_from = frame->n_vectors;
2086   next_index = node->cached_next_index;
2087
2088   while (n_left_from > 0)
2089     {
2090       u32 n_left_to_next;
2091
2092       vlib_get_next_frame (vm, node, next_index,
2093                            to_next, n_left_to_next);
2094
2095       while (n_left_from >= 4 && n_left_to_next >= 2)
2096         {
2097           u32 bi0, bi1;
2098           vlib_buffer_t * b0, * b1;
2099           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2100           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2101           u32 sw_if_index0, sw_if_index1;
2102           ip4_header_t * ip0, * ip1;
2103           ip_csum_t sum0, sum1;
2104           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2105           u16 new_port0, old_port0, old_port1, new_port1;
2106           udp_header_t * udp0, * udp1;
2107           tcp_header_t * tcp0, * tcp1;
2108           u32 proto0, proto1;
2109           snat_det_out_key_t key0, key1;
2110           snat_det_map_t * dm0, * dm1;
2111           snat_det_session_t * ses0 = 0, * ses1 = 0;
2112           u32 rx_fib_index0, rx_fib_index1;
2113           icmp46_header_t * icmp0, * icmp1;
2114
2115           /* Prefetch next iteration. */
2116           {
2117             vlib_buffer_t * p2, * p3;
2118
2119             p2 = vlib_get_buffer (vm, from[2]);
2120             p3 = vlib_get_buffer (vm, from[3]);
2121
2122             vlib_prefetch_buffer_header (p2, LOAD);
2123             vlib_prefetch_buffer_header (p3, LOAD);
2124
2125             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2126             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2127           }
2128
2129           /* speculatively enqueue b0 and b1 to the current next frame */
2130           to_next[0] = bi0 = from[0];
2131           to_next[1] = bi1 = from[1];
2132           from += 2;
2133           to_next += 2;
2134           n_left_from -= 2;
2135           n_left_to_next -= 2;
2136
2137           b0 = vlib_get_buffer (vm, bi0);
2138           b1 = vlib_get_buffer (vm, bi1);
2139
2140           ip0 = vlib_buffer_get_current (b0);
2141           udp0 = ip4_next_header (ip0);
2142           tcp0 = (tcp_header_t *) udp0;
2143
2144           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2145
2146           if (PREDICT_FALSE(ip0->ttl == 1))
2147             {
2148               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2149               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2150                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2151                                            0);
2152               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2153               goto trace0;
2154             }
2155
2156           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2157
2158           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2159             {
2160               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2161               icmp0 = (icmp46_header_t *) udp0;
2162
2163               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2164                                   rx_fib_index0, node, next0, thread_index,
2165                                   &ses0, &dm0);
2166               goto trace0;
2167             }
2168
2169           key0.ext_host_addr = ip0->src_address;
2170           key0.ext_host_port = tcp0->src;
2171           key0.out_port = tcp0->dst;
2172
2173           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2174           if (PREDICT_FALSE(!dm0))
2175             {
2176               clib_warning("unknown dst address:  %U",
2177                            format_ip4_address, &ip0->dst_address);
2178               next0 = SNAT_OUT2IN_NEXT_DROP;
2179               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2180               goto trace0;
2181             }
2182
2183           snat_det_reverse(dm0, &ip0->dst_address,
2184                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2185
2186           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2187           if (PREDICT_FALSE(!ses0))
2188             {
2189               clib_warning("no match src %U:%d dst %U:%d for user %U",
2190                            format_ip4_address, &ip0->src_address,
2191                            clib_net_to_host_u16 (tcp0->src),
2192                            format_ip4_address, &ip0->dst_address,
2193                            clib_net_to_host_u16 (tcp0->dst),
2194                            format_ip4_address, &new_addr0);
2195               next0 = SNAT_OUT2IN_NEXT_DROP;
2196               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2197               goto trace0;
2198             }
2199           new_port0 = ses0->in_port;
2200
2201           old_addr0 = ip0->dst_address;
2202           ip0->dst_address = new_addr0;
2203           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2204
2205           sum0 = ip0->checksum;
2206           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2207                                  ip4_header_t,
2208                                  dst_address /* changed member */);
2209           ip0->checksum = ip_csum_fold (sum0);
2210
2211           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2212             {
2213               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2214                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2215               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2216                 snat_det_ses_close(dm0, ses0);
2217
2218               old_port0 = tcp0->dst;
2219               tcp0->dst = new_port0;
2220
2221               sum0 = tcp0->checksum;
2222               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2223                                      ip4_header_t,
2224                                      dst_address /* changed member */);
2225
2226               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2227                                      ip4_header_t /* cheat */,
2228                                      length /* changed member */);
2229               tcp0->checksum = ip_csum_fold(sum0);
2230             }
2231           else
2232             {
2233               old_port0 = udp0->dst_port;
2234               udp0->dst_port = new_port0;
2235               udp0->checksum = 0;
2236             }
2237
2238         trace0:
2239
2240           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2241                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2242             {
2243               snat_out2in_trace_t *t =
2244                  vlib_add_trace (vm, node, b0, sizeof (*t));
2245               t->sw_if_index = sw_if_index0;
2246               t->next_index = next0;
2247               t->session_index = ~0;
2248               if (ses0)
2249                 t->session_index = ses0 - dm0->sessions;
2250             }
2251
2252           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2253
2254           b1 = vlib_get_buffer (vm, bi1);
2255
2256           ip1 = vlib_buffer_get_current (b1);
2257           udp1 = ip4_next_header (ip1);
2258           tcp1 = (tcp_header_t *) udp1;
2259
2260           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2261
2262           if (PREDICT_FALSE(ip1->ttl == 1))
2263             {
2264               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2265               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2266                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2267                                            0);
2268               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2269               goto trace1;
2270             }
2271
2272           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2273
2274           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2275             {
2276               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2277               icmp1 = (icmp46_header_t *) udp1;
2278
2279               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2280                                   rx_fib_index1, node, next1, thread_index,
2281                                   &ses1, &dm1);
2282               goto trace1;
2283             }
2284
2285           key1.ext_host_addr = ip1->src_address;
2286           key1.ext_host_port = tcp1->src;
2287           key1.out_port = tcp1->dst;
2288
2289           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2290           if (PREDICT_FALSE(!dm1))
2291             {
2292               clib_warning("unknown dst address:  %U",
2293                            format_ip4_address, &ip1->dst_address);
2294               next1 = SNAT_OUT2IN_NEXT_DROP;
2295               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2296               goto trace1;
2297             }
2298
2299           snat_det_reverse(dm1, &ip1->dst_address,
2300                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2301
2302           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2303           if (PREDICT_FALSE(!ses1))
2304             {
2305               clib_warning("no match src %U:%d dst %U:%d for user %U",
2306                            format_ip4_address, &ip1->src_address,
2307                            clib_net_to_host_u16 (tcp1->src),
2308                            format_ip4_address, &ip1->dst_address,
2309                            clib_net_to_host_u16 (tcp1->dst),
2310                            format_ip4_address, &new_addr1);
2311               next1 = SNAT_OUT2IN_NEXT_DROP;
2312               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2313               goto trace1;
2314             }
2315           new_port1 = ses1->in_port;
2316
2317           old_addr1 = ip1->dst_address;
2318           ip1->dst_address = new_addr1;
2319           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2320
2321           sum1 = ip1->checksum;
2322           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2323                                  ip4_header_t,
2324                                  dst_address /* changed member */);
2325           ip1->checksum = ip_csum_fold (sum1);
2326
2327           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2328             {
2329               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2330                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2331               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2332                 snat_det_ses_close(dm1, ses1);
2333
2334               old_port1 = tcp1->dst;
2335               tcp1->dst = new_port1;
2336
2337               sum1 = tcp1->checksum;
2338               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2339                                      ip4_header_t,
2340                                      dst_address /* changed member */);
2341
2342               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2343                                      ip4_header_t /* cheat */,
2344                                      length /* changed member */);
2345               tcp1->checksum = ip_csum_fold(sum1);
2346             }
2347           else
2348             {
2349               old_port1 = udp1->dst_port;
2350               udp1->dst_port = new_port1;
2351               udp1->checksum = 0;
2352             }
2353
2354         trace1:
2355
2356           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2357                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2358             {
2359               snat_out2in_trace_t *t =
2360                  vlib_add_trace (vm, node, b1, sizeof (*t));
2361               t->sw_if_index = sw_if_index1;
2362               t->next_index = next1;
2363               t->session_index = ~0;
2364               if (ses1)
2365                 t->session_index = ses1 - dm1->sessions;
2366             }
2367
2368           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2369
2370           /* verify speculative enqueues, maybe switch current next frame */
2371           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2372                                            to_next, n_left_to_next,
2373                                            bi0, bi1, next0, next1);
2374          }
2375
2376       while (n_left_from > 0 && n_left_to_next > 0)
2377         {
2378           u32 bi0;
2379           vlib_buffer_t * b0;
2380           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2381           u32 sw_if_index0;
2382           ip4_header_t * ip0;
2383           ip_csum_t sum0;
2384           ip4_address_t new_addr0, old_addr0;
2385           u16 new_port0, old_port0;
2386           udp_header_t * udp0;
2387           tcp_header_t * tcp0;
2388           u32 proto0;
2389           snat_det_out_key_t key0;
2390           snat_det_map_t * dm0;
2391           snat_det_session_t * ses0 = 0;
2392           u32 rx_fib_index0;
2393           icmp46_header_t * icmp0;
2394
2395           /* speculatively enqueue b0 to the current next frame */
2396           bi0 = from[0];
2397           to_next[0] = bi0;
2398           from += 1;
2399           to_next += 1;
2400           n_left_from -= 1;
2401           n_left_to_next -= 1;
2402
2403           b0 = vlib_get_buffer (vm, bi0);
2404
2405           ip0 = vlib_buffer_get_current (b0);
2406           udp0 = ip4_next_header (ip0);
2407           tcp0 = (tcp_header_t *) udp0;
2408
2409           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2410
2411           if (PREDICT_FALSE(ip0->ttl == 1))
2412             {
2413               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2414               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2415                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2416                                            0);
2417               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2418               goto trace00;
2419             }
2420
2421           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2422
2423           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2424             {
2425               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2426               icmp0 = (icmp46_header_t *) udp0;
2427
2428               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2429                                   rx_fib_index0, node, next0, thread_index,
2430                                   &ses0, &dm0);
2431               goto trace00;
2432             }
2433
2434           key0.ext_host_addr = ip0->src_address;
2435           key0.ext_host_port = tcp0->src;
2436           key0.out_port = tcp0->dst;
2437
2438           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2439           if (PREDICT_FALSE(!dm0))
2440             {
2441               clib_warning("unknown dst address:  %U",
2442                            format_ip4_address, &ip0->dst_address);
2443               next0 = SNAT_OUT2IN_NEXT_DROP;
2444               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2445               goto trace00;
2446             }
2447
2448           snat_det_reverse(dm0, &ip0->dst_address,
2449                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2450
2451           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2452           if (PREDICT_FALSE(!ses0))
2453             {
2454               clib_warning("no match src %U:%d dst %U:%d for user %U",
2455                            format_ip4_address, &ip0->src_address,
2456                            clib_net_to_host_u16 (tcp0->src),
2457                            format_ip4_address, &ip0->dst_address,
2458                            clib_net_to_host_u16 (tcp0->dst),
2459                            format_ip4_address, &new_addr0);
2460               next0 = SNAT_OUT2IN_NEXT_DROP;
2461               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2462               goto trace00;
2463             }
2464           new_port0 = ses0->in_port;
2465
2466           old_addr0 = ip0->dst_address;
2467           ip0->dst_address = new_addr0;
2468           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2469
2470           sum0 = ip0->checksum;
2471           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2472                                  ip4_header_t,
2473                                  dst_address /* changed member */);
2474           ip0->checksum = ip_csum_fold (sum0);
2475
2476           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2477             {
2478               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2479                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2480               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2481                 snat_det_ses_close(dm0, ses0);
2482
2483               old_port0 = tcp0->dst;
2484               tcp0->dst = new_port0;
2485
2486               sum0 = tcp0->checksum;
2487               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2488                                      ip4_header_t,
2489                                      dst_address /* changed member */);
2490
2491               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2492                                      ip4_header_t /* cheat */,
2493                                      length /* changed member */);
2494               tcp0->checksum = ip_csum_fold(sum0);
2495             }
2496           else
2497             {
2498               old_port0 = udp0->dst_port;
2499               udp0->dst_port = new_port0;
2500               udp0->checksum = 0;
2501             }
2502
2503         trace00:
2504
2505           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2506                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2507             {
2508               snat_out2in_trace_t *t =
2509                  vlib_add_trace (vm, node, b0, sizeof (*t));
2510               t->sw_if_index = sw_if_index0;
2511               t->next_index = next0;
2512               t->session_index = ~0;
2513               if (ses0)
2514                 t->session_index = ses0 - dm0->sessions;
2515             }
2516
2517           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2518
2519           /* verify speculative enqueue, maybe switch current next frame */
2520           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2521                                            to_next, n_left_to_next,
2522                                            bi0, next0);
2523         }
2524
2525       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2526     }
2527
2528   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2529                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2530                                pkts_processed);
2531   return frame->n_vectors;
2532 }
2533
2534 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2535   .function = snat_det_out2in_node_fn,
2536   .name = "nat44-det-out2in",
2537   .vector_size = sizeof (u32),
2538   .format_trace = format_snat_out2in_trace,
2539   .type = VLIB_NODE_TYPE_INTERNAL,
2540
2541   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2542   .error_strings = snat_out2in_error_strings,
2543
2544   .runtime_data_bytes = sizeof (snat_runtime_t),
2545
2546   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2547
2548   /* edit / add dispositions here */
2549   .next_nodes = {
2550     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2551     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2552     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2553     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2554     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2555   },
2556 };
2557 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2558
2559 /**
2560  * Get address and port values to be used for ICMP packet translation
2561  * and create session if needed
2562  *
2563  * @param[in,out] sm             NAT main
2564  * @param[in,out] node           NAT node runtime
2565  * @param[in] thread_index       thread index
2566  * @param[in,out] b0             buffer containing packet to be translated
2567  * @param[out] p_proto           protocol used for matching
2568  * @param[out] p_value           address and port after NAT translation
2569  * @param[out] p_dont_translate  if packet should not be translated
2570  * @param d                      optional parameter
2571  * @param e                      optional parameter
2572  */
2573 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2574                           u32 thread_index, vlib_buffer_t *b0,
2575                           ip4_header_t *ip0, u8 *p_proto,
2576                           snat_session_key_t *p_value,
2577                           u8 *p_dont_translate, void *d, void *e)
2578 {
2579   icmp46_header_t *icmp0;
2580   u32 sw_if_index0;
2581   u8 protocol;
2582   snat_det_out_key_t key0;
2583   u8 dont_translate = 0;
2584   u32 next0 = ~0;
2585   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2586   ip4_header_t *inner_ip0;
2587   void *l4_header = 0;
2588   icmp46_header_t *inner_icmp0;
2589   snat_det_map_t * dm0 = 0;
2590   ip4_address_t new_addr0 = {{0}};
2591   snat_det_session_t * ses0 = 0;
2592   ip4_address_t out_addr;
2593
2594   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2595   echo0 = (icmp_echo_header_t *)(icmp0+1);
2596   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2597
2598   if (!icmp_is_error_message (icmp0))
2599     {
2600       protocol = SNAT_PROTOCOL_ICMP;
2601       key0.ext_host_addr = ip0->src_address;
2602       key0.ext_host_port = 0;
2603       key0.out_port = echo0->identifier;
2604       out_addr = ip0->dst_address;
2605     }
2606   else
2607     {
2608       inner_ip0 = (ip4_header_t *)(echo0+1);
2609       l4_header = ip4_next_header (inner_ip0);
2610       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2611       key0.ext_host_addr = inner_ip0->dst_address;
2612       out_addr = inner_ip0->src_address;
2613       switch (protocol)
2614         {
2615         case SNAT_PROTOCOL_ICMP:
2616           inner_icmp0 = (icmp46_header_t*)l4_header;
2617           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2618           key0.ext_host_port = 0;
2619           key0.out_port = inner_echo0->identifier;
2620           break;
2621         case SNAT_PROTOCOL_UDP:
2622         case SNAT_PROTOCOL_TCP:
2623           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2624           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2625           break;
2626         default:
2627           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2628           next0 = SNAT_OUT2IN_NEXT_DROP;
2629           goto out;
2630         }
2631     }
2632
2633   dm0 = snat_det_map_by_out(sm, &out_addr);
2634   if (PREDICT_FALSE(!dm0))
2635     {
2636       /* Don't NAT packet aimed at the intfc address */
2637       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2638                                           ip0->dst_address.as_u32)))
2639         {
2640           dont_translate = 1;
2641           goto out;
2642         }
2643       clib_warning("unknown dst address:  %U",
2644                    format_ip4_address, &ip0->dst_address);
2645       goto out;
2646     }
2647
2648   snat_det_reverse(dm0, &ip0->dst_address,
2649                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2650
2651   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2652   if (PREDICT_FALSE(!ses0))
2653     {
2654       /* Don't NAT packet aimed at the intfc address */
2655       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2656                                           ip0->dst_address.as_u32)))
2657         {
2658           dont_translate = 1;
2659           goto out;
2660         }
2661       clib_warning("no match src %U:%d dst %U:%d for user %U",
2662                    format_ip4_address, &key0.ext_host_addr,
2663                    clib_net_to_host_u16 (key0.ext_host_port),
2664                    format_ip4_address, &out_addr,
2665                    clib_net_to_host_u16 (key0.out_port),
2666                    format_ip4_address, &new_addr0);
2667       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2668       next0 = SNAT_OUT2IN_NEXT_DROP;
2669       goto out;
2670     }
2671
2672   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2673                     !icmp_is_error_message (icmp0)))
2674     {
2675       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2676       next0 = SNAT_OUT2IN_NEXT_DROP;
2677       goto out;
2678     }
2679
2680   goto out;
2681
2682 out:
2683   *p_proto = protocol;
2684   if (ses0)
2685     {
2686       p_value->addr = new_addr0;
2687       p_value->fib_index = sm->inside_fib_index;
2688       p_value->port = ses0->in_port;
2689     }
2690   *p_dont_translate = dont_translate;
2691   if (d)
2692     *(snat_det_session_t**)d = ses0;
2693   if (e)
2694     *(snat_det_map_t**)e = dm0;
2695   return next0;
2696 }
2697
2698 /**********************/
2699 /*** worker handoff ***/
2700 /**********************/
2701 static uword
2702 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2703                                vlib_node_runtime_t * node,
2704                                vlib_frame_t * frame)
2705 {
2706   snat_main_t *sm = &snat_main;
2707   vlib_thread_main_t *tm = vlib_get_thread_main ();
2708   u32 n_left_from, *from, *to_next = 0;
2709   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2710   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2711     = 0;
2712   vlib_frame_queue_elt_t *hf = 0;
2713   vlib_frame_t *f = 0;
2714   int i;
2715   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2716   u32 next_worker_index = 0;
2717   u32 current_worker_index = ~0;
2718   u32 thread_index = vlib_get_thread_index ();
2719
2720   ASSERT (vec_len (sm->workers));
2721
2722   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2723     {
2724       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2725
2726       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2727                                sm->first_worker_index + sm->num_workers - 1,
2728                                (vlib_frame_queue_t *) (~0));
2729     }
2730
2731   from = vlib_frame_vector_args (frame);
2732   n_left_from = frame->n_vectors;
2733
2734   while (n_left_from > 0)
2735     {
2736       u32 bi0;
2737       vlib_buffer_t *b0;
2738       u32 sw_if_index0;
2739       u32 rx_fib_index0;
2740       ip4_header_t * ip0;
2741       u8 do_handoff;
2742
2743       bi0 = from[0];
2744       from += 1;
2745       n_left_from -= 1;
2746
2747       b0 = vlib_get_buffer (vm, bi0);
2748
2749       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2750       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2751
2752       ip0 = vlib_buffer_get_current (b0);
2753
2754       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2755
2756       if (PREDICT_FALSE (next_worker_index != thread_index))
2757         {
2758           do_handoff = 1;
2759
2760           if (next_worker_index != current_worker_index)
2761             {
2762               if (hf)
2763                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2764
2765               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2766                                                       next_worker_index,
2767                                                       handoff_queue_elt_by_worker_index);
2768
2769               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2770               to_next_worker = &hf->buffer_index[hf->n_vectors];
2771               current_worker_index = next_worker_index;
2772             }
2773
2774           /* enqueue to correct worker thread */
2775           to_next_worker[0] = bi0;
2776           to_next_worker++;
2777           n_left_to_next_worker--;
2778
2779           if (n_left_to_next_worker == 0)
2780             {
2781               hf->n_vectors = VLIB_FRAME_SIZE;
2782               vlib_put_frame_queue_elt (hf);
2783               current_worker_index = ~0;
2784               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2785               hf = 0;
2786             }
2787         }
2788       else
2789         {
2790           do_handoff = 0;
2791           /* if this is 1st frame */
2792           if (!f)
2793             {
2794               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2795               to_next = vlib_frame_vector_args (f);
2796             }
2797
2798           to_next[0] = bi0;
2799           to_next += 1;
2800           f->n_vectors++;
2801         }
2802
2803       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2804                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2805         {
2806           snat_out2in_worker_handoff_trace_t *t =
2807             vlib_add_trace (vm, node, b0, sizeof (*t));
2808           t->next_worker_index = next_worker_index;
2809           t->do_handoff = do_handoff;
2810         }
2811     }
2812
2813   if (f)
2814     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2815
2816   if (hf)
2817     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2818
2819   /* Ship frames to the worker nodes */
2820   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2821     {
2822       if (handoff_queue_elt_by_worker_index[i])
2823         {
2824           hf = handoff_queue_elt_by_worker_index[i];
2825           /*
2826            * It works better to let the handoff node
2827            * rate-adapt, always ship the handoff queue element.
2828            */
2829           if (1 || hf->n_vectors == hf->last_n_vectors)
2830             {
2831               vlib_put_frame_queue_elt (hf);
2832               handoff_queue_elt_by_worker_index[i] = 0;
2833             }
2834           else
2835             hf->last_n_vectors = hf->n_vectors;
2836         }
2837       congested_handoff_queue_by_worker_index[i] =
2838         (vlib_frame_queue_t *) (~0);
2839     }
2840   hf = 0;
2841   current_worker_index = ~0;
2842   return frame->n_vectors;
2843 }
2844
2845 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2846   .function = snat_out2in_worker_handoff_fn,
2847   .name = "nat44-out2in-worker-handoff",
2848   .vector_size = sizeof (u32),
2849   .format_trace = format_snat_out2in_worker_handoff_trace,
2850   .type = VLIB_NODE_TYPE_INTERNAL,
2851
2852   .n_next_nodes = 1,
2853
2854   .next_nodes = {
2855     [0] = "error-drop",
2856   },
2857 };
2858
2859 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2860
2861 static uword
2862 snat_out2in_fast_node_fn (vlib_main_t * vm,
2863                           vlib_node_runtime_t * node,
2864                           vlib_frame_t * frame)
2865 {
2866   u32 n_left_from, * from, * to_next;
2867   snat_out2in_next_t next_index;
2868   u32 pkts_processed = 0;
2869   snat_main_t * sm = &snat_main;
2870
2871   from = vlib_frame_vector_args (frame);
2872   n_left_from = frame->n_vectors;
2873   next_index = node->cached_next_index;
2874
2875   while (n_left_from > 0)
2876     {
2877       u32 n_left_to_next;
2878
2879       vlib_get_next_frame (vm, node, next_index,
2880                            to_next, n_left_to_next);
2881
2882       while (n_left_from > 0 && n_left_to_next > 0)
2883         {
2884           u32 bi0;
2885           vlib_buffer_t * b0;
2886           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2887           u32 sw_if_index0;
2888           ip4_header_t * ip0;
2889           ip_csum_t sum0;
2890           u32 new_addr0, old_addr0;
2891           u16 new_port0, old_port0;
2892           udp_header_t * udp0;
2893           tcp_header_t * tcp0;
2894           icmp46_header_t * icmp0;
2895           snat_session_key_t key0, sm0;
2896           u32 proto0;
2897           u32 rx_fib_index0;
2898
2899           /* speculatively enqueue b0 to the current next frame */
2900           bi0 = from[0];
2901           to_next[0] = bi0;
2902           from += 1;
2903           to_next += 1;
2904           n_left_from -= 1;
2905           n_left_to_next -= 1;
2906
2907           b0 = vlib_get_buffer (vm, bi0);
2908
2909           ip0 = vlib_buffer_get_current (b0);
2910           udp0 = ip4_next_header (ip0);
2911           tcp0 = (tcp_header_t *) udp0;
2912           icmp0 = (icmp46_header_t *) udp0;
2913
2914           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2915           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2916
2917           vnet_feature_next (sw_if_index0, &next0, b0);
2918
2919           if (PREDICT_FALSE(ip0->ttl == 1))
2920             {
2921               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2922               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2923                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2924                                            0);
2925               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2926               goto trace00;
2927             }
2928
2929           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2930
2931           if (PREDICT_FALSE (proto0 == ~0))
2932               goto trace00;
2933
2934           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2935             {
2936               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2937                                   rx_fib_index0, node, next0, ~0, 0, 0);
2938               goto trace00;
2939             }
2940
2941           key0.addr = ip0->dst_address;
2942           key0.port = udp0->dst_port;
2943           key0.fib_index = rx_fib_index0;
2944
2945           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2946             {
2947               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2948               goto trace00;
2949             }
2950
2951           new_addr0 = sm0.addr.as_u32;
2952           new_port0 = sm0.port;
2953           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2954           old_addr0 = ip0->dst_address.as_u32;
2955           ip0->dst_address.as_u32 = new_addr0;
2956
2957           sum0 = ip0->checksum;
2958           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2959                                  ip4_header_t,
2960                                  dst_address /* changed member */);
2961           ip0->checksum = ip_csum_fold (sum0);
2962
2963           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2964             {
2965                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2966                 {
2967                   old_port0 = tcp0->dst_port;
2968                   tcp0->dst_port = new_port0;
2969
2970                   sum0 = tcp0->checksum;
2971                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2972                                          ip4_header_t,
2973                                          dst_address /* changed member */);
2974
2975                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2976                                          ip4_header_t /* cheat */,
2977                                          length /* changed member */);
2978                   tcp0->checksum = ip_csum_fold(sum0);
2979                 }
2980               else
2981                 {
2982                   old_port0 = udp0->dst_port;
2983                   udp0->dst_port = new_port0;
2984                   udp0->checksum = 0;
2985                 }
2986             }
2987           else
2988             {
2989               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2990                 {
2991                   sum0 = tcp0->checksum;
2992                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2993                                          ip4_header_t,
2994                                          dst_address /* changed member */);
2995
2996                   tcp0->checksum = ip_csum_fold(sum0);
2997                 }
2998             }
2999
3000         trace00:
3001
3002           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3003                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3004             {
3005               snat_out2in_trace_t *t =
3006                  vlib_add_trace (vm, node, b0, sizeof (*t));
3007               t->sw_if_index = sw_if_index0;
3008               t->next_index = next0;
3009             }
3010
3011           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3012
3013           /* verify speculative enqueue, maybe switch current next frame */
3014           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3015                                            to_next, n_left_to_next,
3016                                            bi0, next0);
3017         }
3018
3019       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3020     }
3021
3022   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3023                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3024                                pkts_processed);
3025   return frame->n_vectors;
3026 }
3027
3028 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3029   .function = snat_out2in_fast_node_fn,
3030   .name = "nat44-out2in-fast",
3031   .vector_size = sizeof (u32),
3032   .format_trace = format_snat_out2in_fast_trace,
3033   .type = VLIB_NODE_TYPE_INTERNAL,
3034
3035   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3036   .error_strings = snat_out2in_error_strings,
3037
3038   .runtime_data_bytes = sizeof (snat_runtime_t),
3039
3040   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3041
3042   /* edit / add dispositions here */
3043   .next_nodes = {
3044     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3045     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3046     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3047     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3048     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3049   },
3050 };
3051 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);