src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   user_session_increment (sm, u, 1 /* static */);
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip)
 337 {
 338   nat_ed_ses_key_t key;
 339   clib_bihash_kv_16_8_t kv;
 340   udp_header_t *udp;
 341
 342   if (ip->protocol == IP_PROTOCOL_ICMP)
 343     {
 344       if (icmp_get_ed_key (ip, &key))
 345         return;
 346     }
 347   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 348     {
 349       udp = ip4_next_header(ip);
 350       key.r_addr = ip->src_address;
 351       key.l_addr = ip->dst_address;
 352       key.proto = ip->protocol;
 353       key.l_port = udp->dst_port;
 354       key.r_port = udp->src_port;
 355     }
 356   else
 357     {
 358       key.r_addr = ip->src_address;
 359       key.l_addr = ip->dst_address;
 360       key.proto = ip->protocol;
 361       key.l_port = key.r_port = 0;
 362     }
 363   key.fib_index = 0;
 364   kv.key[0] = key.as_u64[0];
 365   kv.key[1] = key.as_u64[1];
 366   kv.value = ~0ULL;
 367
 368   if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 369     clib_warning ("in2out_ed key add failed");
 370 }
 371
 372 /**
 373  * Get address and port values to be used for ICMP packet translation
 374  * and create session if needed
 375  *
 376  * @param[in,out] sm             NAT main
 377  * @param[in,out] node           NAT node runtime
 378  * @param[in] thread_index       thread index
 379  * @param[in,out] b0             buffer containing packet to be translated
 380  * @param[out] p_proto           protocol used for matching
 381  * @param[out] p_value           address and port after NAT translation
 382  * @param[out] p_dont_translate  if packet should not be translated
 383  * @param d                      optional parameter
 384  * @param e                      optional parameter
 385  */
 386 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 387                            u32 thread_index, vlib_buffer_t *b0,
 388                            ip4_header_t *ip0, u8 *p_proto,
 389                            snat_session_key_t *p_value,
 390                            u8 *p_dont_translate, void *d, void *e)
 391 {
 392   icmp46_header_t *icmp0;
 393   u32 sw_if_index0;
 394   u32 rx_fib_index0;
 395   snat_session_key_t key0;
 396   snat_session_key_t sm0;
 397   snat_session_t *s0 = 0;
 398   u8 dont_translate = 0;
 399   clib_bihash_kv_8_8_t kv0, value0;
 400   u8 is_addr_only;
 401   u32 next0 = ~0;
 402   int err;
 403
 404   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 405   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 406   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 407
 408   key0.protocol = 0;
 409
 410   err = icmp_get_key (ip0, &key0);
 411   if (err != -1)
 412     {
 413       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 414       next0 = SNAT_OUT2IN_NEXT_DROP;
 415       goto out;
 416     }
 417   key0.fib_index = rx_fib_index0;
 418
 419   kv0.key = key0.as_u64;
 420
 421   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 422                               &value0))
 423     {
 424       /* Try to match static mapping by external address and port,
 425          destination address and port in packet */
 426       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 427         {
 428           if (!sm->forwarding_enabled)
 429             {
 430               /* Don't NAT packet aimed at the intfc address */
 431               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 432                                                   ip0->dst_address.as_u32)))
 433                 {
 434                   dont_translate = 1;
 435                   goto out;
 436                 }
 437               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 438               next0 = SNAT_OUT2IN_NEXT_DROP;
 439               goto out;
 440             }
 441           else
 442             {
 443               dont_translate = 1;
 444               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 445                 {
 446                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 447                   goto out;
 448                 }
 449               create_bypass_for_fwd(sm, ip0);
 450               goto out;
 451             }
 452         }
 453
 454       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 455                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 456         {
 457           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 458           next0 = SNAT_OUT2IN_NEXT_DROP;
 459           goto out;
 460         }
 461
 462       /* Create session initiated by host from external network */
 463       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 464                                              node, thread_index);
 465
 466       if (!s0)
 467         {
 468           next0 = SNAT_OUT2IN_NEXT_DROP;
 469           goto out;
 470         }
 471     }
 472   else
 473     {
 474       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 475                         icmp0->type != ICMP4_echo_request &&
 476                         !icmp_is_error_message (icmp0)))
 477         {
 478           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 479           next0 = SNAT_OUT2IN_NEXT_DROP;
 480           goto out;
 481         }
 482
 483       if (PREDICT_FALSE (value0.value == ~0ULL))
 484         {
 485           nat_ed_ses_key_t key;
 486           clib_bihash_kv_16_8_t s_kv, s_value;
 487
 488           key.as_u64[0] = 0;
 489           key.as_u64[1] = 0;
 490           if (icmp_get_ed_key (ip0, &key))
 491             {
 492               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 493               next0 = SNAT_OUT2IN_NEXT_DROP;
 494               goto out;
 495             }
 496           key.fib_index = rx_fib_index0;
 497           s_kv.key[0] = key.as_u64[0];
 498           s_kv.key[1] = key.as_u64[1];
 499           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 500             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 501                                     s_value.value);
 502           else
 503            {
 504               next0 = SNAT_OUT2IN_NEXT_DROP;
 505               goto out;
 506            }
 507         }
 508       else
 509         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 510                                 value0.value);
 511     }
 512
 513 out:
 514   *p_proto = key0.protocol;
 515   if (s0)
 516     *p_value = s0->in2out;
 517   *p_dont_translate = dont_translate;
 518   if (d)
 519     *(snat_session_t**)d = s0;
 520   return next0;
 521 }
 522
 523 /**
 524  * Get address and port values to be used for ICMP packet translation
 525  *
 526  * @param[in] sm                 NAT main
 527  * @param[in,out] node           NAT node runtime
 528  * @param[in] thread_index       thread index
 529  * @param[in,out] b0             buffer containing packet to be translated
 530  * @param[out] p_proto           protocol used for matching
 531  * @param[out] p_value           address and port after NAT translation
 532  * @param[out] p_dont_translate  if packet should not be translated
 533  * @param d                      optional parameter
 534  * @param e                      optional parameter
 535  */
 536 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 537                            u32 thread_index, vlib_buffer_t *b0,
 538                            ip4_header_t *ip0, u8 *p_proto,
 539                            snat_session_key_t *p_value,
 540                            u8 *p_dont_translate, void *d, void *e)
 541 {
 542   icmp46_header_t *icmp0;
 543   u32 sw_if_index0;
 544   u32 rx_fib_index0;
 545   snat_session_key_t key0;
 546   snat_session_key_t sm0;
 547   u8 dont_translate = 0;
 548   u8 is_addr_only;
 549   u32 next0 = ~0;
 550   int err;
 551
 552   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 553   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 554   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 555
 556   err = icmp_get_key (ip0, &key0);
 557   if (err != -1)
 558     {
 559       b0->error = node->errors[err];
 560       next0 = SNAT_OUT2IN_NEXT_DROP;
 561       goto out2;
 562     }
 563   key0.fib_index = rx_fib_index0;
 564
 565   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 566     {
 567       /* Don't NAT packet aimed at the intfc address */
 568       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 569         {
 570           dont_translate = 1;
 571           goto out;
 572         }
 573       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 574       next0 = SNAT_OUT2IN_NEXT_DROP;
 575       goto out;
 576     }
 577
 578   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 579                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 580                     !icmp_is_error_message (icmp0)))
 581     {
 582       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 583       next0 = SNAT_OUT2IN_NEXT_DROP;
 584       goto out;
 585     }
 586
 587 out:
 588   *p_value = sm0;
 589 out2:
 590   *p_proto = key0.protocol;
 591   *p_dont_translate = dont_translate;
 592   return next0;
 593 }
 594
 595 static inline u32 icmp_out2in (snat_main_t *sm,
 596                                vlib_buffer_t * b0,
 597                                ip4_header_t * ip0,
 598                                icmp46_header_t * icmp0,
 599                                u32 sw_if_index0,
 600                                u32 rx_fib_index0,
 601                                vlib_node_runtime_t * node,
 602                                u32 next0,
 603                                u32 thread_index,
 604                                void *d,
 605                                void *e)
 606 {
 607   snat_session_key_t sm0;
 608   u8 protocol;
 609   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 610   ip4_header_t *inner_ip0 = 0;
 611   void *l4_header = 0;
 612   icmp46_header_t *inner_icmp0;
 613   u8 dont_translate;
 614   u32 new_addr0, old_addr0;
 615   u16 old_id0, new_id0;
 616   ip_csum_t sum0;
 617   u16 checksum0;
 618   u32 next0_tmp;
 619
 620   echo0 = (icmp_echo_header_t *)(icmp0+1);
 621
 622   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 623                                        &protocol, &sm0, &dont_translate, d, e);
 624   if (next0_tmp != ~0)
 625     next0 = next0_tmp;
 626   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 627     goto out;
 628
 629   sum0 = ip_incremental_checksum (0, icmp0,
 630                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 631   checksum0 = ~ip_csum_fold (sum0);
 632   if (checksum0 != 0 && checksum0 != 0xffff)
 633     {
 634       next0 = SNAT_OUT2IN_NEXT_DROP;
 635       goto out;
 636     }
 637
 638   old_addr0 = ip0->dst_address.as_u32;
 639   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 640   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 641
 642   sum0 = ip0->checksum;
 643   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 644                          dst_address /* changed member */);
 645   ip0->checksum = ip_csum_fold (sum0);
 646
 647   if (icmp0->checksum == 0)
 648     icmp0->checksum = 0xffff;
 649
 650   if (!icmp_is_error_message (icmp0))
 651     {
 652       new_id0 = sm0.port;
 653       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 654         {
 655           old_id0 = echo0->identifier;
 656           new_id0 = sm0.port;
 657           echo0->identifier = new_id0;
 658
 659           sum0 = icmp0->checksum;
 660           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 661                                  identifier /* changed member */);
 662           icmp0->checksum = ip_csum_fold (sum0);
 663         }
 664     }
 665   else
 666     {
 667       inner_ip0 = (ip4_header_t *)(echo0+1);
 668       l4_header = ip4_next_header (inner_ip0);
 669
 670       if (!ip4_header_checksum_is_valid (inner_ip0))
 671         {
 672           next0 = SNAT_OUT2IN_NEXT_DROP;
 673           goto out;
 674         }
 675
 676       old_addr0 = inner_ip0->src_address.as_u32;
 677       inner_ip0->src_address = sm0.addr;
 678       new_addr0 = inner_ip0->src_address.as_u32;
 679
 680       sum0 = icmp0->checksum;
 681       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 682                              src_address /* changed member */);
 683       icmp0->checksum = ip_csum_fold (sum0);
 684
 685       switch (protocol)
 686         {
 687         case SNAT_PROTOCOL_ICMP:
 688           inner_icmp0 = (icmp46_header_t*)l4_header;
 689           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 690
 691           old_id0 = inner_echo0->identifier;
 692           new_id0 = sm0.port;
 693           inner_echo0->identifier = new_id0;
 694
 695           sum0 = icmp0->checksum;
 696           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 697                                  identifier);
 698           icmp0->checksum = ip_csum_fold (sum0);
 699           break;
 700         case SNAT_PROTOCOL_UDP:
 701         case SNAT_PROTOCOL_TCP:
 702           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 703           new_id0 = sm0.port;
 704           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 705
 706           sum0 = icmp0->checksum;
 707           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 708                                  src_port);
 709           icmp0->checksum = ip_csum_fold (sum0);
 710           break;
 711         default:
 712           ASSERT(0);
 713         }
 714     }
 715
 716 out:
 717   return next0;
 718 }
 719
 720
 721 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 722                                          vlib_buffer_t * b0,
 723                                          ip4_header_t * ip0,
 724                                          icmp46_header_t * icmp0,
 725                                          u32 sw_if_index0,
 726                                          u32 rx_fib_index0,
 727                                          vlib_node_runtime_t * node,
 728                                          u32 next0, f64 now,
 729                                          u32 thread_index,
 730                                          snat_session_t ** p_s0)
 731 {
 732   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 733                       next0, thread_index, p_s0, 0);
 734   snat_session_t * s0 = *p_s0;
 735   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 736     {
 737       /* Accounting */
 738       s0->last_heard = now;
 739       s0->total_pkts++;
 740       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 741       /* Per-user LRU list maintenance */
 742       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 743                          s0->per_user_index);
 744       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 745                           s0->per_user_list_head_index,
 746                           s0->per_user_index);
 747     }
 748   return next0;
 749 }
 750
 751 static snat_session_t *
 752 snat_out2in_unknown_proto (snat_main_t *sm,
 753                            vlib_buffer_t * b,
 754                            ip4_header_t * ip,
 755                            u32 rx_fib_index,
 756                            u32 thread_index,
 757                            f64 now,
 758                            vlib_main_t * vm,
 759                            vlib_node_runtime_t * node)
 760 {
 761   clib_bihash_kv_8_8_t kv, value;
 762   clib_bihash_kv_16_8_t s_kv, s_value;
 763   snat_static_mapping_t *m;
 764   snat_session_key_t m_key;
 765   u32 old_addr, new_addr;
 766   ip_csum_t sum;
 767   nat_ed_ses_key_t key;
 768   snat_session_t * s;
 769   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 770   snat_user_t *u;
 771
 772   old_addr = ip->dst_address.as_u32;
 773
 774   key.l_addr = ip->dst_address;
 775   key.r_addr = ip->src_address;
 776   key.fib_index = rx_fib_index;
 777   key.proto = ip->protocol;
 778   key.r_port = 0;
 779   key.l_port = 0;
 780   s_kv.key[0] = key.as_u64[0];
 781   s_kv.key[1] = key.as_u64[1];
 782
 783   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 784     {
 785       s = pool_elt_at_index (tsm->sessions, s_value.value);
 786       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 787     }
 788   else
 789     {
 790       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 791         {
 792           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 793           return 0;
 794         }
 795
 796       m_key.addr = ip->dst_address;
 797       m_key.port = 0;
 798       m_key.protocol = 0;
 799       m_key.fib_index = rx_fib_index;
 800       kv.key = m_key.as_u64;
 801       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 802         {
 803           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 804           return 0;
 805         }
 806
 807       m = pool_elt_at_index (sm->static_mappings, value.value);
 808
 809       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 810
 811       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 812                                   thread_index);
 813       if (!u)
 814         {
 815           clib_warning ("create NAT user failed");
 816           return 0;
 817         }
 818
 819       /* Create a new session */
 820       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 821       if (!s)
 822         {
 823           clib_warning ("create NAT session failed");
 824           return 0;
 825         }
 826
 827       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 828       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 829       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 830       s->outside_address_index = ~0;
 831       s->out2in.addr.as_u32 = old_addr;
 832       s->out2in.fib_index = rx_fib_index;
 833       s->in2out.addr.as_u32 = new_addr;
 834       s->in2out.fib_index = m->fib_index;
 835       s->in2out.port = s->out2in.port = ip->protocol;
 836       user_session_increment (sm, u, 1 /* static */);
 837
 838       /* Add to lookup tables */
 839       s_kv.value = s - tsm->sessions;
 840       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 841         clib_warning ("out2in key add failed");
 842
 843       key.l_addr = ip->dst_address;
 844       key.fib_index = m->fib_index;
 845       s_kv.key[0] = key.as_u64[0];
 846       s_kv.key[1] = key.as_u64[1];
 847       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 848         clib_warning ("in2out key add failed");
 849    }
 850
 851   /* Update IP checksum */
 852   sum = ip->checksum;
 853   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 854   ip->checksum = ip_csum_fold (sum);
 855
 856   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 857
 858   /* Accounting */
 859   s->last_heard = now;
 860   s->total_pkts++;
 861   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 862   /* Per-user LRU list maintenance */
 863   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 864   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 865                       s->per_user_index);
 866
 867   return s;
 868 }
 869
 870 static snat_session_t *
 871 snat_out2in_lb (snat_main_t *sm,
 872                 vlib_buffer_t * b,
 873                 ip4_header_t * ip,
 874                 u32 rx_fib_index,
 875                 u32 thread_index,
 876                 f64 now,
 877                 vlib_main_t * vm,
 878                 vlib_node_runtime_t * node)
 879 {
 880   nat_ed_ses_key_t key;
 881   clib_bihash_kv_16_8_t s_kv, s_value;
 882   udp_header_t *udp = ip4_next_header (ip);
 883   tcp_header_t *tcp = (tcp_header_t *) udp;
 884   snat_session_t *s = 0;
 885   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 886   snat_session_key_t e_key, l_key;
 887   u32 old_addr, new_addr;
 888   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 889   u16 new_port, old_port;
 890   ip_csum_t sum;
 891   snat_user_t *u;
 892   u32 address_index;
 893   snat_session_key_t eh_key;
 894   u8 twice_nat, lb;
 895
 896   old_addr = ip->dst_address.as_u32;
 897
 898   key.l_addr = ip->dst_address;
 899   key.r_addr = ip->src_address;
 900   key.fib_index = rx_fib_index;
 901   key.proto = ip->protocol;
 902   key.r_port = udp->src_port;
 903   key.l_port = udp->dst_port;
 904   s_kv.key[0] = key.as_u64[0];
 905   s_kv.key[1] = key.as_u64[1];
 906
 907   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 908     {
 909       s = pool_elt_at_index (tsm->sessions, s_value.value);
 910     }
 911   else
 912     {
 913       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 914         {
 915           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 916           return 0;
 917         }
 918
 919       e_key.addr = ip->dst_address;
 920       e_key.port = udp->dst_port;
 921       e_key.protocol = proto;
 922       e_key.fib_index = rx_fib_index;
 923       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 924         return 0;
 925
 926       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 927                                   thread_index);
 928       if (!u)
 929       {
 930         clib_warning ("create NAT user failed");
 931         return 0;
 932       }
 933
 934       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 935       if (!s)
 936         {
 937           clib_warning ("create NAT session failed");
 938           return 0;
 939         }
 940
 941       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 942       s->ext_host_port = udp->src_port;
 943       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 944       if (lb)
 945         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 946       s->outside_address_index = ~0;
 947       s->out2in = e_key;
 948       s->in2out = l_key;
 949       user_session_increment (sm, u, 1 /* static */);
 950
 951       /* Add to lookup tables */
 952       s_kv.value = s - tsm->sessions;
 953       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 954         clib_warning ("out2in-ed key add failed");
 955
 956       if (twice_nat)
 957         {
 958           eh_key.protocol = proto;
 959           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 960                                                    thread_index, &eh_key,
 961                                                    &address_index,
 962                                                    sm->port_per_thread,
 963                                                    sm->per_thread_data[thread_index].snat_thread_index))
 964             {
 965               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
 966               return 0;
 967             }
 968           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 969           key.r_port = s->ext_host_nat_port = eh_key.port;
 970           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 971         }
 972       key.l_addr = l_key.addr;
 973       key.fib_index = l_key.fib_index;
 974       key.l_port = l_key.port;
 975       s_kv.key[0] = key.as_u64[0];
 976       s_kv.key[1] = key.as_u64[1];
 977       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 978         clib_warning ("in2out-ed key add failed");
 979     }
 980
 981   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 982
 983   /* Update IP checksum */
 984   sum = ip->checksum;
 985   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 986   if (is_twice_nat_session (s))
 987     sum = ip_csum_update (sum, ip->src_address.as_u32,
 988                           s->ext_host_nat_addr.as_u32, ip4_header_t,
 989                           src_address);
 990   ip->checksum = ip_csum_fold (sum);
 991
 992   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 993     {
 994       old_port = tcp->dst_port;
 995       tcp->dst_port = s->in2out.port;
 996       new_port = tcp->dst_port;
 997
 998       sum = tcp->checksum;
 999       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1000       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1001       if (is_twice_nat_session (s))
1002         {
1003           sum = ip_csum_update (sum, ip->src_address.as_u32,
1004                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1005                                 dst_address);
1006           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1007                                 ip4_header_t, length);
1008           tcp->src_port = s->ext_host_nat_port;
1009           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1010         }
1011       tcp->checksum = ip_csum_fold(sum);
1012     }
1013   else
1014     {
1015       udp->dst_port = s->in2out.port;
1016       if (is_twice_nat_session (s))
1017         {
1018           udp->src_port = s->ext_host_nat_port;
1019           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1020         }
1021       udp->checksum = 0;
1022     }
1023
1024   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1025
1026   /* Accounting */
1027   s->last_heard = now;
1028   s->total_pkts++;
1029   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1030   /* Per-user LRU list maintenance */
1031   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1032   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1033                       s->per_user_index);
1034
1035   return s;
1036 }
1037
1038 static uword
1039 snat_out2in_node_fn (vlib_main_t * vm,
1040                   vlib_node_runtime_t * node,
1041                   vlib_frame_t * frame)
1042 {
1043   u32 n_left_from, * from, * to_next;
1044   snat_out2in_next_t next_index;
1045   u32 pkts_processed = 0;
1046   snat_main_t * sm = &snat_main;
1047   f64 now = vlib_time_now (vm);
1048   u32 thread_index = vlib_get_thread_index ();
1049
1050   from = vlib_frame_vector_args (frame);
1051   n_left_from = frame->n_vectors;
1052   next_index = node->cached_next_index;
1053
1054   while (n_left_from > 0)
1055     {
1056       u32 n_left_to_next;
1057
1058       vlib_get_next_frame (vm, node, next_index,
1059                            to_next, n_left_to_next);
1060
1061       while (n_left_from >= 4 && n_left_to_next >= 2)
1062         {
1063           u32 bi0, bi1;
1064           vlib_buffer_t * b0, * b1;
1065           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1066           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1067           u32 sw_if_index0, sw_if_index1;
1068           ip4_header_t * ip0, *ip1;
1069           ip_csum_t sum0, sum1;
1070           u32 new_addr0, old_addr0;
1071           u16 new_port0, old_port0;
1072           u32 new_addr1, old_addr1;
1073           u16 new_port1, old_port1;
1074           udp_header_t * udp0, * udp1;
1075           tcp_header_t * tcp0, * tcp1;
1076           icmp46_header_t * icmp0, * icmp1;
1077           snat_session_key_t key0, key1, sm0, sm1;
1078           u32 rx_fib_index0, rx_fib_index1;
1079           u32 proto0, proto1;
1080           snat_session_t * s0 = 0, * s1 = 0;
1081           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1082
1083           /* Prefetch next iteration. */
1084           {
1085             vlib_buffer_t * p2, * p3;
1086
1087             p2 = vlib_get_buffer (vm, from[2]);
1088             p3 = vlib_get_buffer (vm, from[3]);
1089
1090             vlib_prefetch_buffer_header (p2, LOAD);
1091             vlib_prefetch_buffer_header (p3, LOAD);
1092
1093             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1094             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1095           }
1096
1097           /* speculatively enqueue b0 and b1 to the current next frame */
1098           to_next[0] = bi0 = from[0];
1099           to_next[1] = bi1 = from[1];
1100           from += 2;
1101           to_next += 2;
1102           n_left_from -= 2;
1103           n_left_to_next -= 2;
1104
1105           b0 = vlib_get_buffer (vm, bi0);
1106           b1 = vlib_get_buffer (vm, bi1);
1107
1108           vnet_buffer (b0)->snat.flags = 0;
1109           vnet_buffer (b1)->snat.flags = 0;
1110
1111           ip0 = vlib_buffer_get_current (b0);
1112           udp0 = ip4_next_header (ip0);
1113           tcp0 = (tcp_header_t *) udp0;
1114           icmp0 = (icmp46_header_t *) udp0;
1115
1116           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1117           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1118                                    sw_if_index0);
1119
1120           if (PREDICT_FALSE(ip0->ttl == 1))
1121             {
1122               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1123               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1124                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1125                                            0);
1126               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1127               goto trace0;
1128             }
1129
1130           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1131
1132           if (PREDICT_FALSE (proto0 == ~0))
1133             {
1134               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1135                                              thread_index, now, vm, node);
1136               if (!sm->forwarding_enabled)
1137                 if (!s0)
1138                   next0 = SNAT_OUT2IN_NEXT_DROP;
1139               goto trace0;
1140             }
1141
1142           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1143             {
1144               next0 = icmp_out2in_slow_path
1145                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1146                  next0, now, thread_index, &s0);
1147               goto trace0;
1148             }
1149
1150           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1151             {
1152               next0 = SNAT_OUT2IN_NEXT_REASS;
1153               goto trace0;
1154             }
1155
1156           key0.addr = ip0->dst_address;
1157           key0.port = udp0->dst_port;
1158           key0.protocol = proto0;
1159           key0.fib_index = rx_fib_index0;
1160
1161           kv0.key = key0.as_u64;
1162
1163           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1164                                       &kv0, &value0))
1165             {
1166               /* Try to match static mapping by external address and port,
1167                  destination address and port in packet */
1168               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1169                 {
1170                   /*
1171                    * Send DHCP packets to the ipv4 stack, or we won't
1172                    * be able to use dhcp client on the outside interface
1173                    */
1174                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1175                       && (udp0->dst_port ==
1176                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1177                     {
1178                       vnet_feature_next
1179                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1180                       goto trace0;
1181                     }
1182
1183                   if (!sm->forwarding_enabled)
1184                     {
1185                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1186                       next0 = SNAT_OUT2IN_NEXT_DROP;
1187                       goto trace0;
1188                     }
1189                   else
1190                     {
1191                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1192                         {
1193                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1194                           goto trace0;
1195                         }
1196                       create_bypass_for_fwd(sm, ip0);
1197                       goto trace0;
1198                     }
1199                 }
1200
1201               /* Create session initiated by host from external network */
1202               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1203                                                      thread_index);
1204               if (!s0)
1205                 {
1206                   next0 = SNAT_OUT2IN_NEXT_DROP;
1207                   goto trace0;
1208                 }
1209             }
1210           else
1211             {
1212               if (PREDICT_FALSE (value0.value == ~0ULL))
1213                 {
1214                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1215                                       now, vm, node);
1216                   if (!s0)
1217                     next0 = SNAT_OUT2IN_NEXT_DROP;
1218                   goto trace0;
1219                 }
1220               else
1221                 {
1222                   s0 = pool_elt_at_index (
1223                     sm->per_thread_data[thread_index].sessions,
1224                     value0.value);
1225                 }
1226             }
1227
1228           old_addr0 = ip0->dst_address.as_u32;
1229           ip0->dst_address = s0->in2out.addr;
1230           new_addr0 = ip0->dst_address.as_u32;
1231           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1232
1233           sum0 = ip0->checksum;
1234           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1235                                  ip4_header_t,
1236                                  dst_address /* changed member */);
1237           ip0->checksum = ip_csum_fold (sum0);
1238
1239           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1240             {
1241               old_port0 = tcp0->dst_port;
1242               tcp0->dst_port = s0->in2out.port;
1243               new_port0 = tcp0->dst_port;
1244
1245               sum0 = tcp0->checksum;
1246               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1247                                      ip4_header_t,
1248                                      dst_address /* changed member */);
1249
1250               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1251                                      ip4_header_t /* cheat */,
1252                                      length /* changed member */);
1253               tcp0->checksum = ip_csum_fold(sum0);
1254             }
1255           else
1256             {
1257               old_port0 = udp0->dst_port;
1258               udp0->dst_port = s0->in2out.port;
1259               udp0->checksum = 0;
1260             }
1261
1262           /* Accounting */
1263           s0->last_heard = now;
1264           s0->total_pkts++;
1265           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1266           /* Per-user LRU list maintenance */
1267           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1268                              s0->per_user_index);
1269           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1270                               s0->per_user_list_head_index,
1271                               s0->per_user_index);
1272         trace0:
1273
1274           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1275                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1276             {
1277               snat_out2in_trace_t *t =
1278                  vlib_add_trace (vm, node, b0, sizeof (*t));
1279               t->sw_if_index = sw_if_index0;
1280               t->next_index = next0;
1281               t->session_index = ~0;
1282               if (s0)
1283                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1284             }
1285
1286           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1287
1288
1289           ip1 = vlib_buffer_get_current (b1);
1290           udp1 = ip4_next_header (ip1);
1291           tcp1 = (tcp_header_t *) udp1;
1292           icmp1 = (icmp46_header_t *) udp1;
1293
1294           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1295           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1296                                    sw_if_index1);
1297
1298           if (PREDICT_FALSE(ip1->ttl == 1))
1299             {
1300               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1301               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1302                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1303                                            0);
1304               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1305               goto trace1;
1306             }
1307
1308           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1309
1310           if (PREDICT_FALSE (proto1 == ~0))
1311             {
1312               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1313                                              thread_index, now, vm, node);
1314               if (!sm->forwarding_enabled)
1315                 if (!s1)
1316                   next1 = SNAT_OUT2IN_NEXT_DROP;
1317               goto trace1;
1318             }
1319
1320           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1321             {
1322               next1 = icmp_out2in_slow_path
1323                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1324                  next1, now, thread_index, &s1);
1325               goto trace1;
1326             }
1327
1328           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1329             {
1330               next1 = SNAT_OUT2IN_NEXT_REASS;
1331               goto trace1;
1332             }
1333
1334           key1.addr = ip1->dst_address;
1335           key1.port = udp1->dst_port;
1336           key1.protocol = proto1;
1337           key1.fib_index = rx_fib_index1;
1338
1339           kv1.key = key1.as_u64;
1340
1341           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1342                                       &kv1, &value1))
1343             {
1344               /* Try to match static mapping by external address and port,
1345                  destination address and port in packet */
1346               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1347                 {
1348                   /*
1349                    * Send DHCP packets to the ipv4 stack, or we won't
1350                    * be able to use dhcp client on the outside interface
1351                    */
1352                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1353                       && (udp1->dst_port ==
1354                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1355                     {
1356                       vnet_feature_next
1357                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1358                       goto trace1;
1359                     }
1360
1361                   if (!sm->forwarding_enabled)
1362                     {
1363                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1364                       next1 = SNAT_OUT2IN_NEXT_DROP;
1365                       goto trace1;
1366                     }
1367                   else
1368                     {
1369                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1370                         {
1371                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1372                           goto trace1;
1373                         }
1374                       create_bypass_for_fwd(sm, ip1);
1375                       goto trace1;
1376                     }
1377                 }
1378
1379               /* Create session initiated by host from external network */
1380               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1381                                                      thread_index);
1382               if (!s1)
1383                 {
1384                   next1 = SNAT_OUT2IN_NEXT_DROP;
1385                   goto trace1;
1386                 }
1387             }
1388           else
1389             {
1390               if (PREDICT_FALSE (value1.value == ~0ULL))
1391                 {
1392                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1393                                       now, vm, node);
1394                   if (!s1)
1395                     next1 = SNAT_OUT2IN_NEXT_DROP;
1396                   goto trace1;
1397                 }
1398               else
1399                 {
1400                   s1 = pool_elt_at_index (
1401                     sm->per_thread_data[thread_index].sessions,
1402                     value1.value);
1403                 }
1404             }
1405
1406           old_addr1 = ip1->dst_address.as_u32;
1407           ip1->dst_address = s1->in2out.addr;
1408           new_addr1 = ip1->dst_address.as_u32;
1409           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1410
1411           sum1 = ip1->checksum;
1412           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1413                                  ip4_header_t,
1414                                  dst_address /* changed member */);
1415           ip1->checksum = ip_csum_fold (sum1);
1416
1417           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1418             {
1419               old_port1 = tcp1->dst_port;
1420               tcp1->dst_port = s1->in2out.port;
1421               new_port1 = tcp1->dst_port;
1422
1423               sum1 = tcp1->checksum;
1424               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1425                                      ip4_header_t,
1426                                      dst_address /* changed member */);
1427
1428               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1429                                      ip4_header_t /* cheat */,
1430                                      length /* changed member */);
1431               tcp1->checksum = ip_csum_fold(sum1);
1432             }
1433           else
1434             {
1435               old_port1 = udp1->dst_port;
1436               udp1->dst_port = s1->in2out.port;
1437               udp1->checksum = 0;
1438             }
1439
1440           /* Accounting */
1441           s1->last_heard = now;
1442           s1->total_pkts++;
1443           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1444           /* Per-user LRU list maintenance */
1445           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1446                              s1->per_user_index);
1447           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1448                               s1->per_user_list_head_index,
1449                               s1->per_user_index);
1450         trace1:
1451
1452           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1453                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1454             {
1455               snat_out2in_trace_t *t =
1456                  vlib_add_trace (vm, node, b1, sizeof (*t));
1457               t->sw_if_index = sw_if_index1;
1458               t->next_index = next1;
1459               t->session_index = ~0;
1460               if (s1)
1461                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1462             }
1463
1464           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1465
1466           /* verify speculative enqueues, maybe switch current next frame */
1467           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1468                                            to_next, n_left_to_next,
1469                                            bi0, bi1, next0, next1);
1470         }
1471
1472       while (n_left_from > 0 && n_left_to_next > 0)
1473         {
1474           u32 bi0;
1475           vlib_buffer_t * b0;
1476           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1477           u32 sw_if_index0;
1478           ip4_header_t * ip0;
1479           ip_csum_t sum0;
1480           u32 new_addr0, old_addr0;
1481           u16 new_port0, old_port0;
1482           udp_header_t * udp0;
1483           tcp_header_t * tcp0;
1484           icmp46_header_t * icmp0;
1485           snat_session_key_t key0, sm0;
1486           u32 rx_fib_index0;
1487           u32 proto0;
1488           snat_session_t * s0 = 0;
1489           clib_bihash_kv_8_8_t kv0, value0;
1490
1491           /* speculatively enqueue b0 to the current next frame */
1492           bi0 = from[0];
1493           to_next[0] = bi0;
1494           from += 1;
1495           to_next += 1;
1496           n_left_from -= 1;
1497           n_left_to_next -= 1;
1498
1499           b0 = vlib_get_buffer (vm, bi0);
1500
1501           vnet_buffer (b0)->snat.flags = 0;
1502
1503           ip0 = vlib_buffer_get_current (b0);
1504           udp0 = ip4_next_header (ip0);
1505           tcp0 = (tcp_header_t *) udp0;
1506           icmp0 = (icmp46_header_t *) udp0;
1507
1508           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1509           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1510                                    sw_if_index0);
1511
1512           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1513
1514           if (PREDICT_FALSE (proto0 == ~0))
1515             {
1516               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1517                                              thread_index, now, vm, node);
1518               if (!sm->forwarding_enabled)
1519                 if (!s0)
1520                   next0 = SNAT_OUT2IN_NEXT_DROP;
1521               goto trace00;
1522             }
1523
1524           if (PREDICT_FALSE(ip0->ttl == 1))
1525             {
1526               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1527               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1528                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1529                                            0);
1530               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1531               goto trace00;
1532             }
1533
1534           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1535             {
1536               next0 = icmp_out2in_slow_path
1537                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1538                  next0, now, thread_index, &s0);
1539               goto trace00;
1540             }
1541
1542           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1543             {
1544               next0 = SNAT_OUT2IN_NEXT_REASS;
1545               goto trace00;
1546             }
1547
1548           key0.addr = ip0->dst_address;
1549           key0.port = udp0->dst_port;
1550           key0.protocol = proto0;
1551           key0.fib_index = rx_fib_index0;
1552
1553           kv0.key = key0.as_u64;
1554
1555           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1556                                       &kv0, &value0))
1557             {
1558               /* Try to match static mapping by external address and port,
1559                  destination address and port in packet */
1560               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1561                 {
1562                   /*
1563                    * Send DHCP packets to the ipv4 stack, or we won't
1564                    * be able to use dhcp client on the outside interface
1565                    */
1566                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1567                       && (udp0->dst_port ==
1568                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1569                     {
1570                       vnet_feature_next
1571                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1572                       goto trace00;
1573                     }
1574
1575                   if (!sm->forwarding_enabled)
1576                     {
1577                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1578                       next0 = SNAT_OUT2IN_NEXT_DROP;
1579                       goto trace00;
1580                     }
1581                   else
1582                     {
1583                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1584                         {
1585                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1586                           goto trace00;
1587                         }
1588                       create_bypass_for_fwd(sm, ip0);
1589                       goto trace00;
1590                     }
1591                 }
1592
1593               /* Create session initiated by host from external network */
1594               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1595                                                      thread_index);
1596               if (!s0)
1597                 {
1598                   next0 = SNAT_OUT2IN_NEXT_DROP;
1599                   goto trace00;
1600                 }
1601             }
1602           else
1603             {
1604               if (PREDICT_FALSE (value0.value == ~0ULL))
1605                 {
1606                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1607                                       now, vm, node);
1608                   if (!s0)
1609                     next0 = SNAT_OUT2IN_NEXT_DROP;
1610                   goto trace00;
1611                 }
1612               else
1613                 {
1614                   s0 = pool_elt_at_index (
1615                     sm->per_thread_data[thread_index].sessions,
1616                     value0.value);
1617                 }
1618             }
1619
1620           old_addr0 = ip0->dst_address.as_u32;
1621           ip0->dst_address = s0->in2out.addr;
1622           new_addr0 = ip0->dst_address.as_u32;
1623           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1624
1625           sum0 = ip0->checksum;
1626           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1627                                  ip4_header_t,
1628                                  dst_address /* changed member */);
1629           ip0->checksum = ip_csum_fold (sum0);
1630
1631           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1632             {
1633               old_port0 = tcp0->dst_port;
1634               tcp0->dst_port = s0->in2out.port;
1635               new_port0 = tcp0->dst_port;
1636
1637               sum0 = tcp0->checksum;
1638               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1639                                      ip4_header_t,
1640                                      dst_address /* changed member */);
1641
1642               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1643                                      ip4_header_t /* cheat */,
1644                                      length /* changed member */);
1645               tcp0->checksum = ip_csum_fold(sum0);
1646             }
1647           else
1648             {
1649               old_port0 = udp0->dst_port;
1650               udp0->dst_port = s0->in2out.port;
1651               udp0->checksum = 0;
1652             }
1653
1654           /* Accounting */
1655           s0->last_heard = now;
1656           s0->total_pkts++;
1657           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1658           /* Per-user LRU list maintenance */
1659           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1660                              s0->per_user_index);
1661           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1662                               s0->per_user_list_head_index,
1663                               s0->per_user_index);
1664         trace00:
1665
1666           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1667                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1668             {
1669               snat_out2in_trace_t *t =
1670                  vlib_add_trace (vm, node, b0, sizeof (*t));
1671               t->sw_if_index = sw_if_index0;
1672               t->next_index = next0;
1673               t->session_index = ~0;
1674               if (s0)
1675                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1676             }
1677
1678           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1679
1680           /* verify speculative enqueue, maybe switch current next frame */
1681           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1682                                            to_next, n_left_to_next,
1683                                            bi0, next0);
1684         }
1685
1686       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1687     }
1688
1689   vlib_node_increment_counter (vm, snat_out2in_node.index,
1690                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1691                                pkts_processed);
1692   return frame->n_vectors;
1693 }
1694
1695 VLIB_REGISTER_NODE (snat_out2in_node) = {
1696   .function = snat_out2in_node_fn,
1697   .name = "nat44-out2in",
1698   .vector_size = sizeof (u32),
1699   .format_trace = format_snat_out2in_trace,
1700   .type = VLIB_NODE_TYPE_INTERNAL,
1701
1702   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1703   .error_strings = snat_out2in_error_strings,
1704
1705   .runtime_data_bytes = sizeof (snat_runtime_t),
1706
1707   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1708
1709   /* edit / add dispositions here */
1710   .next_nodes = {
1711     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1712     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1713     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1714     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1715     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1716   },
1717 };
1718 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1719
1720 static uword
1721 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1722                             vlib_node_runtime_t * node,
1723                             vlib_frame_t * frame)
1724 {
1725   u32 n_left_from, *from, *to_next;
1726   snat_out2in_next_t next_index;
1727   u32 pkts_processed = 0;
1728   snat_main_t *sm = &snat_main;
1729   f64 now = vlib_time_now (vm);
1730   u32 thread_index = vlib_get_thread_index ();
1731   snat_main_per_thread_data_t *per_thread_data =
1732     &sm->per_thread_data[thread_index];
1733   u32 *fragments_to_drop = 0;
1734   u32 *fragments_to_loopback = 0;
1735
1736   from = vlib_frame_vector_args (frame);
1737   n_left_from = frame->n_vectors;
1738   next_index = node->cached_next_index;
1739
1740   while (n_left_from > 0)
1741     {
1742       u32 n_left_to_next;
1743
1744       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1745
1746       while (n_left_from > 0 && n_left_to_next > 0)
1747        {
1748           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1749           vlib_buffer_t *b0;
1750           u32 next0;
1751           u8 cached0 = 0;
1752           ip4_header_t *ip0;
1753           nat_reass_ip4_t *reass0;
1754           udp_header_t * udp0;
1755           tcp_header_t * tcp0;
1756           snat_session_key_t key0, sm0;
1757           clib_bihash_kv_8_8_t kv0, value0;
1758           snat_session_t * s0 = 0;
1759           u16 old_port0, new_port0;
1760           ip_csum_t sum0;
1761
1762           /* speculatively enqueue b0 to the current next frame */
1763           bi0 = from[0];
1764           to_next[0] = bi0;
1765           from += 1;
1766           to_next += 1;
1767           n_left_from -= 1;
1768           n_left_to_next -= 1;
1769
1770           b0 = vlib_get_buffer (vm, bi0);
1771           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1772
1773           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1774           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1775                                                                sw_if_index0);
1776
1777           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1778             {
1779               next0 = SNAT_OUT2IN_NEXT_DROP;
1780               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1781               goto trace0;
1782             }
1783
1784           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1785           udp0 = ip4_next_header (ip0);
1786           tcp0 = (tcp_header_t *) udp0;
1787           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1788
1789           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1790                                                  ip0->dst_address,
1791                                                  ip0->fragment_id,
1792                                                  ip0->protocol,
1793                                                  1,
1794                                                  &fragments_to_drop);
1795
1796           if (PREDICT_FALSE (!reass0))
1797             {
1798               next0 = SNAT_OUT2IN_NEXT_DROP;
1799               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1800               goto trace0;
1801             }
1802
1803           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1804             {
1805               key0.addr = ip0->dst_address;
1806               key0.port = udp0->dst_port;
1807               key0.protocol = proto0;
1808               key0.fib_index = rx_fib_index0;
1809               kv0.key = key0.as_u64;
1810
1811               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1812                 {
1813                   /* Try to match static mapping by external address and port,
1814                      destination address and port in packet */
1815                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1816                     {
1817                       /*
1818                        * Send DHCP packets to the ipv4 stack, or we won't
1819                        * be able to use dhcp client on the outside interface
1820                        */
1821                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1822                           && (udp0->dst_port
1823                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1824                         {
1825                           vnet_feature_next
1826                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1827                              &next0, b0);
1828                           goto trace0;
1829                         }
1830
1831                       if (!sm->forwarding_enabled)
1832                         {
1833                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1834                           next0 = SNAT_OUT2IN_NEXT_DROP;
1835                           goto trace0;
1836                         }
1837                       else
1838                         {
1839                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1840                             {
1841                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1842                               goto trace0;
1843                             }
1844                           create_bypass_for_fwd(sm, ip0);
1845                           goto trace0;
1846                         }
1847                     }
1848
1849                   /* Create session initiated by host from external network */
1850                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1851                                                          thread_index);
1852                   if (!s0)
1853                     {
1854                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1855                       next0 = SNAT_OUT2IN_NEXT_DROP;
1856                       goto trace0;
1857                     }
1858                   reass0->sess_index = s0 - per_thread_data->sessions;
1859                   reass0->thread_index = thread_index;
1860                 }
1861               else
1862                 {
1863                   s0 = pool_elt_at_index (per_thread_data->sessions,
1864                                           value0.value);
1865                   reass0->sess_index = value0.value;
1866                 }
1867               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1868             }
1869           else
1870             {
1871               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1872                 {
1873                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1874                     {
1875                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1876                       next0 = SNAT_OUT2IN_NEXT_DROP;
1877                       goto trace0;
1878                     }
1879                   cached0 = 1;
1880                   goto trace0;
1881                 }
1882               s0 = pool_elt_at_index (per_thread_data->sessions,
1883                                       reass0->sess_index);
1884             }
1885
1886           old_addr0 = ip0->dst_address.as_u32;
1887           ip0->dst_address = s0->in2out.addr;
1888           new_addr0 = ip0->dst_address.as_u32;
1889           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1890
1891           sum0 = ip0->checksum;
1892           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1893                                  ip4_header_t,
1894                                  dst_address /* changed member */);
1895           ip0->checksum = ip_csum_fold (sum0);
1896
1897           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1898             {
1899               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1900                 {
1901                   old_port0 = tcp0->dst_port;
1902                   tcp0->dst_port = s0->in2out.port;
1903                   new_port0 = tcp0->dst_port;
1904
1905                   sum0 = tcp0->checksum;
1906                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1907                                          ip4_header_t,
1908                                          dst_address /* changed member */);
1909
1910                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1911                                          ip4_header_t /* cheat */,
1912                                          length /* changed member */);
1913                   tcp0->checksum = ip_csum_fold(sum0);
1914                 }
1915               else
1916                 {
1917                   old_port0 = udp0->dst_port;
1918                   udp0->dst_port = s0->in2out.port;
1919                   udp0->checksum = 0;
1920                 }
1921             }
1922
1923           /* Accounting */
1924           s0->last_heard = now;
1925           s0->total_pkts++;
1926           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1927           /* Per-user LRU list maintenance */
1928           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1929                              s0->per_user_index);
1930           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1931                               s0->per_user_list_head_index,
1932                               s0->per_user_index);
1933
1934         trace0:
1935           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1936                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1937             {
1938               nat44_out2in_reass_trace_t *t =
1939                  vlib_add_trace (vm, node, b0, sizeof (*t));
1940               t->cached = cached0;
1941               t->sw_if_index = sw_if_index0;
1942               t->next_index = next0;
1943             }
1944
1945           if (cached0)
1946             {
1947               n_left_to_next++;
1948               to_next--;
1949             }
1950           else
1951             {
1952               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1953
1954               /* verify speculative enqueue, maybe switch current next frame */
1955               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1956                                                to_next, n_left_to_next,
1957                                                bi0, next0);
1958             }
1959
1960           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1961             {
1962               from = vlib_frame_vector_args (frame);
1963               u32 len = vec_len (fragments_to_loopback);
1964               if (len <= VLIB_FRAME_SIZE)
1965                 {
1966                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1967                   n_left_from = len;
1968                   vec_reset_length (fragments_to_loopback);
1969                 }
1970               else
1971                 {
1972                   clib_memcpy (from,
1973                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1974                                sizeof (u32) * VLIB_FRAME_SIZE);
1975                   n_left_from = VLIB_FRAME_SIZE;
1976                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1977                 }
1978             }
1979        }
1980
1981       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1982     }
1983
1984   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1985                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1986                                pkts_processed);
1987
1988   nat_send_all_to_node (vm, fragments_to_drop, node,
1989                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1990                         SNAT_OUT2IN_NEXT_DROP);
1991
1992   vec_free (fragments_to_drop);
1993   vec_free (fragments_to_loopback);
1994   return frame->n_vectors;
1995 }
1996
1997 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1998   .function = nat44_out2in_reass_node_fn,
1999   .name = "nat44-out2in-reass",
2000   .vector_size = sizeof (u32),
2001   .format_trace = format_nat44_out2in_reass_trace,
2002   .type = VLIB_NODE_TYPE_INTERNAL,
2003
2004   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2005   .error_strings = snat_out2in_error_strings,
2006
2007   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2008
2009   /* edit / add dispositions here */
2010   .next_nodes = {
2011     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2012     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2013     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2014     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2015     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2016   },
2017 };
2018 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2019                               nat44_out2in_reass_node_fn);
2020
2021 /**************************/
2022 /*** deterministic mode ***/
2023 /**************************/
2024 static uword
2025 snat_det_out2in_node_fn (vlib_main_t * vm,
2026                          vlib_node_runtime_t * node,
2027                          vlib_frame_t * frame)
2028 {
2029   u32 n_left_from, * from, * to_next;
2030   snat_out2in_next_t next_index;
2031   u32 pkts_processed = 0;
2032   snat_main_t * sm = &snat_main;
2033   u32 thread_index = vlib_get_thread_index ();
2034
2035   from = vlib_frame_vector_args (frame);
2036   n_left_from = frame->n_vectors;
2037   next_index = node->cached_next_index;
2038
2039   while (n_left_from > 0)
2040     {
2041       u32 n_left_to_next;
2042
2043       vlib_get_next_frame (vm, node, next_index,
2044                            to_next, n_left_to_next);
2045
2046       while (n_left_from >= 4 && n_left_to_next >= 2)
2047         {
2048           u32 bi0, bi1;
2049           vlib_buffer_t * b0, * b1;
2050           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2051           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2052           u32 sw_if_index0, sw_if_index1;
2053           ip4_header_t * ip0, * ip1;
2054           ip_csum_t sum0, sum1;
2055           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2056           u16 new_port0, old_port0, old_port1, new_port1;
2057           udp_header_t * udp0, * udp1;
2058           tcp_header_t * tcp0, * tcp1;
2059           u32 proto0, proto1;
2060           snat_det_out_key_t key0, key1;
2061           snat_det_map_t * dm0, * dm1;
2062           snat_det_session_t * ses0 = 0, * ses1 = 0;
2063           u32 rx_fib_index0, rx_fib_index1;
2064           icmp46_header_t * icmp0, * icmp1;
2065
2066           /* Prefetch next iteration. */
2067           {
2068             vlib_buffer_t * p2, * p3;
2069
2070             p2 = vlib_get_buffer (vm, from[2]);
2071             p3 = vlib_get_buffer (vm, from[3]);
2072
2073             vlib_prefetch_buffer_header (p2, LOAD);
2074             vlib_prefetch_buffer_header (p3, LOAD);
2075
2076             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2077             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2078           }
2079
2080           /* speculatively enqueue b0 and b1 to the current next frame */
2081           to_next[0] = bi0 = from[0];
2082           to_next[1] = bi1 = from[1];
2083           from += 2;
2084           to_next += 2;
2085           n_left_from -= 2;
2086           n_left_to_next -= 2;
2087
2088           b0 = vlib_get_buffer (vm, bi0);
2089           b1 = vlib_get_buffer (vm, bi1);
2090
2091           ip0 = vlib_buffer_get_current (b0);
2092           udp0 = ip4_next_header (ip0);
2093           tcp0 = (tcp_header_t *) udp0;
2094
2095           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2096
2097           if (PREDICT_FALSE(ip0->ttl == 1))
2098             {
2099               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2100               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2101                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2102                                            0);
2103               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2104               goto trace0;
2105             }
2106
2107           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2108
2109           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2110             {
2111               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2112               icmp0 = (icmp46_header_t *) udp0;
2113
2114               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2115                                   rx_fib_index0, node, next0, thread_index,
2116                                   &ses0, &dm0);
2117               goto trace0;
2118             }
2119
2120           key0.ext_host_addr = ip0->src_address;
2121           key0.ext_host_port = tcp0->src;
2122           key0.out_port = tcp0->dst;
2123
2124           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2125           if (PREDICT_FALSE(!dm0))
2126             {
2127               clib_warning("unknown dst address:  %U",
2128                            format_ip4_address, &ip0->dst_address);
2129               next0 = SNAT_OUT2IN_NEXT_DROP;
2130               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2131               goto trace0;
2132             }
2133
2134           snat_det_reverse(dm0, &ip0->dst_address,
2135                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2136
2137           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2138           if (PREDICT_FALSE(!ses0))
2139             {
2140               clib_warning("no match src %U:%d dst %U:%d for user %U",
2141                            format_ip4_address, &ip0->src_address,
2142                            clib_net_to_host_u16 (tcp0->src),
2143                            format_ip4_address, &ip0->dst_address,
2144                            clib_net_to_host_u16 (tcp0->dst),
2145                            format_ip4_address, &new_addr0);
2146               next0 = SNAT_OUT2IN_NEXT_DROP;
2147               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2148               goto trace0;
2149             }
2150           new_port0 = ses0->in_port;
2151
2152           old_addr0 = ip0->dst_address;
2153           ip0->dst_address = new_addr0;
2154           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2155
2156           sum0 = ip0->checksum;
2157           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2158                                  ip4_header_t,
2159                                  dst_address /* changed member */);
2160           ip0->checksum = ip_csum_fold (sum0);
2161
2162           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2163             {
2164               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2165                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2166               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2167                 snat_det_ses_close(dm0, ses0);
2168
2169               old_port0 = tcp0->dst;
2170               tcp0->dst = new_port0;
2171
2172               sum0 = tcp0->checksum;
2173               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2174                                      ip4_header_t,
2175                                      dst_address /* changed member */);
2176
2177               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2178                                      ip4_header_t /* cheat */,
2179                                      length /* changed member */);
2180               tcp0->checksum = ip_csum_fold(sum0);
2181             }
2182           else
2183             {
2184               old_port0 = udp0->dst_port;
2185               udp0->dst_port = new_port0;
2186               udp0->checksum = 0;
2187             }
2188
2189         trace0:
2190
2191           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2192                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2193             {
2194               snat_out2in_trace_t *t =
2195                  vlib_add_trace (vm, node, b0, sizeof (*t));
2196               t->sw_if_index = sw_if_index0;
2197               t->next_index = next0;
2198               t->session_index = ~0;
2199               if (ses0)
2200                 t->session_index = ses0 - dm0->sessions;
2201             }
2202
2203           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2204
2205           b1 = vlib_get_buffer (vm, bi1);
2206
2207           ip1 = vlib_buffer_get_current (b1);
2208           udp1 = ip4_next_header (ip1);
2209           tcp1 = (tcp_header_t *) udp1;
2210
2211           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2212
2213           if (PREDICT_FALSE(ip1->ttl == 1))
2214             {
2215               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2216               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2217                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2218                                            0);
2219               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2220               goto trace1;
2221             }
2222
2223           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2224
2225           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2226             {
2227               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2228               icmp1 = (icmp46_header_t *) udp1;
2229
2230               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2231                                   rx_fib_index1, node, next1, thread_index,
2232                                   &ses1, &dm1);
2233               goto trace1;
2234             }
2235
2236           key1.ext_host_addr = ip1->src_address;
2237           key1.ext_host_port = tcp1->src;
2238           key1.out_port = tcp1->dst;
2239
2240           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2241           if (PREDICT_FALSE(!dm1))
2242             {
2243               clib_warning("unknown dst address:  %U",
2244                            format_ip4_address, &ip1->dst_address);
2245               next1 = SNAT_OUT2IN_NEXT_DROP;
2246               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2247               goto trace1;
2248             }
2249
2250           snat_det_reverse(dm1, &ip1->dst_address,
2251                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2252
2253           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2254           if (PREDICT_FALSE(!ses1))
2255             {
2256               clib_warning("no match src %U:%d dst %U:%d for user %U",
2257                            format_ip4_address, &ip1->src_address,
2258                            clib_net_to_host_u16 (tcp1->src),
2259                            format_ip4_address, &ip1->dst_address,
2260                            clib_net_to_host_u16 (tcp1->dst),
2261                            format_ip4_address, &new_addr1);
2262               next1 = SNAT_OUT2IN_NEXT_DROP;
2263               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2264               goto trace1;
2265             }
2266           new_port1 = ses1->in_port;
2267
2268           old_addr1 = ip1->dst_address;
2269           ip1->dst_address = new_addr1;
2270           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2271
2272           sum1 = ip1->checksum;
2273           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2274                                  ip4_header_t,
2275                                  dst_address /* changed member */);
2276           ip1->checksum = ip_csum_fold (sum1);
2277
2278           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2279             {
2280               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2281                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2282               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2283                 snat_det_ses_close(dm1, ses1);
2284
2285               old_port1 = tcp1->dst;
2286               tcp1->dst = new_port1;
2287
2288               sum1 = tcp1->checksum;
2289               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2290                                      ip4_header_t,
2291                                      dst_address /* changed member */);
2292
2293               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2294                                      ip4_header_t /* cheat */,
2295                                      length /* changed member */);
2296               tcp1->checksum = ip_csum_fold(sum1);
2297             }
2298           else
2299             {
2300               old_port1 = udp1->dst_port;
2301               udp1->dst_port = new_port1;
2302               udp1->checksum = 0;
2303             }
2304
2305         trace1:
2306
2307           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2308                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2309             {
2310               snat_out2in_trace_t *t =
2311                  vlib_add_trace (vm, node, b1, sizeof (*t));
2312               t->sw_if_index = sw_if_index1;
2313               t->next_index = next1;
2314               t->session_index = ~0;
2315               if (ses1)
2316                 t->session_index = ses1 - dm1->sessions;
2317             }
2318
2319           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2320
2321           /* verify speculative enqueues, maybe switch current next frame */
2322           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2323                                            to_next, n_left_to_next,
2324                                            bi0, bi1, next0, next1);
2325          }
2326
2327       while (n_left_from > 0 && n_left_to_next > 0)
2328         {
2329           u32 bi0;
2330           vlib_buffer_t * b0;
2331           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2332           u32 sw_if_index0;
2333           ip4_header_t * ip0;
2334           ip_csum_t sum0;
2335           ip4_address_t new_addr0, old_addr0;
2336           u16 new_port0, old_port0;
2337           udp_header_t * udp0;
2338           tcp_header_t * tcp0;
2339           u32 proto0;
2340           snat_det_out_key_t key0;
2341           snat_det_map_t * dm0;
2342           snat_det_session_t * ses0 = 0;
2343           u32 rx_fib_index0;
2344           icmp46_header_t * icmp0;
2345
2346           /* speculatively enqueue b0 to the current next frame */
2347           bi0 = from[0];
2348           to_next[0] = bi0;
2349           from += 1;
2350           to_next += 1;
2351           n_left_from -= 1;
2352           n_left_to_next -= 1;
2353
2354           b0 = vlib_get_buffer (vm, bi0);
2355
2356           ip0 = vlib_buffer_get_current (b0);
2357           udp0 = ip4_next_header (ip0);
2358           tcp0 = (tcp_header_t *) udp0;
2359
2360           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2361
2362           if (PREDICT_FALSE(ip0->ttl == 1))
2363             {
2364               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2365               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2366                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2367                                            0);
2368               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2369               goto trace00;
2370             }
2371
2372           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2373
2374           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2375             {
2376               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2377               icmp0 = (icmp46_header_t *) udp0;
2378
2379               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2380                                   rx_fib_index0, node, next0, thread_index,
2381                                   &ses0, &dm0);
2382               goto trace00;
2383             }
2384
2385           key0.ext_host_addr = ip0->src_address;
2386           key0.ext_host_port = tcp0->src;
2387           key0.out_port = tcp0->dst;
2388
2389           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2390           if (PREDICT_FALSE(!dm0))
2391             {
2392               clib_warning("unknown dst address:  %U",
2393                            format_ip4_address, &ip0->dst_address);
2394               next0 = SNAT_OUT2IN_NEXT_DROP;
2395               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2396               goto trace00;
2397             }
2398
2399           snat_det_reverse(dm0, &ip0->dst_address,
2400                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2401
2402           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2403           if (PREDICT_FALSE(!ses0))
2404             {
2405               clib_warning("no match src %U:%d dst %U:%d for user %U",
2406                            format_ip4_address, &ip0->src_address,
2407                            clib_net_to_host_u16 (tcp0->src),
2408                            format_ip4_address, &ip0->dst_address,
2409                            clib_net_to_host_u16 (tcp0->dst),
2410                            format_ip4_address, &new_addr0);
2411               next0 = SNAT_OUT2IN_NEXT_DROP;
2412               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2413               goto trace00;
2414             }
2415           new_port0 = ses0->in_port;
2416
2417           old_addr0 = ip0->dst_address;
2418           ip0->dst_address = new_addr0;
2419           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2420
2421           sum0 = ip0->checksum;
2422           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2423                                  ip4_header_t,
2424                                  dst_address /* changed member */);
2425           ip0->checksum = ip_csum_fold (sum0);
2426
2427           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2428             {
2429               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2430                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2431               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2432                 snat_det_ses_close(dm0, ses0);
2433
2434               old_port0 = tcp0->dst;
2435               tcp0->dst = new_port0;
2436
2437               sum0 = tcp0->checksum;
2438               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2439                                      ip4_header_t,
2440                                      dst_address /* changed member */);
2441
2442               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2443                                      ip4_header_t /* cheat */,
2444                                      length /* changed member */);
2445               tcp0->checksum = ip_csum_fold(sum0);
2446             }
2447           else
2448             {
2449               old_port0 = udp0->dst_port;
2450               udp0->dst_port = new_port0;
2451               udp0->checksum = 0;
2452             }
2453
2454         trace00:
2455
2456           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2457                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2458             {
2459               snat_out2in_trace_t *t =
2460                  vlib_add_trace (vm, node, b0, sizeof (*t));
2461               t->sw_if_index = sw_if_index0;
2462               t->next_index = next0;
2463               t->session_index = ~0;
2464               if (ses0)
2465                 t->session_index = ses0 - dm0->sessions;
2466             }
2467
2468           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2469
2470           /* verify speculative enqueue, maybe switch current next frame */
2471           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2472                                            to_next, n_left_to_next,
2473                                            bi0, next0);
2474         }
2475
2476       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2477     }
2478
2479   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2480                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2481                                pkts_processed);
2482   return frame->n_vectors;
2483 }
2484
2485 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2486   .function = snat_det_out2in_node_fn,
2487   .name = "nat44-det-out2in",
2488   .vector_size = sizeof (u32),
2489   .format_trace = format_snat_out2in_trace,
2490   .type = VLIB_NODE_TYPE_INTERNAL,
2491
2492   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2493   .error_strings = snat_out2in_error_strings,
2494
2495   .runtime_data_bytes = sizeof (snat_runtime_t),
2496
2497   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2498
2499   /* edit / add dispositions here */
2500   .next_nodes = {
2501     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2502     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2503     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2504     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2505     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2506   },
2507 };
2508 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2509
2510 /**
2511  * Get address and port values to be used for ICMP packet translation
2512  * and create session if needed
2513  *
2514  * @param[in,out] sm             NAT main
2515  * @param[in,out] node           NAT node runtime
2516  * @param[in] thread_index       thread index
2517  * @param[in,out] b0             buffer containing packet to be translated
2518  * @param[out] p_proto           protocol used for matching
2519  * @param[out] p_value           address and port after NAT translation
2520  * @param[out] p_dont_translate  if packet should not be translated
2521  * @param d                      optional parameter
2522  * @param e                      optional parameter
2523  */
2524 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2525                           u32 thread_index, vlib_buffer_t *b0,
2526                           ip4_header_t *ip0, u8 *p_proto,
2527                           snat_session_key_t *p_value,
2528                           u8 *p_dont_translate, void *d, void *e)
2529 {
2530   icmp46_header_t *icmp0;
2531   u32 sw_if_index0;
2532   u8 protocol;
2533   snat_det_out_key_t key0;
2534   u8 dont_translate = 0;
2535   u32 next0 = ~0;
2536   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2537   ip4_header_t *inner_ip0;
2538   void *l4_header = 0;
2539   icmp46_header_t *inner_icmp0;
2540   snat_det_map_t * dm0 = 0;
2541   ip4_address_t new_addr0 = {{0}};
2542   snat_det_session_t * ses0 = 0;
2543   ip4_address_t out_addr;
2544
2545   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2546   echo0 = (icmp_echo_header_t *)(icmp0+1);
2547   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2548
2549   if (!icmp_is_error_message (icmp0))
2550     {
2551       protocol = SNAT_PROTOCOL_ICMP;
2552       key0.ext_host_addr = ip0->src_address;
2553       key0.ext_host_port = 0;
2554       key0.out_port = echo0->identifier;
2555       out_addr = ip0->dst_address;
2556     }
2557   else
2558     {
2559       inner_ip0 = (ip4_header_t *)(echo0+1);
2560       l4_header = ip4_next_header (inner_ip0);
2561       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2562       key0.ext_host_addr = inner_ip0->dst_address;
2563       out_addr = inner_ip0->src_address;
2564       switch (protocol)
2565         {
2566         case SNAT_PROTOCOL_ICMP:
2567           inner_icmp0 = (icmp46_header_t*)l4_header;
2568           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2569           key0.ext_host_port = 0;
2570           key0.out_port = inner_echo0->identifier;
2571           break;
2572         case SNAT_PROTOCOL_UDP:
2573         case SNAT_PROTOCOL_TCP:
2574           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2575           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2576           break;
2577         default:
2578           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2579           next0 = SNAT_OUT2IN_NEXT_DROP;
2580           goto out;
2581         }
2582     }
2583
2584   dm0 = snat_det_map_by_out(sm, &out_addr);
2585   if (PREDICT_FALSE(!dm0))
2586     {
2587       /* Don't NAT packet aimed at the intfc address */
2588       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2589                                           ip0->dst_address.as_u32)))
2590         {
2591           dont_translate = 1;
2592           goto out;
2593         }
2594       clib_warning("unknown dst address:  %U",
2595                    format_ip4_address, &ip0->dst_address);
2596       goto out;
2597     }
2598
2599   snat_det_reverse(dm0, &ip0->dst_address,
2600                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2601
2602   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2603   if (PREDICT_FALSE(!ses0))
2604     {
2605       /* Don't NAT packet aimed at the intfc address */
2606       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2607                                           ip0->dst_address.as_u32)))
2608         {
2609           dont_translate = 1;
2610           goto out;
2611         }
2612       clib_warning("no match src %U:%d dst %U:%d for user %U",
2613                    format_ip4_address, &key0.ext_host_addr,
2614                    clib_net_to_host_u16 (key0.ext_host_port),
2615                    format_ip4_address, &out_addr,
2616                    clib_net_to_host_u16 (key0.out_port),
2617                    format_ip4_address, &new_addr0);
2618       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2619       next0 = SNAT_OUT2IN_NEXT_DROP;
2620       goto out;
2621     }
2622
2623   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2624                     !icmp_is_error_message (icmp0)))
2625     {
2626       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2627       next0 = SNAT_OUT2IN_NEXT_DROP;
2628       goto out;
2629     }
2630
2631   goto out;
2632
2633 out:
2634   *p_proto = protocol;
2635   if (ses0)
2636     {
2637       p_value->addr = new_addr0;
2638       p_value->fib_index = sm->inside_fib_index;
2639       p_value->port = ses0->in_port;
2640     }
2641   *p_dont_translate = dont_translate;
2642   if (d)
2643     *(snat_det_session_t**)d = ses0;
2644   if (e)
2645     *(snat_det_map_t**)e = dm0;
2646   return next0;
2647 }
2648
2649 /**********************/
2650 /*** worker handoff ***/
2651 /**********************/
2652 static uword
2653 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2654                                vlib_node_runtime_t * node,
2655                                vlib_frame_t * frame)
2656 {
2657   snat_main_t *sm = &snat_main;
2658   vlib_thread_main_t *tm = vlib_get_thread_main ();
2659   u32 n_left_from, *from, *to_next = 0;
2660   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2661   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2662     = 0;
2663   vlib_frame_queue_elt_t *hf = 0;
2664   vlib_frame_t *f = 0;
2665   int i;
2666   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2667   u32 next_worker_index = 0;
2668   u32 current_worker_index = ~0;
2669   u32 thread_index = vlib_get_thread_index ();
2670
2671   ASSERT (vec_len (sm->workers));
2672
2673   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2674     {
2675       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2676
2677       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2678                                sm->first_worker_index + sm->num_workers - 1,
2679                                (vlib_frame_queue_t *) (~0));
2680     }
2681
2682   from = vlib_frame_vector_args (frame);
2683   n_left_from = frame->n_vectors;
2684
2685   while (n_left_from > 0)
2686     {
2687       u32 bi0;
2688       vlib_buffer_t *b0;
2689       u32 sw_if_index0;
2690       u32 rx_fib_index0;
2691       ip4_header_t * ip0;
2692       u8 do_handoff;
2693
2694       bi0 = from[0];
2695       from += 1;
2696       n_left_from -= 1;
2697
2698       b0 = vlib_get_buffer (vm, bi0);
2699
2700       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2701       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2702
2703       ip0 = vlib_buffer_get_current (b0);
2704
2705       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2706
2707       if (PREDICT_FALSE (next_worker_index != thread_index))
2708         {
2709           do_handoff = 1;
2710
2711           if (next_worker_index != current_worker_index)
2712             {
2713               if (hf)
2714                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2715
2716               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2717                                                       next_worker_index,
2718                                                       handoff_queue_elt_by_worker_index);
2719
2720               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2721               to_next_worker = &hf->buffer_index[hf->n_vectors];
2722               current_worker_index = next_worker_index;
2723             }
2724
2725           /* enqueue to correct worker thread */
2726           to_next_worker[0] = bi0;
2727           to_next_worker++;
2728           n_left_to_next_worker--;
2729
2730           if (n_left_to_next_worker == 0)
2731             {
2732               hf->n_vectors = VLIB_FRAME_SIZE;
2733               vlib_put_frame_queue_elt (hf);
2734               current_worker_index = ~0;
2735               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2736               hf = 0;
2737             }
2738         }
2739       else
2740         {
2741           do_handoff = 0;
2742           /* if this is 1st frame */
2743           if (!f)
2744             {
2745               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2746               to_next = vlib_frame_vector_args (f);
2747             }
2748
2749           to_next[0] = bi0;
2750           to_next += 1;
2751           f->n_vectors++;
2752         }
2753
2754       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2755                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2756         {
2757           snat_out2in_worker_handoff_trace_t *t =
2758             vlib_add_trace (vm, node, b0, sizeof (*t));
2759           t->next_worker_index = next_worker_index;
2760           t->do_handoff = do_handoff;
2761         }
2762     }
2763
2764   if (f)
2765     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2766
2767   if (hf)
2768     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2769
2770   /* Ship frames to the worker nodes */
2771   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2772     {
2773       if (handoff_queue_elt_by_worker_index[i])
2774         {
2775           hf = handoff_queue_elt_by_worker_index[i];
2776           /*
2777            * It works better to let the handoff node
2778            * rate-adapt, always ship the handoff queue element.
2779            */
2780           if (1 || hf->n_vectors == hf->last_n_vectors)
2781             {
2782               vlib_put_frame_queue_elt (hf);
2783               handoff_queue_elt_by_worker_index[i] = 0;
2784             }
2785           else
2786             hf->last_n_vectors = hf->n_vectors;
2787         }
2788       congested_handoff_queue_by_worker_index[i] =
2789         (vlib_frame_queue_t *) (~0);
2790     }
2791   hf = 0;
2792   current_worker_index = ~0;
2793   return frame->n_vectors;
2794 }
2795
2796 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2797   .function = snat_out2in_worker_handoff_fn,
2798   .name = "nat44-out2in-worker-handoff",
2799   .vector_size = sizeof (u32),
2800   .format_trace = format_snat_out2in_worker_handoff_trace,
2801   .type = VLIB_NODE_TYPE_INTERNAL,
2802
2803   .n_next_nodes = 1,
2804
2805   .next_nodes = {
2806     [0] = "error-drop",
2807   },
2808 };
2809
2810 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2811
2812 static uword
2813 snat_out2in_fast_node_fn (vlib_main_t * vm,
2814                           vlib_node_runtime_t * node,
2815                           vlib_frame_t * frame)
2816 {
2817   u32 n_left_from, * from, * to_next;
2818   snat_out2in_next_t next_index;
2819   u32 pkts_processed = 0;
2820   snat_main_t * sm = &snat_main;
2821
2822   from = vlib_frame_vector_args (frame);
2823   n_left_from = frame->n_vectors;
2824   next_index = node->cached_next_index;
2825
2826   while (n_left_from > 0)
2827     {
2828       u32 n_left_to_next;
2829
2830       vlib_get_next_frame (vm, node, next_index,
2831                            to_next, n_left_to_next);
2832
2833       while (n_left_from > 0 && n_left_to_next > 0)
2834         {
2835           u32 bi0;
2836           vlib_buffer_t * b0;
2837           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2838           u32 sw_if_index0;
2839           ip4_header_t * ip0;
2840           ip_csum_t sum0;
2841           u32 new_addr0, old_addr0;
2842           u16 new_port0, old_port0;
2843           udp_header_t * udp0;
2844           tcp_header_t * tcp0;
2845           icmp46_header_t * icmp0;
2846           snat_session_key_t key0, sm0;
2847           u32 proto0;
2848           u32 rx_fib_index0;
2849
2850           /* speculatively enqueue b0 to the current next frame */
2851           bi0 = from[0];
2852           to_next[0] = bi0;
2853           from += 1;
2854           to_next += 1;
2855           n_left_from -= 1;
2856           n_left_to_next -= 1;
2857
2858           b0 = vlib_get_buffer (vm, bi0);
2859
2860           ip0 = vlib_buffer_get_current (b0);
2861           udp0 = ip4_next_header (ip0);
2862           tcp0 = (tcp_header_t *) udp0;
2863           icmp0 = (icmp46_header_t *) udp0;
2864
2865           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2866           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2867
2868           vnet_feature_next (sw_if_index0, &next0, b0);
2869
2870           if (PREDICT_FALSE(ip0->ttl == 1))
2871             {
2872               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2873               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2874                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2875                                            0);
2876               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2877               goto trace00;
2878             }
2879
2880           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2881
2882           if (PREDICT_FALSE (proto0 == ~0))
2883               goto trace00;
2884
2885           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2886             {
2887               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2888                                   rx_fib_index0, node, next0, ~0, 0, 0);
2889               goto trace00;
2890             }
2891
2892           key0.addr = ip0->dst_address;
2893           key0.port = udp0->dst_port;
2894           key0.fib_index = rx_fib_index0;
2895
2896           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2897             {
2898               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2899               goto trace00;
2900             }
2901
2902           new_addr0 = sm0.addr.as_u32;
2903           new_port0 = sm0.port;
2904           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2905           old_addr0 = ip0->dst_address.as_u32;
2906           ip0->dst_address.as_u32 = new_addr0;
2907
2908           sum0 = ip0->checksum;
2909           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2910                                  ip4_header_t,
2911                                  dst_address /* changed member */);
2912           ip0->checksum = ip_csum_fold (sum0);
2913
2914           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2915             {
2916                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2917                 {
2918                   old_port0 = tcp0->dst_port;
2919                   tcp0->dst_port = new_port0;
2920
2921                   sum0 = tcp0->checksum;
2922                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2923                                          ip4_header_t,
2924                                          dst_address /* changed member */);
2925
2926                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2927                                          ip4_header_t /* cheat */,
2928                                          length /* changed member */);
2929                   tcp0->checksum = ip_csum_fold(sum0);
2930                 }
2931               else
2932                 {
2933                   old_port0 = udp0->dst_port;
2934                   udp0->dst_port = new_port0;
2935                   udp0->checksum = 0;
2936                 }
2937             }
2938           else
2939             {
2940               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2941                 {
2942                   sum0 = tcp0->checksum;
2943                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2944                                          ip4_header_t,
2945                                          dst_address /* changed member */);
2946
2947                   tcp0->checksum = ip_csum_fold(sum0);
2948                 }
2949             }
2950
2951         trace00:
2952
2953           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2954                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2955             {
2956               snat_out2in_trace_t *t =
2957                  vlib_add_trace (vm, node, b0, sizeof (*t));
2958               t->sw_if_index = sw_if_index0;
2959               t->next_index = next0;
2960             }
2961
2962           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2963
2964           /* verify speculative enqueue, maybe switch current next frame */
2965           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2966                                            to_next, n_left_to_next,
2967                                            bi0, next0);
2968         }
2969
2970       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2971     }
2972
2973   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2974                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2975                                pkts_processed);
2976   return frame->n_vectors;
2977 }
2978
2979 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2980   .function = snat_out2in_fast_node_fn,
2981   .name = "nat44-out2in-fast",
2982   .vector_size = sizeof (u32),
2983   .format_trace = format_snat_out2in_fast_trace,
2984   .type = VLIB_NODE_TYPE_INTERNAL,
2985
2986   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2987   .error_strings = snat_out2in_error_strings,
2988
2989   .runtime_data_bytes = sizeof (snat_runtime_t),
2990
2991   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2992
2993   /* edit / add dispositions here */
2994   .next_nodes = {
2995     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2996     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2997     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2998     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2999     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3000   },
3001 };
3002 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);