src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   u->nstaticsessions++;
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip)
 337 {
 338   nat_ed_ses_key_t key;
 339   clib_bihash_kv_16_8_t kv;
 340   udp_header_t *udp;
 341
 342   if (ip->protocol == IP_PROTOCOL_ICMP)
 343     {
 344       if (icmp_get_ed_key (ip, &key))
 345         return;
 346     }
 347   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 348     {
 349       udp = ip4_next_header(ip);
 350       key.r_addr = ip->src_address;
 351       key.l_addr = ip->dst_address;
 352       key.proto = ip->protocol;
 353       key.l_port = udp->dst_port;
 354       key.r_port = udp->src_port;
 355     }
 356   else
 357     {
 358       key.r_addr = ip->src_address;
 359       key.l_addr = ip->dst_address;
 360       key.proto = ip->protocol;
 361       key.l_port = key.r_port = 0;
 362     }
 363   key.fib_index = 0;
 364   kv.key[0] = key.as_u64[0];
 365   kv.key[1] = key.as_u64[1];
 366   kv.value = ~0ULL;
 367
 368   if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 369     clib_warning ("in2out_ed key add failed");
 370 }
 371
 372 /**
 373  * Get address and port values to be used for ICMP packet translation
 374  * and create session if needed
 375  *
 376  * @param[in,out] sm             NAT main
 377  * @param[in,out] node           NAT node runtime
 378  * @param[in] thread_index       thread index
 379  * @param[in,out] b0             buffer containing packet to be translated
 380  * @param[out] p_proto           protocol used for matching
 381  * @param[out] p_value           address and port after NAT translation
 382  * @param[out] p_dont_translate  if packet should not be translated
 383  * @param d                      optional parameter
 384  * @param e                      optional parameter
 385  */
 386 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 387                            u32 thread_index, vlib_buffer_t *b0,
 388                            ip4_header_t *ip0, u8 *p_proto,
 389                            snat_session_key_t *p_value,
 390                            u8 *p_dont_translate, void *d, void *e)
 391 {
 392   icmp46_header_t *icmp0;
 393   u32 sw_if_index0;
 394   u32 rx_fib_index0;
 395   snat_session_key_t key0;
 396   snat_session_key_t sm0;
 397   snat_session_t *s0 = 0;
 398   u8 dont_translate = 0;
 399   clib_bihash_kv_8_8_t kv0, value0;
 400   u8 is_addr_only;
 401   u32 next0 = ~0;
 402   int err;
 403
 404   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 405   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 406   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 407
 408   key0.protocol = 0;
 409
 410   err = icmp_get_key (ip0, &key0);
 411   if (err != -1)
 412     {
 413       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 414       next0 = SNAT_OUT2IN_NEXT_DROP;
 415       goto out;
 416     }
 417   key0.fib_index = rx_fib_index0;
 418
 419   kv0.key = key0.as_u64;
 420
 421   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 422                               &value0))
 423     {
 424       /* Try to match static mapping by external address and port,
 425          destination address and port in packet */
 426       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0))
 427         {
 428           if (!sm->forwarding_enabled)
 429             {
 430               /* Don't NAT packet aimed at the intfc address */
 431               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 432                                                   ip0->dst_address.as_u32)))
 433                 {
 434                   dont_translate = 1;
 435                   goto out;
 436                 }
 437               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 438               next0 = SNAT_OUT2IN_NEXT_DROP;
 439               goto out;
 440             }
 441           else
 442             {
 443               dont_translate = 1;
 444               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 445                 {
 446                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 447                   goto out;
 448                 }
 449               create_bypass_for_fwd(sm, ip0);
 450               goto out;
 451             }
 452         }
 453
 454       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 455                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 456         {
 457           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 458           next0 = SNAT_OUT2IN_NEXT_DROP;
 459           goto out;
 460         }
 461
 462       /* Create session initiated by host from external network */
 463       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 464                                              node, thread_index);
 465
 466       if (!s0)
 467         {
 468           next0 = SNAT_OUT2IN_NEXT_DROP;
 469           goto out;
 470         }
 471     }
 472   else
 473     {
 474       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 475                         icmp0->type != ICMP4_echo_request &&
 476                         !icmp_is_error_message (icmp0)))
 477         {
 478           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 479           next0 = SNAT_OUT2IN_NEXT_DROP;
 480           goto out;
 481         }
 482
 483       if (PREDICT_FALSE (value0.value == ~0ULL))
 484         {
 485           nat_ed_ses_key_t key;
 486           clib_bihash_kv_16_8_t s_kv, s_value;
 487
 488           key.as_u64[0] = 0;
 489           key.as_u64[1] = 0;
 490           if (icmp_get_ed_key (ip0, &key))
 491             {
 492               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 493               next0 = SNAT_OUT2IN_NEXT_DROP;
 494               goto out;
 495             }
 496           key.fib_index = rx_fib_index0;
 497           s_kv.key[0] = key.as_u64[0];
 498           s_kv.key[1] = key.as_u64[1];
 499           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 500             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 501                                     s_value.value);
 502           else
 503            {
 504               next0 = SNAT_OUT2IN_NEXT_DROP;
 505               goto out;
 506            }
 507         }
 508       else
 509         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 510                                 value0.value);
 511     }
 512
 513 out:
 514   *p_proto = key0.protocol;
 515   if (s0)
 516     *p_value = s0->in2out;
 517   *p_dont_translate = dont_translate;
 518   if (d)
 519     *(snat_session_t**)d = s0;
 520   return next0;
 521 }
 522
 523 /**
 524  * Get address and port values to be used for ICMP packet translation
 525  *
 526  * @param[in] sm                 NAT main
 527  * @param[in,out] node           NAT node runtime
 528  * @param[in] thread_index       thread index
 529  * @param[in,out] b0             buffer containing packet to be translated
 530  * @param[out] p_proto           protocol used for matching
 531  * @param[out] p_value           address and port after NAT translation
 532  * @param[out] p_dont_translate  if packet should not be translated
 533  * @param d                      optional parameter
 534  * @param e                      optional parameter
 535  */
 536 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 537                            u32 thread_index, vlib_buffer_t *b0,
 538                            ip4_header_t *ip0, u8 *p_proto,
 539                            snat_session_key_t *p_value,
 540                            u8 *p_dont_translate, void *d, void *e)
 541 {
 542   icmp46_header_t *icmp0;
 543   u32 sw_if_index0;
 544   u32 rx_fib_index0;
 545   snat_session_key_t key0;
 546   snat_session_key_t sm0;
 547   u8 dont_translate = 0;
 548   u8 is_addr_only;
 549   u32 next0 = ~0;
 550   int err;
 551
 552   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 553   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 554   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 555
 556   err = icmp_get_key (ip0, &key0);
 557   if (err != -1)
 558     {
 559       b0->error = node->errors[err];
 560       next0 = SNAT_OUT2IN_NEXT_DROP;
 561       goto out2;
 562     }
 563   key0.fib_index = rx_fib_index0;
 564
 565   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0))
 566     {
 567       /* Don't NAT packet aimed at the intfc address */
 568       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 569         {
 570           dont_translate = 1;
 571           goto out;
 572         }
 573       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 574       next0 = SNAT_OUT2IN_NEXT_DROP;
 575       goto out;
 576     }
 577
 578   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 579                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 580                     !icmp_is_error_message (icmp0)))
 581     {
 582       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 583       next0 = SNAT_OUT2IN_NEXT_DROP;
 584       goto out;
 585     }
 586
 587 out:
 588   *p_value = sm0;
 589 out2:
 590   *p_proto = key0.protocol;
 591   *p_dont_translate = dont_translate;
 592   return next0;
 593 }
 594
 595 static inline u32 icmp_out2in (snat_main_t *sm,
 596                                vlib_buffer_t * b0,
 597                                ip4_header_t * ip0,
 598                                icmp46_header_t * icmp0,
 599                                u32 sw_if_index0,
 600                                u32 rx_fib_index0,
 601                                vlib_node_runtime_t * node,
 602                                u32 next0,
 603                                u32 thread_index,
 604                                void *d,
 605                                void *e)
 606 {
 607   snat_session_key_t sm0;
 608   u8 protocol;
 609   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 610   ip4_header_t *inner_ip0 = 0;
 611   void *l4_header = 0;
 612   icmp46_header_t *inner_icmp0;
 613   u8 dont_translate;
 614   u32 new_addr0, old_addr0;
 615   u16 old_id0, new_id0;
 616   ip_csum_t sum0;
 617   u16 checksum0;
 618   u32 next0_tmp;
 619
 620   echo0 = (icmp_echo_header_t *)(icmp0+1);
 621
 622   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 623                                        &protocol, &sm0, &dont_translate, d, e);
 624   if (next0_tmp != ~0)
 625     next0 = next0_tmp;
 626   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 627     goto out;
 628
 629   sum0 = ip_incremental_checksum (0, icmp0,
 630                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 631   checksum0 = ~ip_csum_fold (sum0);
 632   if (checksum0 != 0 && checksum0 != 0xffff)
 633     {
 634       next0 = SNAT_OUT2IN_NEXT_DROP;
 635       goto out;
 636     }
 637
 638   old_addr0 = ip0->dst_address.as_u32;
 639   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 640   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 641
 642   sum0 = ip0->checksum;
 643   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 644                          dst_address /* changed member */);
 645   ip0->checksum = ip_csum_fold (sum0);
 646
 647   if (!icmp_is_error_message (icmp0))
 648     {
 649       new_id0 = sm0.port;
 650       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 651         {
 652           old_id0 = echo0->identifier;
 653           new_id0 = sm0.port;
 654           echo0->identifier = new_id0;
 655
 656           sum0 = icmp0->checksum;
 657           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 658                                  identifier /* changed member */);
 659           icmp0->checksum = ip_csum_fold (sum0);
 660         }
 661     }
 662   else
 663     {
 664       inner_ip0 = (ip4_header_t *)(echo0+1);
 665       l4_header = ip4_next_header (inner_ip0);
 666
 667       if (!ip4_header_checksum_is_valid (inner_ip0))
 668         {
 669           next0 = SNAT_OUT2IN_NEXT_DROP;
 670           goto out;
 671         }
 672
 673       old_addr0 = inner_ip0->src_address.as_u32;
 674       inner_ip0->src_address = sm0.addr;
 675       new_addr0 = inner_ip0->src_address.as_u32;
 676
 677       sum0 = icmp0->checksum;
 678       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 679                              src_address /* changed member */);
 680       icmp0->checksum = ip_csum_fold (sum0);
 681
 682       switch (protocol)
 683         {
 684         case SNAT_PROTOCOL_ICMP:
 685           inner_icmp0 = (icmp46_header_t*)l4_header;
 686           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 687
 688           old_id0 = inner_echo0->identifier;
 689           new_id0 = sm0.port;
 690           inner_echo0->identifier = new_id0;
 691
 692           sum0 = icmp0->checksum;
 693           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 694                                  identifier);
 695           icmp0->checksum = ip_csum_fold (sum0);
 696           break;
 697         case SNAT_PROTOCOL_UDP:
 698         case SNAT_PROTOCOL_TCP:
 699           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 700           new_id0 = sm0.port;
 701           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 702
 703           sum0 = icmp0->checksum;
 704           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 705                                  src_port);
 706           icmp0->checksum = ip_csum_fold (sum0);
 707           break;
 708         default:
 709           ASSERT(0);
 710         }
 711     }
 712
 713 out:
 714   return next0;
 715 }
 716
 717
 718 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 719                                          vlib_buffer_t * b0,
 720                                          ip4_header_t * ip0,
 721                                          icmp46_header_t * icmp0,
 722                                          u32 sw_if_index0,
 723                                          u32 rx_fib_index0,
 724                                          vlib_node_runtime_t * node,
 725                                          u32 next0, f64 now,
 726                                          u32 thread_index,
 727                                          snat_session_t ** p_s0)
 728 {
 729   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 730                       next0, thread_index, p_s0, 0);
 731   snat_session_t * s0 = *p_s0;
 732   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 733     {
 734       /* Accounting */
 735       s0->last_heard = now;
 736       s0->total_pkts++;
 737       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 738       /* Per-user LRU list maintenance */
 739       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 740                          s0->per_user_index);
 741       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 742                           s0->per_user_list_head_index,
 743                           s0->per_user_index);
 744     }
 745   return next0;
 746 }
 747
 748 static snat_session_t *
 749 snat_out2in_unknown_proto (snat_main_t *sm,
 750                            vlib_buffer_t * b,
 751                            ip4_header_t * ip,
 752                            u32 rx_fib_index,
 753                            u32 thread_index,
 754                            f64 now,
 755                            vlib_main_t * vm,
 756                            vlib_node_runtime_t * node)
 757 {
 758   clib_bihash_kv_8_8_t kv, value;
 759   clib_bihash_kv_16_8_t s_kv, s_value;
 760   snat_static_mapping_t *m;
 761   snat_session_key_t m_key;
 762   u32 old_addr, new_addr;
 763   ip_csum_t sum;
 764   nat_ed_ses_key_t key;
 765   snat_session_t * s;
 766   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 767   snat_user_t *u;
 768
 769   old_addr = ip->dst_address.as_u32;
 770
 771   key.l_addr = ip->dst_address;
 772   key.r_addr = ip->src_address;
 773   key.fib_index = rx_fib_index;
 774   key.proto = ip->protocol;
 775   key.r_port = 0;
 776   key.l_port = 0;
 777   s_kv.key[0] = key.as_u64[0];
 778   s_kv.key[1] = key.as_u64[1];
 779
 780   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 781     {
 782       s = pool_elt_at_index (tsm->sessions, s_value.value);
 783       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 784     }
 785   else
 786     {
 787       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 788         {
 789           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 790           return 0;
 791         }
 792
 793       m_key.addr = ip->dst_address;
 794       m_key.port = 0;
 795       m_key.protocol = 0;
 796       m_key.fib_index = rx_fib_index;
 797       kv.key = m_key.as_u64;
 798       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 799         {
 800           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 801           return 0;
 802         }
 803
 804       m = pool_elt_at_index (sm->static_mappings, value.value);
 805
 806       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 807
 808       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 809                                   thread_index);
 810       if (!u)
 811         {
 812           clib_warning ("create NAT user failed");
 813           return 0;
 814         }
 815
 816       /* Create a new session */
 817       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 818       if (!s)
 819         {
 820           clib_warning ("create NAT session failed");
 821           return 0;
 822         }
 823
 824       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 825       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 826       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 827       s->outside_address_index = ~0;
 828       s->out2in.addr.as_u32 = old_addr;
 829       s->out2in.fib_index = rx_fib_index;
 830       s->in2out.addr.as_u32 = new_addr;
 831       s->in2out.fib_index = m->fib_index;
 832       s->in2out.port = s->out2in.port = ip->protocol;
 833       u->nstaticsessions++;
 834
 835       /* Add to lookup tables */
 836       s_kv.value = s - tsm->sessions;
 837       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 838         clib_warning ("out2in key add failed");
 839
 840       key.l_addr = ip->dst_address;
 841       key.fib_index = m->fib_index;
 842       s_kv.key[0] = key.as_u64[0];
 843       s_kv.key[1] = key.as_u64[1];
 844       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 845         clib_warning ("in2out key add failed");
 846    }
 847
 848   /* Update IP checksum */
 849   sum = ip->checksum;
 850   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 851   ip->checksum = ip_csum_fold (sum);
 852
 853   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 854
 855   /* Accounting */
 856   s->last_heard = now;
 857   s->total_pkts++;
 858   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 859   /* Per-user LRU list maintenance */
 860   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 861   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 862                       s->per_user_index);
 863
 864   return s;
 865 }
 866
 867 static snat_session_t *
 868 snat_out2in_lb (snat_main_t *sm,
 869                 vlib_buffer_t * b,
 870                 ip4_header_t * ip,
 871                 u32 rx_fib_index,
 872                 u32 thread_index,
 873                 f64 now,
 874                 vlib_main_t * vm,
 875                 vlib_node_runtime_t * node)
 876 {
 877   nat_ed_ses_key_t key;
 878   clib_bihash_kv_16_8_t s_kv, s_value;
 879   udp_header_t *udp = ip4_next_header (ip);
 880   tcp_header_t *tcp = (tcp_header_t *) udp;
 881   snat_session_t *s = 0;
 882   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 883   snat_session_key_t e_key, l_key;
 884   u32 old_addr, new_addr;
 885   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 886   u16 new_port, old_port;
 887   ip_csum_t sum;
 888   snat_user_t *u;
 889   u32 address_index;
 890   snat_session_key_t eh_key;
 891   u8 twice_nat;
 892
 893   old_addr = ip->dst_address.as_u32;
 894
 895   key.l_addr = ip->dst_address;
 896   key.r_addr = ip->src_address;
 897   key.fib_index = rx_fib_index;
 898   key.proto = ip->protocol;
 899   key.r_port = udp->src_port;
 900   key.l_port = udp->dst_port;
 901   s_kv.key[0] = key.as_u64[0];
 902   s_kv.key[1] = key.as_u64[1];
 903
 904   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 905     {
 906       s = pool_elt_at_index (tsm->sessions, s_value.value);
 907     }
 908   else
 909     {
 910       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 911         {
 912           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 913           return 0;
 914         }
 915
 916       e_key.addr = ip->dst_address;
 917       e_key.port = udp->dst_port;
 918       e_key.protocol = proto;
 919       e_key.fib_index = rx_fib_index;
 920       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat))
 921         return 0;
 922
 923       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 924                                   thread_index);
 925       if (!u)
 926       {
 927         clib_warning ("create NAT user failed");
 928         return 0;
 929       }
 930
 931       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 932       if (!s)
 933         {
 934           clib_warning ("create NAT session failed");
 935           return 0;
 936         }
 937
 938       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 939       s->ext_host_port = udp->src_port;
 940       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 941       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 942       s->outside_address_index = ~0;
 943       s->out2in = e_key;
 944       s->in2out = l_key;
 945       u->nstaticsessions++;
 946
 947       /* Add to lookup tables */
 948       s_kv.value = s - tsm->sessions;
 949       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 950         clib_warning ("out2in-ed key add failed");
 951
 952       if (twice_nat)
 953         {
 954           eh_key.protocol = proto;
 955           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 956                                                    thread_index, &eh_key,
 957                                                    &address_index,
 958                                                    sm->port_per_thread,
 959                                                    sm->per_thread_data[thread_index].snat_thread_index))
 960             {
 961               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
 962               return 0;
 963             }
 964           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 965           key.r_port = s->ext_host_nat_port = eh_key.port;
 966           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 967         }
 968       key.l_addr = l_key.addr;
 969       key.fib_index = l_key.fib_index;
 970       key.l_port = l_key.port;
 971       s_kv.key[0] = key.as_u64[0];
 972       s_kv.key[1] = key.as_u64[1];
 973       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 974         clib_warning ("in2out-ed key add failed");
 975     }
 976
 977   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 978
 979   /* Update IP checksum */
 980   sum = ip->checksum;
 981   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 982   if (is_twice_nat_session (s))
 983     sum = ip_csum_update (sum, ip->src_address.as_u32,
 984                           s->ext_host_nat_addr.as_u32, ip4_header_t,
 985                           src_address);
 986   ip->checksum = ip_csum_fold (sum);
 987
 988   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 989     {
 990       old_port = tcp->dst_port;
 991       tcp->dst_port = s->in2out.port;
 992       new_port = tcp->dst_port;
 993
 994       sum = tcp->checksum;
 995       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 996       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 997       if (is_twice_nat_session (s))
 998         {
 999           sum = ip_csum_update (sum, ip->src_address.as_u32,
1000                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1001                                 dst_address);
1002           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1003                                 ip4_header_t, length);
1004           tcp->src_port = s->ext_host_nat_port;
1005           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1006         }
1007       tcp->checksum = ip_csum_fold(sum);
1008     }
1009   else
1010     {
1011       udp->dst_port = s->in2out.port;
1012       if (is_twice_nat_session (s))
1013         {
1014           udp->src_port = s->ext_host_nat_port;
1015           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1016         }
1017       udp->checksum = 0;
1018     }
1019
1020   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1021
1022   /* Accounting */
1023   s->last_heard = now;
1024   s->total_pkts++;
1025   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1026   /* Per-user LRU list maintenance */
1027   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1028   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1029                       s->per_user_index);
1030
1031   return s;
1032 }
1033
1034 static uword
1035 snat_out2in_node_fn (vlib_main_t * vm,
1036                   vlib_node_runtime_t * node,
1037                   vlib_frame_t * frame)
1038 {
1039   u32 n_left_from, * from, * to_next;
1040   snat_out2in_next_t next_index;
1041   u32 pkts_processed = 0;
1042   snat_main_t * sm = &snat_main;
1043   f64 now = vlib_time_now (vm);
1044   u32 thread_index = vlib_get_thread_index ();
1045
1046   from = vlib_frame_vector_args (frame);
1047   n_left_from = frame->n_vectors;
1048   next_index = node->cached_next_index;
1049
1050   while (n_left_from > 0)
1051     {
1052       u32 n_left_to_next;
1053
1054       vlib_get_next_frame (vm, node, next_index,
1055                            to_next, n_left_to_next);
1056
1057       while (n_left_from >= 4 && n_left_to_next >= 2)
1058         {
1059           u32 bi0, bi1;
1060           vlib_buffer_t * b0, * b1;
1061           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1062           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1063           u32 sw_if_index0, sw_if_index1;
1064           ip4_header_t * ip0, *ip1;
1065           ip_csum_t sum0, sum1;
1066           u32 new_addr0, old_addr0;
1067           u16 new_port0, old_port0;
1068           u32 new_addr1, old_addr1;
1069           u16 new_port1, old_port1;
1070           udp_header_t * udp0, * udp1;
1071           tcp_header_t * tcp0, * tcp1;
1072           icmp46_header_t * icmp0, * icmp1;
1073           snat_session_key_t key0, key1, sm0, sm1;
1074           u32 rx_fib_index0, rx_fib_index1;
1075           u32 proto0, proto1;
1076           snat_session_t * s0 = 0, * s1 = 0;
1077           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1078
1079           /* Prefetch next iteration. */
1080           {
1081             vlib_buffer_t * p2, * p3;
1082
1083             p2 = vlib_get_buffer (vm, from[2]);
1084             p3 = vlib_get_buffer (vm, from[3]);
1085
1086             vlib_prefetch_buffer_header (p2, LOAD);
1087             vlib_prefetch_buffer_header (p3, LOAD);
1088
1089             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1090             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1091           }
1092
1093           /* speculatively enqueue b0 and b1 to the current next frame */
1094           to_next[0] = bi0 = from[0];
1095           to_next[1] = bi1 = from[1];
1096           from += 2;
1097           to_next += 2;
1098           n_left_from -= 2;
1099           n_left_to_next -= 2;
1100
1101           b0 = vlib_get_buffer (vm, bi0);
1102           b1 = vlib_get_buffer (vm, bi1);
1103
1104           vnet_buffer (b0)->snat.flags = 0;
1105           vnet_buffer (b1)->snat.flags = 0;
1106
1107           ip0 = vlib_buffer_get_current (b0);
1108           udp0 = ip4_next_header (ip0);
1109           tcp0 = (tcp_header_t *) udp0;
1110           icmp0 = (icmp46_header_t *) udp0;
1111
1112           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1113           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1114                                    sw_if_index0);
1115
1116           if (PREDICT_FALSE(ip0->ttl == 1))
1117             {
1118               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1119               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1120                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1121                                            0);
1122               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1123               goto trace0;
1124             }
1125
1126           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1127
1128           if (PREDICT_FALSE (proto0 == ~0))
1129             {
1130               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1131                                              thread_index, now, vm, node);
1132               if (!sm->forwarding_enabled)
1133                 if (!s0)
1134                   next0 = SNAT_OUT2IN_NEXT_DROP;
1135               goto trace0;
1136             }
1137
1138           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1139             {
1140               next0 = icmp_out2in_slow_path
1141                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1142                  next0, now, thread_index, &s0);
1143               goto trace0;
1144             }
1145
1146           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1147             {
1148               next0 = SNAT_OUT2IN_NEXT_REASS;
1149               goto trace0;
1150             }
1151
1152           key0.addr = ip0->dst_address;
1153           key0.port = udp0->dst_port;
1154           key0.protocol = proto0;
1155           key0.fib_index = rx_fib_index0;
1156
1157           kv0.key = key0.as_u64;
1158
1159           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1160                                       &kv0, &value0))
1161             {
1162               /* Try to match static mapping by external address and port,
1163                  destination address and port in packet */
1164               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1165                 {
1166                   if (!sm->forwarding_enabled)
1167                     {
1168                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1169                       /*
1170                        * Send DHCP packets to the ipv4 stack, or we won't
1171                        * be able to use dhcp client on the outside interface
1172                        */
1173                       if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_UDP
1174                           || (udp0->dst_port
1175                               != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1176                         next0 = SNAT_OUT2IN_NEXT_DROP;
1177                       else
1178                         vnet_feature_next
1179                           (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1180                            &next0, b0);
1181                       goto trace0;
1182                     }
1183                   else
1184                     {
1185                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1186                         {
1187                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1188                           goto trace0;
1189                         }
1190                       create_bypass_for_fwd(sm, ip0);
1191                       goto trace0;
1192                     }
1193                 }
1194
1195               /* Create session initiated by host from external network */
1196               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1197                                                      thread_index);
1198               if (!s0)
1199                 {
1200                   next0 = SNAT_OUT2IN_NEXT_DROP;
1201                   goto trace0;
1202                 }
1203             }
1204           else
1205             {
1206               if (PREDICT_FALSE (value0.value == ~0ULL))
1207                 {
1208                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1209                                       now, vm, node);
1210                   if (!s0)
1211                     next0 = SNAT_OUT2IN_NEXT_DROP;
1212                   goto trace0;
1213                 }
1214               else
1215                 {
1216                   s0 = pool_elt_at_index (
1217                     sm->per_thread_data[thread_index].sessions,
1218                     value0.value);
1219                 }
1220             }
1221
1222           old_addr0 = ip0->dst_address.as_u32;
1223           ip0->dst_address = s0->in2out.addr;
1224           new_addr0 = ip0->dst_address.as_u32;
1225           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1226
1227           sum0 = ip0->checksum;
1228           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1229                                  ip4_header_t,
1230                                  dst_address /* changed member */);
1231           ip0->checksum = ip_csum_fold (sum0);
1232
1233           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1234             {
1235               old_port0 = tcp0->dst_port;
1236               tcp0->dst_port = s0->in2out.port;
1237               new_port0 = tcp0->dst_port;
1238
1239               sum0 = tcp0->checksum;
1240               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1241                                      ip4_header_t,
1242                                      dst_address /* changed member */);
1243
1244               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1245                                      ip4_header_t /* cheat */,
1246                                      length /* changed member */);
1247               tcp0->checksum = ip_csum_fold(sum0);
1248             }
1249           else
1250             {
1251               old_port0 = udp0->dst_port;
1252               udp0->dst_port = s0->in2out.port;
1253               udp0->checksum = 0;
1254             }
1255
1256           /* Accounting */
1257           s0->last_heard = now;
1258           s0->total_pkts++;
1259           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1260           /* Per-user LRU list maintenance */
1261           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1262                              s0->per_user_index);
1263           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1264                               s0->per_user_list_head_index,
1265                               s0->per_user_index);
1266         trace0:
1267
1268           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1269                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1270             {
1271               snat_out2in_trace_t *t =
1272                  vlib_add_trace (vm, node, b0, sizeof (*t));
1273               t->sw_if_index = sw_if_index0;
1274               t->next_index = next0;
1275               t->session_index = ~0;
1276               if (s0)
1277                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1278             }
1279
1280           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1281
1282
1283           ip1 = vlib_buffer_get_current (b1);
1284           udp1 = ip4_next_header (ip1);
1285           tcp1 = (tcp_header_t *) udp1;
1286           icmp1 = (icmp46_header_t *) udp1;
1287
1288           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1289           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1290                                    sw_if_index1);
1291
1292           if (PREDICT_FALSE(ip1->ttl == 1))
1293             {
1294               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1295               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1296                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1297                                            0);
1298               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1299               goto trace1;
1300             }
1301
1302           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1303
1304           if (PREDICT_FALSE (proto1 == ~0))
1305             {
1306               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1307                                              thread_index, now, vm, node);
1308               if (!sm->forwarding_enabled)
1309                 if (!s1)
1310                   next1 = SNAT_OUT2IN_NEXT_DROP;
1311               goto trace1;
1312             }
1313
1314           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1315             {
1316               next1 = icmp_out2in_slow_path
1317                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1318                  next1, now, thread_index, &s1);
1319               goto trace1;
1320             }
1321
1322           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1323             {
1324               next1 = SNAT_OUT2IN_NEXT_REASS;
1325               goto trace1;
1326             }
1327
1328           key1.addr = ip1->dst_address;
1329           key1.port = udp1->dst_port;
1330           key1.protocol = proto1;
1331           key1.fib_index = rx_fib_index1;
1332
1333           kv1.key = key1.as_u64;
1334
1335           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1336                                       &kv1, &value1))
1337             {
1338               /* Try to match static mapping by external address and port,
1339                  destination address and port in packet */
1340               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0))
1341                 {
1342                   if (!sm->forwarding_enabled)
1343                     {
1344                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1345                       /*
1346                        * Send DHCP packets to the ipv4 stack, or we won't
1347                        * be able to use dhcp client on the outside interface
1348                        */
1349                       if (PREDICT_TRUE (proto1 != SNAT_PROTOCOL_UDP
1350                           || (udp1->dst_port
1351                               != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1352                         next1 = SNAT_OUT2IN_NEXT_DROP;
1353                       else
1354                         vnet_feature_next
1355                           (vnet_buffer (b1)->sw_if_index[VLIB_RX],
1356                            &next1, b1);
1357                       goto trace1;
1358                     }
1359                   else
1360                     {
1361                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1362                         {
1363                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1364                           goto trace1;
1365                         }
1366                       create_bypass_for_fwd(sm, ip1);
1367                       goto trace1;
1368                     }
1369                 }
1370
1371               /* Create session initiated by host from external network */
1372               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1373                                                      thread_index);
1374               if (!s1)
1375                 {
1376                   next1 = SNAT_OUT2IN_NEXT_DROP;
1377                   goto trace1;
1378                 }
1379             }
1380           else
1381             {
1382               if (PREDICT_FALSE (value1.value == ~0ULL))
1383                 {
1384                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1385                                       now, vm, node);
1386                   if (!s1)
1387                     next1 = SNAT_OUT2IN_NEXT_DROP;
1388                   goto trace1;
1389                 }
1390               else
1391                 {
1392                   s1 = pool_elt_at_index (
1393                     sm->per_thread_data[thread_index].sessions,
1394                     value1.value);
1395                 }
1396             }
1397
1398           old_addr1 = ip1->dst_address.as_u32;
1399           ip1->dst_address = s1->in2out.addr;
1400           new_addr1 = ip1->dst_address.as_u32;
1401           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1402
1403           sum1 = ip1->checksum;
1404           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1405                                  ip4_header_t,
1406                                  dst_address /* changed member */);
1407           ip1->checksum = ip_csum_fold (sum1);
1408
1409           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1410             {
1411               old_port1 = tcp1->dst_port;
1412               tcp1->dst_port = s1->in2out.port;
1413               new_port1 = tcp1->dst_port;
1414
1415               sum1 = tcp1->checksum;
1416               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1417                                      ip4_header_t,
1418                                      dst_address /* changed member */);
1419
1420               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1421                                      ip4_header_t /* cheat */,
1422                                      length /* changed member */);
1423               tcp1->checksum = ip_csum_fold(sum1);
1424             }
1425           else
1426             {
1427               old_port1 = udp1->dst_port;
1428               udp1->dst_port = s1->in2out.port;
1429               udp1->checksum = 0;
1430             }
1431
1432           /* Accounting */
1433           s1->last_heard = now;
1434           s1->total_pkts++;
1435           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1436           /* Per-user LRU list maintenance */
1437           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1438                              s1->per_user_index);
1439           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1440                               s1->per_user_list_head_index,
1441                               s1->per_user_index);
1442         trace1:
1443
1444           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1445                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1446             {
1447               snat_out2in_trace_t *t =
1448                  vlib_add_trace (vm, node, b1, sizeof (*t));
1449               t->sw_if_index = sw_if_index1;
1450               t->next_index = next1;
1451               t->session_index = ~0;
1452               if (s1)
1453                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1454             }
1455
1456           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1457
1458           /* verify speculative enqueues, maybe switch current next frame */
1459           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1460                                            to_next, n_left_to_next,
1461                                            bi0, bi1, next0, next1);
1462         }
1463
1464       while (n_left_from > 0 && n_left_to_next > 0)
1465         {
1466           u32 bi0;
1467           vlib_buffer_t * b0;
1468           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1469           u32 sw_if_index0;
1470           ip4_header_t * ip0;
1471           ip_csum_t sum0;
1472           u32 new_addr0, old_addr0;
1473           u16 new_port0, old_port0;
1474           udp_header_t * udp0;
1475           tcp_header_t * tcp0;
1476           icmp46_header_t * icmp0;
1477           snat_session_key_t key0, sm0;
1478           u32 rx_fib_index0;
1479           u32 proto0;
1480           snat_session_t * s0 = 0;
1481           clib_bihash_kv_8_8_t kv0, value0;
1482
1483           /* speculatively enqueue b0 to the current next frame */
1484           bi0 = from[0];
1485           to_next[0] = bi0;
1486           from += 1;
1487           to_next += 1;
1488           n_left_from -= 1;
1489           n_left_to_next -= 1;
1490
1491           b0 = vlib_get_buffer (vm, bi0);
1492
1493           vnet_buffer (b0)->snat.flags = 0;
1494
1495           ip0 = vlib_buffer_get_current (b0);
1496           udp0 = ip4_next_header (ip0);
1497           tcp0 = (tcp_header_t *) udp0;
1498           icmp0 = (icmp46_header_t *) udp0;
1499
1500           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1501           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1502                                    sw_if_index0);
1503
1504           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1505
1506           if (PREDICT_FALSE (proto0 == ~0))
1507             {
1508               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1509                                              thread_index, now, vm, node);
1510               if (!sm->forwarding_enabled)
1511                 if (!s0)
1512                   next0 = SNAT_OUT2IN_NEXT_DROP;
1513               goto trace00;
1514             }
1515
1516           if (PREDICT_FALSE(ip0->ttl == 1))
1517             {
1518               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1519               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1520                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1521                                            0);
1522               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1523               goto trace00;
1524             }
1525
1526           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1527             {
1528               next0 = icmp_out2in_slow_path
1529                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1530                  next0, now, thread_index, &s0);
1531               goto trace00;
1532             }
1533
1534           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1535             {
1536               next0 = SNAT_OUT2IN_NEXT_REASS;
1537               goto trace00;
1538             }
1539
1540           key0.addr = ip0->dst_address;
1541           key0.port = udp0->dst_port;
1542           key0.protocol = proto0;
1543           key0.fib_index = rx_fib_index0;
1544
1545           kv0.key = key0.as_u64;
1546
1547           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1548                                       &kv0, &value0))
1549             {
1550               /* Try to match static mapping by external address and port,
1551                  destination address and port in packet */
1552               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1553                 {
1554                   if (!sm->forwarding_enabled)
1555                     {
1556                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1557                       /*
1558                        * Send DHCP packets to the ipv4 stack, or we won't
1559                        * be able to use dhcp client on the outside interface
1560                        */
1561                       if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_UDP
1562                           || (udp0->dst_port
1563                               != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1564                         next0 = SNAT_OUT2IN_NEXT_DROP;
1565                       else
1566                         vnet_feature_next
1567                           (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1568                            &next0, b0);
1569                       goto trace00;
1570                     }
1571                   else
1572                     {
1573                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1574                         {
1575                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1576                           goto trace00;
1577                         }
1578                       create_bypass_for_fwd(sm, ip0);
1579                       goto trace00;
1580                     }
1581                 }
1582
1583               /* Create session initiated by host from external network */
1584               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1585                                                      thread_index);
1586               if (!s0)
1587                 {
1588                   next0 = SNAT_OUT2IN_NEXT_DROP;
1589                   goto trace00;
1590                 }
1591             }
1592           else
1593             {
1594               if (PREDICT_FALSE (value0.value == ~0ULL))
1595                 {
1596                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1597                                       now, vm, node);
1598                   if (!s0)
1599                     next0 = SNAT_OUT2IN_NEXT_DROP;
1600                   goto trace00;
1601                 }
1602               else
1603                 {
1604                   s0 = pool_elt_at_index (
1605                     sm->per_thread_data[thread_index].sessions,
1606                     value0.value);
1607                 }
1608             }
1609
1610           old_addr0 = ip0->dst_address.as_u32;
1611           ip0->dst_address = s0->in2out.addr;
1612           new_addr0 = ip0->dst_address.as_u32;
1613           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1614
1615           sum0 = ip0->checksum;
1616           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1617                                  ip4_header_t,
1618                                  dst_address /* changed member */);
1619           ip0->checksum = ip_csum_fold (sum0);
1620
1621           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1622             {
1623               old_port0 = tcp0->dst_port;
1624               tcp0->dst_port = s0->in2out.port;
1625               new_port0 = tcp0->dst_port;
1626
1627               sum0 = tcp0->checksum;
1628               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1629                                      ip4_header_t,
1630                                      dst_address /* changed member */);
1631
1632               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1633                                      ip4_header_t /* cheat */,
1634                                      length /* changed member */);
1635               tcp0->checksum = ip_csum_fold(sum0);
1636             }
1637           else
1638             {
1639               old_port0 = udp0->dst_port;
1640               udp0->dst_port = s0->in2out.port;
1641               udp0->checksum = 0;
1642             }
1643
1644           /* Accounting */
1645           s0->last_heard = now;
1646           s0->total_pkts++;
1647           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1648           /* Per-user LRU list maintenance */
1649           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1650                              s0->per_user_index);
1651           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1652                               s0->per_user_list_head_index,
1653                               s0->per_user_index);
1654         trace00:
1655
1656           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1657                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1658             {
1659               snat_out2in_trace_t *t =
1660                  vlib_add_trace (vm, node, b0, sizeof (*t));
1661               t->sw_if_index = sw_if_index0;
1662               t->next_index = next0;
1663               t->session_index = ~0;
1664               if (s0)
1665                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1666             }
1667
1668           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1669
1670           /* verify speculative enqueue, maybe switch current next frame */
1671           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1672                                            to_next, n_left_to_next,
1673                                            bi0, next0);
1674         }
1675
1676       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1677     }
1678
1679   vlib_node_increment_counter (vm, snat_out2in_node.index,
1680                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1681                                pkts_processed);
1682   return frame->n_vectors;
1683 }
1684
1685 VLIB_REGISTER_NODE (snat_out2in_node) = {
1686   .function = snat_out2in_node_fn,
1687   .name = "nat44-out2in",
1688   .vector_size = sizeof (u32),
1689   .format_trace = format_snat_out2in_trace,
1690   .type = VLIB_NODE_TYPE_INTERNAL,
1691
1692   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1693   .error_strings = snat_out2in_error_strings,
1694
1695   .runtime_data_bytes = sizeof (snat_runtime_t),
1696
1697   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1698
1699   /* edit / add dispositions here */
1700   .next_nodes = {
1701     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1702     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1703     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1704     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1705     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1706   },
1707 };
1708 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1709
1710 static uword
1711 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1712                             vlib_node_runtime_t * node,
1713                             vlib_frame_t * frame)
1714 {
1715   u32 n_left_from, *from, *to_next;
1716   snat_out2in_next_t next_index;
1717   u32 pkts_processed = 0;
1718   snat_main_t *sm = &snat_main;
1719   f64 now = vlib_time_now (vm);
1720   u32 thread_index = vlib_get_thread_index ();
1721   snat_main_per_thread_data_t *per_thread_data =
1722     &sm->per_thread_data[thread_index];
1723   u32 *fragments_to_drop = 0;
1724   u32 *fragments_to_loopback = 0;
1725
1726   from = vlib_frame_vector_args (frame);
1727   n_left_from = frame->n_vectors;
1728   next_index = node->cached_next_index;
1729
1730   while (n_left_from > 0)
1731     {
1732       u32 n_left_to_next;
1733
1734       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1735
1736       while (n_left_from > 0 && n_left_to_next > 0)
1737        {
1738           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1739           vlib_buffer_t *b0;
1740           u32 next0;
1741           u8 cached0 = 0;
1742           ip4_header_t *ip0;
1743           nat_reass_ip4_t *reass0;
1744           udp_header_t * udp0;
1745           tcp_header_t * tcp0;
1746           snat_session_key_t key0, sm0;
1747           clib_bihash_kv_8_8_t kv0, value0;
1748           snat_session_t * s0 = 0;
1749           u16 old_port0, new_port0;
1750           ip_csum_t sum0;
1751
1752           /* speculatively enqueue b0 to the current next frame */
1753           bi0 = from[0];
1754           to_next[0] = bi0;
1755           from += 1;
1756           to_next += 1;
1757           n_left_from -= 1;
1758           n_left_to_next -= 1;
1759
1760           b0 = vlib_get_buffer (vm, bi0);
1761           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1762
1763           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1764           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1765                                                                sw_if_index0);
1766
1767           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1768             {
1769               next0 = SNAT_OUT2IN_NEXT_DROP;
1770               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1771               goto trace0;
1772             }
1773
1774           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1775           udp0 = ip4_next_header (ip0);
1776           tcp0 = (tcp_header_t *) udp0;
1777           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1778
1779           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1780                                                  ip0->dst_address,
1781                                                  ip0->fragment_id,
1782                                                  ip0->protocol,
1783                                                  1,
1784                                                  &fragments_to_drop);
1785
1786           if (PREDICT_FALSE (!reass0))
1787             {
1788               next0 = SNAT_OUT2IN_NEXT_DROP;
1789               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1790               goto trace0;
1791             }
1792
1793           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1794             {
1795               key0.addr = ip0->dst_address;
1796               key0.port = udp0->dst_port;
1797               key0.protocol = proto0;
1798               key0.fib_index = rx_fib_index0;
1799               kv0.key = key0.as_u64;
1800
1801               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1802                 {
1803                   /* Try to match static mapping by external address and port,
1804                      destination address and port in packet */
1805                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1806                     {
1807                       if (!sm->forwarding_enabled)
1808                         {
1809                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1810                           /*
1811                            * Send DHCP packets to the ipv4 stack, or we won't
1812                            * be able to use dhcp client on the outside interface
1813                            */
1814                           if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_UDP
1815                               || (udp0->dst_port
1816                                   != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1817                             next0 = SNAT_OUT2IN_NEXT_DROP;
1818                           else
1819                             vnet_feature_next
1820                               (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1821                                &next0, b0);
1822                           goto trace0;
1823                         }
1824                       else
1825                         {
1826                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1827                             {
1828                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1829                               goto trace0;
1830                             }
1831                           create_bypass_for_fwd(sm, ip0);
1832                           goto trace0;
1833                         }
1834                     }
1835
1836                   /* Create session initiated by host from external network */
1837                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1838                                                          thread_index);
1839                   if (!s0)
1840                     {
1841                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1842                       next0 = SNAT_OUT2IN_NEXT_DROP;
1843                       goto trace0;
1844                     }
1845                   reass0->sess_index = s0 - per_thread_data->sessions;
1846                   reass0->thread_index = thread_index;
1847                 }
1848               else
1849                 {
1850                   s0 = pool_elt_at_index (per_thread_data->sessions,
1851                                           value0.value);
1852                   reass0->sess_index = value0.value;
1853                 }
1854               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1855             }
1856           else
1857             {
1858               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1859                 {
1860                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1861                     {
1862                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1863                       next0 = SNAT_OUT2IN_NEXT_DROP;
1864                       goto trace0;
1865                     }
1866                   cached0 = 1;
1867                   goto trace0;
1868                 }
1869               s0 = pool_elt_at_index (per_thread_data->sessions,
1870                                       reass0->sess_index);
1871             }
1872
1873           old_addr0 = ip0->dst_address.as_u32;
1874           ip0->dst_address = s0->in2out.addr;
1875           new_addr0 = ip0->dst_address.as_u32;
1876           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1877
1878           sum0 = ip0->checksum;
1879           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1880                                  ip4_header_t,
1881                                  dst_address /* changed member */);
1882           ip0->checksum = ip_csum_fold (sum0);
1883
1884           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1885             {
1886               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1887                 {
1888                   old_port0 = tcp0->dst_port;
1889                   tcp0->dst_port = s0->in2out.port;
1890                   new_port0 = tcp0->dst_port;
1891
1892                   sum0 = tcp0->checksum;
1893                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1894                                          ip4_header_t,
1895                                          dst_address /* changed member */);
1896
1897                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1898                                          ip4_header_t /* cheat */,
1899                                          length /* changed member */);
1900                   tcp0->checksum = ip_csum_fold(sum0);
1901                 }
1902               else
1903                 {
1904                   old_port0 = udp0->dst_port;
1905                   udp0->dst_port = s0->in2out.port;
1906                   udp0->checksum = 0;
1907                 }
1908             }
1909
1910           /* Accounting */
1911           s0->last_heard = now;
1912           s0->total_pkts++;
1913           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1914           /* Per-user LRU list maintenance */
1915           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1916                              s0->per_user_index);
1917           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1918                               s0->per_user_list_head_index,
1919                               s0->per_user_index);
1920
1921         trace0:
1922           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1923                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1924             {
1925               nat44_out2in_reass_trace_t *t =
1926                  vlib_add_trace (vm, node, b0, sizeof (*t));
1927               t->cached = cached0;
1928               t->sw_if_index = sw_if_index0;
1929               t->next_index = next0;
1930             }
1931
1932           if (cached0)
1933             {
1934               n_left_to_next++;
1935               to_next--;
1936             }
1937           else
1938             {
1939               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1940
1941               /* verify speculative enqueue, maybe switch current next frame */
1942               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1943                                                to_next, n_left_to_next,
1944                                                bi0, next0);
1945             }
1946
1947           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1948             {
1949               from = vlib_frame_vector_args (frame);
1950               u32 len = vec_len (fragments_to_loopback);
1951               if (len <= VLIB_FRAME_SIZE)
1952                 {
1953                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1954                   n_left_from = len;
1955                   vec_reset_length (fragments_to_loopback);
1956                 }
1957               else
1958                 {
1959                   clib_memcpy (from,
1960                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1961                                sizeof (u32) * VLIB_FRAME_SIZE);
1962                   n_left_from = VLIB_FRAME_SIZE;
1963                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1964                 }
1965             }
1966        }
1967
1968       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1969     }
1970
1971   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1972                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1973                                pkts_processed);
1974
1975   nat_send_all_to_node (vm, fragments_to_drop, node,
1976                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1977                         SNAT_OUT2IN_NEXT_DROP);
1978
1979   vec_free (fragments_to_drop);
1980   vec_free (fragments_to_loopback);
1981   return frame->n_vectors;
1982 }
1983
1984 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1985   .function = nat44_out2in_reass_node_fn,
1986   .name = "nat44-out2in-reass",
1987   .vector_size = sizeof (u32),
1988   .format_trace = format_nat44_out2in_reass_trace,
1989   .type = VLIB_NODE_TYPE_INTERNAL,
1990
1991   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1992   .error_strings = snat_out2in_error_strings,
1993
1994   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1995
1996   /* edit / add dispositions here */
1997   .next_nodes = {
1998     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1999     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2000     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2001     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2002     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2003   },
2004 };
2005 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2006                               nat44_out2in_reass_node_fn);
2007
2008 /**************************/
2009 /*** deterministic mode ***/
2010 /**************************/
2011 static uword
2012 snat_det_out2in_node_fn (vlib_main_t * vm,
2013                          vlib_node_runtime_t * node,
2014                          vlib_frame_t * frame)
2015 {
2016   u32 n_left_from, * from, * to_next;
2017   snat_out2in_next_t next_index;
2018   u32 pkts_processed = 0;
2019   snat_main_t * sm = &snat_main;
2020   u32 thread_index = vlib_get_thread_index ();
2021
2022   from = vlib_frame_vector_args (frame);
2023   n_left_from = frame->n_vectors;
2024   next_index = node->cached_next_index;
2025
2026   while (n_left_from > 0)
2027     {
2028       u32 n_left_to_next;
2029
2030       vlib_get_next_frame (vm, node, next_index,
2031                            to_next, n_left_to_next);
2032
2033       while (n_left_from >= 4 && n_left_to_next >= 2)
2034         {
2035           u32 bi0, bi1;
2036           vlib_buffer_t * b0, * b1;
2037           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2038           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2039           u32 sw_if_index0, sw_if_index1;
2040           ip4_header_t * ip0, * ip1;
2041           ip_csum_t sum0, sum1;
2042           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2043           u16 new_port0, old_port0, old_port1, new_port1;
2044           udp_header_t * udp0, * udp1;
2045           tcp_header_t * tcp0, * tcp1;
2046           u32 proto0, proto1;
2047           snat_det_out_key_t key0, key1;
2048           snat_det_map_t * dm0, * dm1;
2049           snat_det_session_t * ses0 = 0, * ses1 = 0;
2050           u32 rx_fib_index0, rx_fib_index1;
2051           icmp46_header_t * icmp0, * icmp1;
2052
2053           /* Prefetch next iteration. */
2054           {
2055             vlib_buffer_t * p2, * p3;
2056
2057             p2 = vlib_get_buffer (vm, from[2]);
2058             p3 = vlib_get_buffer (vm, from[3]);
2059
2060             vlib_prefetch_buffer_header (p2, LOAD);
2061             vlib_prefetch_buffer_header (p3, LOAD);
2062
2063             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2064             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2065           }
2066
2067           /* speculatively enqueue b0 and b1 to the current next frame */
2068           to_next[0] = bi0 = from[0];
2069           to_next[1] = bi1 = from[1];
2070           from += 2;
2071           to_next += 2;
2072           n_left_from -= 2;
2073           n_left_to_next -= 2;
2074
2075           b0 = vlib_get_buffer (vm, bi0);
2076           b1 = vlib_get_buffer (vm, bi1);
2077
2078           ip0 = vlib_buffer_get_current (b0);
2079           udp0 = ip4_next_header (ip0);
2080           tcp0 = (tcp_header_t *) udp0;
2081
2082           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2083
2084           if (PREDICT_FALSE(ip0->ttl == 1))
2085             {
2086               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2087               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2088                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2089                                            0);
2090               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2091               goto trace0;
2092             }
2093
2094           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2095
2096           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2097             {
2098               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2099               icmp0 = (icmp46_header_t *) udp0;
2100
2101               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2102                                   rx_fib_index0, node, next0, thread_index,
2103                                   &ses0, &dm0);
2104               goto trace0;
2105             }
2106
2107           key0.ext_host_addr = ip0->src_address;
2108           key0.ext_host_port = tcp0->src;
2109           key0.out_port = tcp0->dst;
2110
2111           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2112           if (PREDICT_FALSE(!dm0))
2113             {
2114               clib_warning("unknown dst address:  %U",
2115                            format_ip4_address, &ip0->dst_address);
2116               next0 = SNAT_OUT2IN_NEXT_DROP;
2117               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2118               goto trace0;
2119             }
2120
2121           snat_det_reverse(dm0, &ip0->dst_address,
2122                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2123
2124           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2125           if (PREDICT_FALSE(!ses0))
2126             {
2127               clib_warning("no match src %U:%d dst %U:%d for user %U",
2128                            format_ip4_address, &ip0->src_address,
2129                            clib_net_to_host_u16 (tcp0->src),
2130                            format_ip4_address, &ip0->dst_address,
2131                            clib_net_to_host_u16 (tcp0->dst),
2132                            format_ip4_address, &new_addr0);
2133               next0 = SNAT_OUT2IN_NEXT_DROP;
2134               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2135               goto trace0;
2136             }
2137           new_port0 = ses0->in_port;
2138
2139           old_addr0 = ip0->dst_address;
2140           ip0->dst_address = new_addr0;
2141           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2142
2143           sum0 = ip0->checksum;
2144           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2145                                  ip4_header_t,
2146                                  dst_address /* changed member */);
2147           ip0->checksum = ip_csum_fold (sum0);
2148
2149           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2150             {
2151               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2152                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2153               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2154                 snat_det_ses_close(dm0, ses0);
2155
2156               old_port0 = tcp0->dst;
2157               tcp0->dst = new_port0;
2158
2159               sum0 = tcp0->checksum;
2160               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2161                                      ip4_header_t,
2162                                      dst_address /* changed member */);
2163
2164               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2165                                      ip4_header_t /* cheat */,
2166                                      length /* changed member */);
2167               tcp0->checksum = ip_csum_fold(sum0);
2168             }
2169           else
2170             {
2171               old_port0 = udp0->dst_port;
2172               udp0->dst_port = new_port0;
2173               udp0->checksum = 0;
2174             }
2175
2176         trace0:
2177
2178           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2179                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2180             {
2181               snat_out2in_trace_t *t =
2182                  vlib_add_trace (vm, node, b0, sizeof (*t));
2183               t->sw_if_index = sw_if_index0;
2184               t->next_index = next0;
2185               t->session_index = ~0;
2186               if (ses0)
2187                 t->session_index = ses0 - dm0->sessions;
2188             }
2189
2190           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2191
2192           b1 = vlib_get_buffer (vm, bi1);
2193
2194           ip1 = vlib_buffer_get_current (b1);
2195           udp1 = ip4_next_header (ip1);
2196           tcp1 = (tcp_header_t *) udp1;
2197
2198           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2199
2200           if (PREDICT_FALSE(ip1->ttl == 1))
2201             {
2202               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2203               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2204                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2205                                            0);
2206               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2207               goto trace1;
2208             }
2209
2210           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2211
2212           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2213             {
2214               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2215               icmp1 = (icmp46_header_t *) udp1;
2216
2217               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2218                                   rx_fib_index1, node, next1, thread_index,
2219                                   &ses1, &dm1);
2220               goto trace1;
2221             }
2222
2223           key1.ext_host_addr = ip1->src_address;
2224           key1.ext_host_port = tcp1->src;
2225           key1.out_port = tcp1->dst;
2226
2227           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2228           if (PREDICT_FALSE(!dm1))
2229             {
2230               clib_warning("unknown dst address:  %U",
2231                            format_ip4_address, &ip1->dst_address);
2232               next1 = SNAT_OUT2IN_NEXT_DROP;
2233               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2234               goto trace1;
2235             }
2236
2237           snat_det_reverse(dm1, &ip1->dst_address,
2238                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2239
2240           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2241           if (PREDICT_FALSE(!ses1))
2242             {
2243               clib_warning("no match src %U:%d dst %U:%d for user %U",
2244                            format_ip4_address, &ip1->src_address,
2245                            clib_net_to_host_u16 (tcp1->src),
2246                            format_ip4_address, &ip1->dst_address,
2247                            clib_net_to_host_u16 (tcp1->dst),
2248                            format_ip4_address, &new_addr1);
2249               next1 = SNAT_OUT2IN_NEXT_DROP;
2250               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2251               goto trace1;
2252             }
2253           new_port1 = ses1->in_port;
2254
2255           old_addr1 = ip1->dst_address;
2256           ip1->dst_address = new_addr1;
2257           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2258
2259           sum1 = ip1->checksum;
2260           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2261                                  ip4_header_t,
2262                                  dst_address /* changed member */);
2263           ip1->checksum = ip_csum_fold (sum1);
2264
2265           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2266             {
2267               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2268                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2269               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2270                 snat_det_ses_close(dm1, ses1);
2271
2272               old_port1 = tcp1->dst;
2273               tcp1->dst = new_port1;
2274
2275               sum1 = tcp1->checksum;
2276               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2277                                      ip4_header_t,
2278                                      dst_address /* changed member */);
2279
2280               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2281                                      ip4_header_t /* cheat */,
2282                                      length /* changed member */);
2283               tcp1->checksum = ip_csum_fold(sum1);
2284             }
2285           else
2286             {
2287               old_port1 = udp1->dst_port;
2288               udp1->dst_port = new_port1;
2289               udp1->checksum = 0;
2290             }
2291
2292         trace1:
2293
2294           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2295                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2296             {
2297               snat_out2in_trace_t *t =
2298                  vlib_add_trace (vm, node, b1, sizeof (*t));
2299               t->sw_if_index = sw_if_index1;
2300               t->next_index = next1;
2301               t->session_index = ~0;
2302               if (ses1)
2303                 t->session_index = ses1 - dm1->sessions;
2304             }
2305
2306           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2307
2308           /* verify speculative enqueues, maybe switch current next frame */
2309           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2310                                            to_next, n_left_to_next,
2311                                            bi0, bi1, next0, next1);
2312          }
2313
2314       while (n_left_from > 0 && n_left_to_next > 0)
2315         {
2316           u32 bi0;
2317           vlib_buffer_t * b0;
2318           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2319           u32 sw_if_index0;
2320           ip4_header_t * ip0;
2321           ip_csum_t sum0;
2322           ip4_address_t new_addr0, old_addr0;
2323           u16 new_port0, old_port0;
2324           udp_header_t * udp0;
2325           tcp_header_t * tcp0;
2326           u32 proto0;
2327           snat_det_out_key_t key0;
2328           snat_det_map_t * dm0;
2329           snat_det_session_t * ses0 = 0;
2330           u32 rx_fib_index0;
2331           icmp46_header_t * icmp0;
2332
2333           /* speculatively enqueue b0 to the current next frame */
2334           bi0 = from[0];
2335           to_next[0] = bi0;
2336           from += 1;
2337           to_next += 1;
2338           n_left_from -= 1;
2339           n_left_to_next -= 1;
2340
2341           b0 = vlib_get_buffer (vm, bi0);
2342
2343           ip0 = vlib_buffer_get_current (b0);
2344           udp0 = ip4_next_header (ip0);
2345           tcp0 = (tcp_header_t *) udp0;
2346
2347           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2348
2349           if (PREDICT_FALSE(ip0->ttl == 1))
2350             {
2351               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2352               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2353                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2354                                            0);
2355               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2356               goto trace00;
2357             }
2358
2359           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2360
2361           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2362             {
2363               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2364               icmp0 = (icmp46_header_t *) udp0;
2365
2366               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2367                                   rx_fib_index0, node, next0, thread_index,
2368                                   &ses0, &dm0);
2369               goto trace00;
2370             }
2371
2372           key0.ext_host_addr = ip0->src_address;
2373           key0.ext_host_port = tcp0->src;
2374           key0.out_port = tcp0->dst;
2375
2376           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2377           if (PREDICT_FALSE(!dm0))
2378             {
2379               clib_warning("unknown dst address:  %U",
2380                            format_ip4_address, &ip0->dst_address);
2381               next0 = SNAT_OUT2IN_NEXT_DROP;
2382               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2383               goto trace00;
2384             }
2385
2386           snat_det_reverse(dm0, &ip0->dst_address,
2387                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2388
2389           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2390           if (PREDICT_FALSE(!ses0))
2391             {
2392               clib_warning("no match src %U:%d dst %U:%d for user %U",
2393                            format_ip4_address, &ip0->src_address,
2394                            clib_net_to_host_u16 (tcp0->src),
2395                            format_ip4_address, &ip0->dst_address,
2396                            clib_net_to_host_u16 (tcp0->dst),
2397                            format_ip4_address, &new_addr0);
2398               next0 = SNAT_OUT2IN_NEXT_DROP;
2399               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2400               goto trace00;
2401             }
2402           new_port0 = ses0->in_port;
2403
2404           old_addr0 = ip0->dst_address;
2405           ip0->dst_address = new_addr0;
2406           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2407
2408           sum0 = ip0->checksum;
2409           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2410                                  ip4_header_t,
2411                                  dst_address /* changed member */);
2412           ip0->checksum = ip_csum_fold (sum0);
2413
2414           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2415             {
2416               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2417                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2418               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2419                 snat_det_ses_close(dm0, ses0);
2420
2421               old_port0 = tcp0->dst;
2422               tcp0->dst = new_port0;
2423
2424               sum0 = tcp0->checksum;
2425               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2426                                      ip4_header_t,
2427                                      dst_address /* changed member */);
2428
2429               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2430                                      ip4_header_t /* cheat */,
2431                                      length /* changed member */);
2432               tcp0->checksum = ip_csum_fold(sum0);
2433             }
2434           else
2435             {
2436               old_port0 = udp0->dst_port;
2437               udp0->dst_port = new_port0;
2438               udp0->checksum = 0;
2439             }
2440
2441         trace00:
2442
2443           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2444                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2445             {
2446               snat_out2in_trace_t *t =
2447                  vlib_add_trace (vm, node, b0, sizeof (*t));
2448               t->sw_if_index = sw_if_index0;
2449               t->next_index = next0;
2450               t->session_index = ~0;
2451               if (ses0)
2452                 t->session_index = ses0 - dm0->sessions;
2453             }
2454
2455           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2456
2457           /* verify speculative enqueue, maybe switch current next frame */
2458           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2459                                            to_next, n_left_to_next,
2460                                            bi0, next0);
2461         }
2462
2463       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2464     }
2465
2466   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2467                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2468                                pkts_processed);
2469   return frame->n_vectors;
2470 }
2471
2472 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2473   .function = snat_det_out2in_node_fn,
2474   .name = "nat44-det-out2in",
2475   .vector_size = sizeof (u32),
2476   .format_trace = format_snat_out2in_trace,
2477   .type = VLIB_NODE_TYPE_INTERNAL,
2478
2479   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2480   .error_strings = snat_out2in_error_strings,
2481
2482   .runtime_data_bytes = sizeof (snat_runtime_t),
2483
2484   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2485
2486   /* edit / add dispositions here */
2487   .next_nodes = {
2488     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2489     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2490     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2491     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2492     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2493   },
2494 };
2495 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2496
2497 /**
2498  * Get address and port values to be used for ICMP packet translation
2499  * and create session if needed
2500  *
2501  * @param[in,out] sm             NAT main
2502  * @param[in,out] node           NAT node runtime
2503  * @param[in] thread_index       thread index
2504  * @param[in,out] b0             buffer containing packet to be translated
2505  * @param[out] p_proto           protocol used for matching
2506  * @param[out] p_value           address and port after NAT translation
2507  * @param[out] p_dont_translate  if packet should not be translated
2508  * @param d                      optional parameter
2509  * @param e                      optional parameter
2510  */
2511 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2512                           u32 thread_index, vlib_buffer_t *b0,
2513                           ip4_header_t *ip0, u8 *p_proto,
2514                           snat_session_key_t *p_value,
2515                           u8 *p_dont_translate, void *d, void *e)
2516 {
2517   icmp46_header_t *icmp0;
2518   u32 sw_if_index0;
2519   u8 protocol;
2520   snat_det_out_key_t key0;
2521   u8 dont_translate = 0;
2522   u32 next0 = ~0;
2523   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2524   ip4_header_t *inner_ip0;
2525   void *l4_header = 0;
2526   icmp46_header_t *inner_icmp0;
2527   snat_det_map_t * dm0 = 0;
2528   ip4_address_t new_addr0 = {{0}};
2529   snat_det_session_t * ses0 = 0;
2530   ip4_address_t out_addr;
2531
2532   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2533   echo0 = (icmp_echo_header_t *)(icmp0+1);
2534   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2535
2536   if (!icmp_is_error_message (icmp0))
2537     {
2538       protocol = SNAT_PROTOCOL_ICMP;
2539       key0.ext_host_addr = ip0->src_address;
2540       key0.ext_host_port = 0;
2541       key0.out_port = echo0->identifier;
2542       out_addr = ip0->dst_address;
2543     }
2544   else
2545     {
2546       inner_ip0 = (ip4_header_t *)(echo0+1);
2547       l4_header = ip4_next_header (inner_ip0);
2548       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2549       key0.ext_host_addr = inner_ip0->dst_address;
2550       out_addr = inner_ip0->src_address;
2551       switch (protocol)
2552         {
2553         case SNAT_PROTOCOL_ICMP:
2554           inner_icmp0 = (icmp46_header_t*)l4_header;
2555           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2556           key0.ext_host_port = 0;
2557           key0.out_port = inner_echo0->identifier;
2558           break;
2559         case SNAT_PROTOCOL_UDP:
2560         case SNAT_PROTOCOL_TCP:
2561           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2562           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2563           break;
2564         default:
2565           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2566           next0 = SNAT_OUT2IN_NEXT_DROP;
2567           goto out;
2568         }
2569     }
2570
2571   dm0 = snat_det_map_by_out(sm, &out_addr);
2572   if (PREDICT_FALSE(!dm0))
2573     {
2574       /* Don't NAT packet aimed at the intfc address */
2575       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2576                                           ip0->dst_address.as_u32)))
2577         {
2578           dont_translate = 1;
2579           goto out;
2580         }
2581       clib_warning("unknown dst address:  %U",
2582                    format_ip4_address, &ip0->dst_address);
2583       goto out;
2584     }
2585
2586   snat_det_reverse(dm0, &ip0->dst_address,
2587                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2588
2589   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2590   if (PREDICT_FALSE(!ses0))
2591     {
2592       /* Don't NAT packet aimed at the intfc address */
2593       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2594                                           ip0->dst_address.as_u32)))
2595         {
2596           dont_translate = 1;
2597           goto out;
2598         }
2599       clib_warning("no match src %U:%d dst %U:%d for user %U",
2600                    format_ip4_address, &key0.ext_host_addr,
2601                    clib_net_to_host_u16 (key0.ext_host_port),
2602                    format_ip4_address, &out_addr,
2603                    clib_net_to_host_u16 (key0.out_port),
2604                    format_ip4_address, &new_addr0);
2605       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2606       next0 = SNAT_OUT2IN_NEXT_DROP;
2607       goto out;
2608     }
2609
2610   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2611                     !icmp_is_error_message (icmp0)))
2612     {
2613       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2614       next0 = SNAT_OUT2IN_NEXT_DROP;
2615       goto out;
2616     }
2617
2618   goto out;
2619
2620 out:
2621   *p_proto = protocol;
2622   if (ses0)
2623     {
2624       p_value->addr = new_addr0;
2625       p_value->fib_index = sm->inside_fib_index;
2626       p_value->port = ses0->in_port;
2627     }
2628   *p_dont_translate = dont_translate;
2629   if (d)
2630     *(snat_det_session_t**)d = ses0;
2631   if (e)
2632     *(snat_det_map_t**)e = dm0;
2633   return next0;
2634 }
2635
2636 /**********************/
2637 /*** worker handoff ***/
2638 /**********************/
2639 static uword
2640 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2641                                vlib_node_runtime_t * node,
2642                                vlib_frame_t * frame)
2643 {
2644   snat_main_t *sm = &snat_main;
2645   vlib_thread_main_t *tm = vlib_get_thread_main ();
2646   u32 n_left_from, *from, *to_next = 0;
2647   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2648   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2649     = 0;
2650   vlib_frame_queue_elt_t *hf = 0;
2651   vlib_frame_t *f = 0;
2652   int i;
2653   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2654   u32 next_worker_index = 0;
2655   u32 current_worker_index = ~0;
2656   u32 thread_index = vlib_get_thread_index ();
2657
2658   ASSERT (vec_len (sm->workers));
2659
2660   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2661     {
2662       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2663
2664       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2665                                sm->first_worker_index + sm->num_workers - 1,
2666                                (vlib_frame_queue_t *) (~0));
2667     }
2668
2669   from = vlib_frame_vector_args (frame);
2670   n_left_from = frame->n_vectors;
2671
2672   while (n_left_from > 0)
2673     {
2674       u32 bi0;
2675       vlib_buffer_t *b0;
2676       u32 sw_if_index0;
2677       u32 rx_fib_index0;
2678       ip4_header_t * ip0;
2679       u8 do_handoff;
2680
2681       bi0 = from[0];
2682       from += 1;
2683       n_left_from -= 1;
2684
2685       b0 = vlib_get_buffer (vm, bi0);
2686
2687       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2688       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2689
2690       ip0 = vlib_buffer_get_current (b0);
2691
2692       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2693
2694       if (PREDICT_FALSE (next_worker_index != thread_index))
2695         {
2696           do_handoff = 1;
2697
2698           if (next_worker_index != current_worker_index)
2699             {
2700               if (hf)
2701                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2702
2703               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2704                                                       next_worker_index,
2705                                                       handoff_queue_elt_by_worker_index);
2706
2707               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2708               to_next_worker = &hf->buffer_index[hf->n_vectors];
2709               current_worker_index = next_worker_index;
2710             }
2711
2712           /* enqueue to correct worker thread */
2713           to_next_worker[0] = bi0;
2714           to_next_worker++;
2715           n_left_to_next_worker--;
2716
2717           if (n_left_to_next_worker == 0)
2718             {
2719               hf->n_vectors = VLIB_FRAME_SIZE;
2720               vlib_put_frame_queue_elt (hf);
2721               current_worker_index = ~0;
2722               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2723               hf = 0;
2724             }
2725         }
2726       else
2727         {
2728           do_handoff = 0;
2729           /* if this is 1st frame */
2730           if (!f)
2731             {
2732               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2733               to_next = vlib_frame_vector_args (f);
2734             }
2735
2736           to_next[0] = bi0;
2737           to_next += 1;
2738           f->n_vectors++;
2739         }
2740
2741       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2742                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2743         {
2744           snat_out2in_worker_handoff_trace_t *t =
2745             vlib_add_trace (vm, node, b0, sizeof (*t));
2746           t->next_worker_index = next_worker_index;
2747           t->do_handoff = do_handoff;
2748         }
2749     }
2750
2751   if (f)
2752     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2753
2754   if (hf)
2755     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2756
2757   /* Ship frames to the worker nodes */
2758   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2759     {
2760       if (handoff_queue_elt_by_worker_index[i])
2761         {
2762           hf = handoff_queue_elt_by_worker_index[i];
2763           /*
2764            * It works better to let the handoff node
2765            * rate-adapt, always ship the handoff queue element.
2766            */
2767           if (1 || hf->n_vectors == hf->last_n_vectors)
2768             {
2769               vlib_put_frame_queue_elt (hf);
2770               handoff_queue_elt_by_worker_index[i] = 0;
2771             }
2772           else
2773             hf->last_n_vectors = hf->n_vectors;
2774         }
2775       congested_handoff_queue_by_worker_index[i] =
2776         (vlib_frame_queue_t *) (~0);
2777     }
2778   hf = 0;
2779   current_worker_index = ~0;
2780   return frame->n_vectors;
2781 }
2782
2783 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2784   .function = snat_out2in_worker_handoff_fn,
2785   .name = "nat44-out2in-worker-handoff",
2786   .vector_size = sizeof (u32),
2787   .format_trace = format_snat_out2in_worker_handoff_trace,
2788   .type = VLIB_NODE_TYPE_INTERNAL,
2789
2790   .n_next_nodes = 1,
2791
2792   .next_nodes = {
2793     [0] = "error-drop",
2794   },
2795 };
2796
2797 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2798
2799 static uword
2800 snat_out2in_fast_node_fn (vlib_main_t * vm,
2801                           vlib_node_runtime_t * node,
2802                           vlib_frame_t * frame)
2803 {
2804   u32 n_left_from, * from, * to_next;
2805   snat_out2in_next_t next_index;
2806   u32 pkts_processed = 0;
2807   snat_main_t * sm = &snat_main;
2808
2809   from = vlib_frame_vector_args (frame);
2810   n_left_from = frame->n_vectors;
2811   next_index = node->cached_next_index;
2812
2813   while (n_left_from > 0)
2814     {
2815       u32 n_left_to_next;
2816
2817       vlib_get_next_frame (vm, node, next_index,
2818                            to_next, n_left_to_next);
2819
2820       while (n_left_from > 0 && n_left_to_next > 0)
2821         {
2822           u32 bi0;
2823           vlib_buffer_t * b0;
2824           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2825           u32 sw_if_index0;
2826           ip4_header_t * ip0;
2827           ip_csum_t sum0;
2828           u32 new_addr0, old_addr0;
2829           u16 new_port0, old_port0;
2830           udp_header_t * udp0;
2831           tcp_header_t * tcp0;
2832           icmp46_header_t * icmp0;
2833           snat_session_key_t key0, sm0;
2834           u32 proto0;
2835           u32 rx_fib_index0;
2836
2837           /* speculatively enqueue b0 to the current next frame */
2838           bi0 = from[0];
2839           to_next[0] = bi0;
2840           from += 1;
2841           to_next += 1;
2842           n_left_from -= 1;
2843           n_left_to_next -= 1;
2844
2845           b0 = vlib_get_buffer (vm, bi0);
2846
2847           ip0 = vlib_buffer_get_current (b0);
2848           udp0 = ip4_next_header (ip0);
2849           tcp0 = (tcp_header_t *) udp0;
2850           icmp0 = (icmp46_header_t *) udp0;
2851
2852           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2853           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2854
2855           vnet_feature_next (sw_if_index0, &next0, b0);
2856
2857           if (PREDICT_FALSE(ip0->ttl == 1))
2858             {
2859               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2860               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2861                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2862                                            0);
2863               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2864               goto trace00;
2865             }
2866
2867           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2868
2869           if (PREDICT_FALSE (proto0 == ~0))
2870               goto trace00;
2871
2872           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2873             {
2874               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2875                                   rx_fib_index0, node, next0, ~0, 0, 0);
2876               goto trace00;
2877             }
2878
2879           key0.addr = ip0->dst_address;
2880           key0.port = udp0->dst_port;
2881           key0.fib_index = rx_fib_index0;
2882
2883           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2884             {
2885               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2886               goto trace00;
2887             }
2888
2889           new_addr0 = sm0.addr.as_u32;
2890           new_port0 = sm0.port;
2891           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2892           old_addr0 = ip0->dst_address.as_u32;
2893           ip0->dst_address.as_u32 = new_addr0;
2894
2895           sum0 = ip0->checksum;
2896           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2897                                  ip4_header_t,
2898                                  dst_address /* changed member */);
2899           ip0->checksum = ip_csum_fold (sum0);
2900
2901           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2902             {
2903                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2904                 {
2905                   old_port0 = tcp0->dst_port;
2906                   tcp0->dst_port = new_port0;
2907
2908                   sum0 = tcp0->checksum;
2909                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2910                                          ip4_header_t,
2911                                          dst_address /* changed member */);
2912
2913                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2914                                          ip4_header_t /* cheat */,
2915                                          length /* changed member */);
2916                   tcp0->checksum = ip_csum_fold(sum0);
2917                 }
2918               else
2919                 {
2920                   old_port0 = udp0->dst_port;
2921                   udp0->dst_port = new_port0;
2922                   udp0->checksum = 0;
2923                 }
2924             }
2925           else
2926             {
2927               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2928                 {
2929                   sum0 = tcp0->checksum;
2930                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2931                                          ip4_header_t,
2932                                          dst_address /* changed member */);
2933
2934                   tcp0->checksum = ip_csum_fold(sum0);
2935                 }
2936             }
2937
2938         trace00:
2939
2940           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2941                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2942             {
2943               snat_out2in_trace_t *t =
2944                  vlib_add_trace (vm, node, b0, sizeof (*t));
2945               t->sw_if_index = sw_if_index0;
2946               t->next_index = next0;
2947             }
2948
2949           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2950
2951           /* verify speculative enqueue, maybe switch current next frame */
2952           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2953                                            to_next, n_left_to_next,
2954                                            bi0, next0);
2955         }
2956
2957       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2958     }
2959
2960   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2961                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2962                                pkts_processed);
2963   return frame->n_vectors;
2964 }
2965
2966 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2967   .function = snat_out2in_fast_node_fn,
2968   .name = "nat44-out2in-fast",
2969   .vector_size = sizeof (u32),
2970   .format_trace = format_snat_out2in_fast_trace,
2971   .type = VLIB_NODE_TYPE_INTERNAL,
2972
2973   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2974   .error_strings = snat_out2in_error_strings,
2975
2976   .runtime_data_bytes = sizeof (snat_runtime_t),
2977
2978   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2979
2980   /* edit / add dispositions here */
2981   .next_nodes = {
2982     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2983     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2984     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2985     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2986     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2987   },
2988 };
2989 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);