src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   user_session_increment (sm, u, 1 /* static */);
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 337                       u32 thread_index)
 338 {
 339   nat_ed_ses_key_t key;
 340   clib_bihash_kv_16_8_t kv, value;
 341   udp_header_t *udp;
 342   snat_user_t *u;
 343   snat_session_t *s = 0;
 344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 345
 346   if (ip->protocol == IP_PROTOCOL_ICMP)
 347     {
 348       if (icmp_get_ed_key (ip, &key))
 349         return;
 350     }
 351   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 352     {
 353       udp = ip4_next_header(ip);
 354       key.r_addr = ip->src_address;
 355       key.l_addr = ip->dst_address;
 356       key.proto = ip->protocol;
 357       key.l_port = udp->dst_port;
 358       key.r_port = udp->src_port;
 359     }
 360   else
 361     {
 362       key.r_addr = ip->src_address;
 363       key.l_addr = ip->dst_address;
 364       key.proto = ip->protocol;
 365       key.l_port = key.r_port = 0;
 366     }
 367   key.fib_index = 0;
 368   kv.key[0] = key.as_u64[0];
 369   kv.key[1] = key.as_u64[1];
 370
 371   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 372     {
 373       s = pool_elt_at_index (tsm->sessions, value.value);
 374     }
 375   else
 376     {
 377       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 378         return;
 379
 380       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
 381       if (!u)
 382         {
 383           clib_warning ("create NAT user failed");
 384           return;
 385         }
 386
 387       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 388       if (!s)
 389         {
 390           clib_warning ("create NAT session failed");
 391           return;
 392         }
 393
 394       s->ext_host_addr = key.r_addr;
 395       s->ext_host_port = key.r_port;
 396       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 397       s->outside_address_index = ~0;
 398       s->out2in.addr = key.l_addr;
 399       s->out2in.port = key.l_port;
 400       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 401       s->out2in.fib_index = 0;
 402       s->in2out = s->out2in;
 403       user_session_increment (sm, u, 0);
 404
 405       kv.value = s - tsm->sessions;
 406       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 407         clib_warning ("in2out_ed key add failed");
 408     }
 409
 410   if (ip->protocol == IP_PROTOCOL_TCP)
 411     {
 412       tcp_header_t *tcp = ip4_next_header(ip);
 413       if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
 414         return;
 415     }
 416   /* Per-user LRU list maintenance */
 417   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 418   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 419                       s->per_user_index);
 420 }
 421
 422 /**
 423  * Get address and port values to be used for ICMP packet translation
 424  * and create session if needed
 425  *
 426  * @param[in,out] sm             NAT main
 427  * @param[in,out] node           NAT node runtime
 428  * @param[in] thread_index       thread index
 429  * @param[in,out] b0             buffer containing packet to be translated
 430  * @param[out] p_proto           protocol used for matching
 431  * @param[out] p_value           address and port after NAT translation
 432  * @param[out] p_dont_translate  if packet should not be translated
 433  * @param d                      optional parameter
 434  * @param e                      optional parameter
 435  */
 436 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 437                            u32 thread_index, vlib_buffer_t *b0,
 438                            ip4_header_t *ip0, u8 *p_proto,
 439                            snat_session_key_t *p_value,
 440                            u8 *p_dont_translate, void *d, void *e)
 441 {
 442   icmp46_header_t *icmp0;
 443   u32 sw_if_index0;
 444   u32 rx_fib_index0;
 445   snat_session_key_t key0;
 446   snat_session_key_t sm0;
 447   snat_session_t *s0 = 0;
 448   u8 dont_translate = 0;
 449   clib_bihash_kv_8_8_t kv0, value0;
 450   u8 is_addr_only;
 451   u32 next0 = ~0;
 452   int err;
 453
 454   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 455   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 456   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 457
 458   key0.protocol = 0;
 459
 460   err = icmp_get_key (ip0, &key0);
 461   if (err != -1)
 462     {
 463       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 464       next0 = SNAT_OUT2IN_NEXT_DROP;
 465       goto out;
 466     }
 467   key0.fib_index = rx_fib_index0;
 468
 469   kv0.key = key0.as_u64;
 470
 471   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 472                               &value0))
 473     {
 474       /* Try to match static mapping by external address and port,
 475          destination address and port in packet */
 476       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 477         {
 478           if (!sm->forwarding_enabled)
 479             {
 480               /* Don't NAT packet aimed at the intfc address */
 481               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 482                                                   ip0->dst_address.as_u32)))
 483                 {
 484                   dont_translate = 1;
 485                   goto out;
 486                 }
 487               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 488               next0 = SNAT_OUT2IN_NEXT_DROP;
 489               goto out;
 490             }
 491           else
 492             {
 493               dont_translate = 1;
 494               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 495                 {
 496                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 497                   goto out;
 498                 }
 499               create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
 500               goto out;
 501             }
 502         }
 503
 504       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 505                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 506         {
 507           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 508           next0 = SNAT_OUT2IN_NEXT_DROP;
 509           goto out;
 510         }
 511
 512       /* Create session initiated by host from external network */
 513       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 514                                              node, thread_index);
 515
 516       if (!s0)
 517         {
 518           next0 = SNAT_OUT2IN_NEXT_DROP;
 519           goto out;
 520         }
 521     }
 522   else
 523     {
 524       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 525                         icmp0->type != ICMP4_echo_request &&
 526                         !icmp_is_error_message (icmp0)))
 527         {
 528           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 529           next0 = SNAT_OUT2IN_NEXT_DROP;
 530           goto out;
 531         }
 532
 533       if (PREDICT_FALSE (value0.value == ~0ULL))
 534         {
 535           nat_ed_ses_key_t key;
 536           clib_bihash_kv_16_8_t s_kv, s_value;
 537
 538           key.as_u64[0] = 0;
 539           key.as_u64[1] = 0;
 540           if (icmp_get_ed_key (ip0, &key))
 541             {
 542               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 543               next0 = SNAT_OUT2IN_NEXT_DROP;
 544               goto out;
 545             }
 546           key.fib_index = rx_fib_index0;
 547           s_kv.key[0] = key.as_u64[0];
 548           s_kv.key[1] = key.as_u64[1];
 549           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 550             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 551                                     s_value.value);
 552           else
 553            {
 554               next0 = SNAT_OUT2IN_NEXT_DROP;
 555               goto out;
 556            }
 557         }
 558       else
 559         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 560                                 value0.value);
 561     }
 562
 563 out:
 564   *p_proto = key0.protocol;
 565   if (s0)
 566     *p_value = s0->in2out;
 567   *p_dont_translate = dont_translate;
 568   if (d)
 569     *(snat_session_t**)d = s0;
 570   return next0;
 571 }
 572
 573 /**
 574  * Get address and port values to be used for ICMP packet translation
 575  *
 576  * @param[in] sm                 NAT main
 577  * @param[in,out] node           NAT node runtime
 578  * @param[in] thread_index       thread index
 579  * @param[in,out] b0             buffer containing packet to be translated
 580  * @param[out] p_proto           protocol used for matching
 581  * @param[out] p_value           address and port after NAT translation
 582  * @param[out] p_dont_translate  if packet should not be translated
 583  * @param d                      optional parameter
 584  * @param e                      optional parameter
 585  */
 586 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 587                            u32 thread_index, vlib_buffer_t *b0,
 588                            ip4_header_t *ip0, u8 *p_proto,
 589                            snat_session_key_t *p_value,
 590                            u8 *p_dont_translate, void *d, void *e)
 591 {
 592   icmp46_header_t *icmp0;
 593   u32 sw_if_index0;
 594   u32 rx_fib_index0;
 595   snat_session_key_t key0;
 596   snat_session_key_t sm0;
 597   u8 dont_translate = 0;
 598   u8 is_addr_only;
 599   u32 next0 = ~0;
 600   int err;
 601
 602   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 603   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 604   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 605
 606   err = icmp_get_key (ip0, &key0);
 607   if (err != -1)
 608     {
 609       b0->error = node->errors[err];
 610       next0 = SNAT_OUT2IN_NEXT_DROP;
 611       goto out2;
 612     }
 613   key0.fib_index = rx_fib_index0;
 614
 615   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 616     {
 617       /* Don't NAT packet aimed at the intfc address */
 618       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 619         {
 620           dont_translate = 1;
 621           goto out;
 622         }
 623       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 624       next0 = SNAT_OUT2IN_NEXT_DROP;
 625       goto out;
 626     }
 627
 628   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 629                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 630                     !icmp_is_error_message (icmp0)))
 631     {
 632       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 633       next0 = SNAT_OUT2IN_NEXT_DROP;
 634       goto out;
 635     }
 636
 637 out:
 638   *p_value = sm0;
 639 out2:
 640   *p_proto = key0.protocol;
 641   *p_dont_translate = dont_translate;
 642   return next0;
 643 }
 644
 645 static inline u32 icmp_out2in (snat_main_t *sm,
 646                                vlib_buffer_t * b0,
 647                                ip4_header_t * ip0,
 648                                icmp46_header_t * icmp0,
 649                                u32 sw_if_index0,
 650                                u32 rx_fib_index0,
 651                                vlib_node_runtime_t * node,
 652                                u32 next0,
 653                                u32 thread_index,
 654                                void *d,
 655                                void *e)
 656 {
 657   snat_session_key_t sm0;
 658   u8 protocol;
 659   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 660   ip4_header_t *inner_ip0 = 0;
 661   void *l4_header = 0;
 662   icmp46_header_t *inner_icmp0;
 663   u8 dont_translate;
 664   u32 new_addr0, old_addr0;
 665   u16 old_id0, new_id0;
 666   ip_csum_t sum0;
 667   u16 checksum0;
 668   u32 next0_tmp;
 669
 670   echo0 = (icmp_echo_header_t *)(icmp0+1);
 671
 672   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 673                                        &protocol, &sm0, &dont_translate, d, e);
 674   if (next0_tmp != ~0)
 675     next0 = next0_tmp;
 676   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 677     goto out;
 678
 679   sum0 = ip_incremental_checksum (0, icmp0,
 680                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 681   checksum0 = ~ip_csum_fold (sum0);
 682   if (checksum0 != 0 && checksum0 != 0xffff)
 683     {
 684       next0 = SNAT_OUT2IN_NEXT_DROP;
 685       goto out;
 686     }
 687
 688   old_addr0 = ip0->dst_address.as_u32;
 689   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 690   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 691
 692   sum0 = ip0->checksum;
 693   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 694                          dst_address /* changed member */);
 695   ip0->checksum = ip_csum_fold (sum0);
 696
 697   if (icmp0->checksum == 0)
 698     icmp0->checksum = 0xffff;
 699
 700   if (!icmp_is_error_message (icmp0))
 701     {
 702       new_id0 = sm0.port;
 703       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 704         {
 705           old_id0 = echo0->identifier;
 706           new_id0 = sm0.port;
 707           echo0->identifier = new_id0;
 708
 709           sum0 = icmp0->checksum;
 710           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 711                                  identifier /* changed member */);
 712           icmp0->checksum = ip_csum_fold (sum0);
 713         }
 714     }
 715   else
 716     {
 717       inner_ip0 = (ip4_header_t *)(echo0+1);
 718       l4_header = ip4_next_header (inner_ip0);
 719
 720       if (!ip4_header_checksum_is_valid (inner_ip0))
 721         {
 722           next0 = SNAT_OUT2IN_NEXT_DROP;
 723           goto out;
 724         }
 725
 726       old_addr0 = inner_ip0->src_address.as_u32;
 727       inner_ip0->src_address = sm0.addr;
 728       new_addr0 = inner_ip0->src_address.as_u32;
 729
 730       sum0 = icmp0->checksum;
 731       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 732                              src_address /* changed member */);
 733       icmp0->checksum = ip_csum_fold (sum0);
 734
 735       switch (protocol)
 736         {
 737         case SNAT_PROTOCOL_ICMP:
 738           inner_icmp0 = (icmp46_header_t*)l4_header;
 739           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 740
 741           old_id0 = inner_echo0->identifier;
 742           new_id0 = sm0.port;
 743           inner_echo0->identifier = new_id0;
 744
 745           sum0 = icmp0->checksum;
 746           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 747                                  identifier);
 748           icmp0->checksum = ip_csum_fold (sum0);
 749           break;
 750         case SNAT_PROTOCOL_UDP:
 751         case SNAT_PROTOCOL_TCP:
 752           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 753           new_id0 = sm0.port;
 754           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 755
 756           sum0 = icmp0->checksum;
 757           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 758                                  src_port);
 759           icmp0->checksum = ip_csum_fold (sum0);
 760           break;
 761         default:
 762           ASSERT(0);
 763         }
 764     }
 765
 766 out:
 767   return next0;
 768 }
 769
 770
 771 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 772                                          vlib_buffer_t * b0,
 773                                          ip4_header_t * ip0,
 774                                          icmp46_header_t * icmp0,
 775                                          u32 sw_if_index0,
 776                                          u32 rx_fib_index0,
 777                                          vlib_node_runtime_t * node,
 778                                          u32 next0, f64 now,
 779                                          u32 thread_index,
 780                                          snat_session_t ** p_s0)
 781 {
 782   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 783                       next0, thread_index, p_s0, 0);
 784   snat_session_t * s0 = *p_s0;
 785   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 786     {
 787       /* Accounting */
 788       s0->last_heard = now;
 789       s0->total_pkts++;
 790       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 791       /* Per-user LRU list maintenance */
 792       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 793                          s0->per_user_index);
 794       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 795                           s0->per_user_list_head_index,
 796                           s0->per_user_index);
 797     }
 798   return next0;
 799 }
 800
 801 static snat_session_t *
 802 snat_out2in_unknown_proto (snat_main_t *sm,
 803                            vlib_buffer_t * b,
 804                            ip4_header_t * ip,
 805                            u32 rx_fib_index,
 806                            u32 thread_index,
 807                            f64 now,
 808                            vlib_main_t * vm,
 809                            vlib_node_runtime_t * node)
 810 {
 811   clib_bihash_kv_8_8_t kv, value;
 812   clib_bihash_kv_16_8_t s_kv, s_value;
 813   snat_static_mapping_t *m;
 814   snat_session_key_t m_key;
 815   u32 old_addr, new_addr;
 816   ip_csum_t sum;
 817   nat_ed_ses_key_t key;
 818   snat_session_t * s;
 819   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 820   snat_user_t *u;
 821
 822   old_addr = ip->dst_address.as_u32;
 823
 824   key.l_addr = ip->dst_address;
 825   key.r_addr = ip->src_address;
 826   key.fib_index = rx_fib_index;
 827   key.proto = ip->protocol;
 828   key.r_port = 0;
 829   key.l_port = 0;
 830   s_kv.key[0] = key.as_u64[0];
 831   s_kv.key[1] = key.as_u64[1];
 832
 833   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 834     {
 835       s = pool_elt_at_index (tsm->sessions, s_value.value);
 836       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 837     }
 838   else
 839     {
 840       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 841         {
 842           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 843           return 0;
 844         }
 845
 846       m_key.addr = ip->dst_address;
 847       m_key.port = 0;
 848       m_key.protocol = 0;
 849       m_key.fib_index = rx_fib_index;
 850       kv.key = m_key.as_u64;
 851       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 852         {
 853           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 854           return 0;
 855         }
 856
 857       m = pool_elt_at_index (sm->static_mappings, value.value);
 858
 859       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 860
 861       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 862                                   thread_index);
 863       if (!u)
 864         {
 865           clib_warning ("create NAT user failed");
 866           return 0;
 867         }
 868
 869       /* Create a new session */
 870       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 871       if (!s)
 872         {
 873           clib_warning ("create NAT session failed");
 874           return 0;
 875         }
 876
 877       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 878       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 879       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 880       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 881       s->outside_address_index = ~0;
 882       s->out2in.addr.as_u32 = old_addr;
 883       s->out2in.fib_index = rx_fib_index;
 884       s->in2out.addr.as_u32 = new_addr;
 885       s->in2out.fib_index = m->fib_index;
 886       s->in2out.port = s->out2in.port = ip->protocol;
 887       user_session_increment (sm, u, 1 /* static */);
 888
 889       /* Add to lookup tables */
 890       s_kv.value = s - tsm->sessions;
 891       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 892         clib_warning ("out2in key add failed");
 893
 894       key.l_addr = ip->dst_address;
 895       key.fib_index = m->fib_index;
 896       s_kv.key[0] = key.as_u64[0];
 897       s_kv.key[1] = key.as_u64[1];
 898       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 899         clib_warning ("in2out key add failed");
 900    }
 901
 902   /* Update IP checksum */
 903   sum = ip->checksum;
 904   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 905   ip->checksum = ip_csum_fold (sum);
 906
 907   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 908
 909   /* Accounting */
 910   s->last_heard = now;
 911   s->total_pkts++;
 912   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 913   /* Per-user LRU list maintenance */
 914   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 915   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 916                       s->per_user_index);
 917
 918   return s;
 919 }
 920
 921 static snat_session_t *
 922 snat_out2in_lb (snat_main_t *sm,
 923                 vlib_buffer_t * b,
 924                 ip4_header_t * ip,
 925                 u32 rx_fib_index,
 926                 u32 thread_index,
 927                 f64 now,
 928                 vlib_main_t * vm,
 929                 vlib_node_runtime_t * node)
 930 {
 931   nat_ed_ses_key_t key;
 932   clib_bihash_kv_16_8_t s_kv, s_value;
 933   udp_header_t *udp = ip4_next_header (ip);
 934   tcp_header_t *tcp = (tcp_header_t *) udp;
 935   snat_session_t *s = 0;
 936   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 937   snat_session_key_t e_key, l_key;
 938   u32 old_addr, new_addr;
 939   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 940   u16 new_port, old_port;
 941   ip_csum_t sum;
 942   snat_user_t *u;
 943   u32 address_index;
 944   snat_session_key_t eh_key;
 945   twice_nat_type_t twice_nat;
 946   u8 lb;
 947
 948   old_addr = ip->dst_address.as_u32;
 949
 950   key.l_addr = ip->dst_address;
 951   key.r_addr = ip->src_address;
 952   key.fib_index = rx_fib_index;
 953   key.proto = ip->protocol;
 954   key.r_port = udp->src_port;
 955   key.l_port = udp->dst_port;
 956   s_kv.key[0] = key.as_u64[0];
 957   s_kv.key[1] = key.as_u64[1];
 958
 959   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 960     {
 961       s = pool_elt_at_index (tsm->sessions, s_value.value);
 962     }
 963   else
 964     {
 965       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 966         {
 967           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 968           return 0;
 969         }
 970
 971       e_key.addr = ip->dst_address;
 972       e_key.port = udp->dst_port;
 973       e_key.protocol = proto;
 974       e_key.fib_index = rx_fib_index;
 975       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 976         return 0;
 977
 978       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 979                                   thread_index);
 980       if (!u)
 981       {
 982         clib_warning ("create NAT user failed");
 983         return 0;
 984       }
 985
 986       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 987       if (!s)
 988         {
 989           clib_warning ("create NAT session failed");
 990           return 0;
 991         }
 992
 993       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 994       s->ext_host_port = udp->src_port;
 995       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 996       if (lb)
 997         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 998       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 999       s->outside_address_index = ~0;
1000       s->out2in = e_key;
1001       s->in2out = l_key;
1002       user_session_increment (sm, u, 1 /* static */);
1003
1004       /* Add to lookup tables */
1005       s_kv.value = s - tsm->sessions;
1006       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1007         clib_warning ("out2in-ed key add failed");
1008
1009       if (twice_nat == TWICE_NAT ||
1010           (twice_nat == TWICE_NAT_SELF &&
1011            ip->src_address.as_u32 == l_key.addr.as_u32))
1012         {
1013           eh_key.protocol = proto;
1014           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1015                                                    thread_index, &eh_key,
1016                                                    &address_index,
1017                                                    sm->port_per_thread,
1018                                                    sm->per_thread_data[thread_index].snat_thread_index))
1019             {
1020               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1021               return 0;
1022             }
1023           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1024           key.r_port = s->ext_host_nat_port = eh_key.port;
1025           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1026         }
1027       key.l_addr = l_key.addr;
1028       key.fib_index = l_key.fib_index;
1029       key.l_port = l_key.port;
1030       s_kv.key[0] = key.as_u64[0];
1031       s_kv.key[1] = key.as_u64[1];
1032       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1033         clib_warning ("in2out-ed key add failed");
1034     }
1035
1036   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1037
1038   /* Update IP checksum */
1039   sum = ip->checksum;
1040   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1041   if (is_twice_nat_session (s))
1042     sum = ip_csum_update (sum, ip->src_address.as_u32,
1043                           s->ext_host_nat_addr.as_u32, ip4_header_t,
1044                           src_address);
1045   ip->checksum = ip_csum_fold (sum);
1046
1047   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1048
1049   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1050     {
1051       old_port = tcp->dst_port;
1052       tcp->dst_port = s->in2out.port;
1053       new_port = tcp->dst_port;
1054
1055       sum = tcp->checksum;
1056       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1057       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1058       if (is_twice_nat_session (s))
1059         {
1060           sum = ip_csum_update (sum, ip->src_address.as_u32,
1061                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1062                                 dst_address);
1063           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1064                                 ip4_header_t, length);
1065           tcp->src_port = s->ext_host_nat_port;
1066           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1067         }
1068       tcp->checksum = ip_csum_fold(sum);
1069       if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
1070         return s;
1071     }
1072   else
1073     {
1074       udp->dst_port = s->in2out.port;
1075       if (is_twice_nat_session (s))
1076         {
1077           udp->src_port = s->ext_host_nat_port;
1078           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1079         }
1080       udp->checksum = 0;
1081     }
1082
1083   /* Accounting */
1084   s->last_heard = now;
1085   s->total_pkts++;
1086   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1087   /* Per-user LRU list maintenance */
1088   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1089   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1090                       s->per_user_index);
1091
1092   return s;
1093 }
1094
1095 static uword
1096 snat_out2in_node_fn (vlib_main_t * vm,
1097                   vlib_node_runtime_t * node,
1098                   vlib_frame_t * frame)
1099 {
1100   u32 n_left_from, * from, * to_next;
1101   snat_out2in_next_t next_index;
1102   u32 pkts_processed = 0;
1103   snat_main_t * sm = &snat_main;
1104   f64 now = vlib_time_now (vm);
1105   u32 thread_index = vlib_get_thread_index ();
1106
1107   from = vlib_frame_vector_args (frame);
1108   n_left_from = frame->n_vectors;
1109   next_index = node->cached_next_index;
1110
1111   while (n_left_from > 0)
1112     {
1113       u32 n_left_to_next;
1114
1115       vlib_get_next_frame (vm, node, next_index,
1116                            to_next, n_left_to_next);
1117
1118       while (n_left_from >= 4 && n_left_to_next >= 2)
1119         {
1120           u32 bi0, bi1;
1121           vlib_buffer_t * b0, * b1;
1122           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1123           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1124           u32 sw_if_index0, sw_if_index1;
1125           ip4_header_t * ip0, *ip1;
1126           ip_csum_t sum0, sum1;
1127           u32 new_addr0, old_addr0;
1128           u16 new_port0, old_port0;
1129           u32 new_addr1, old_addr1;
1130           u16 new_port1, old_port1;
1131           udp_header_t * udp0, * udp1;
1132           tcp_header_t * tcp0, * tcp1;
1133           icmp46_header_t * icmp0, * icmp1;
1134           snat_session_key_t key0, key1, sm0, sm1;
1135           u32 rx_fib_index0, rx_fib_index1;
1136           u32 proto0, proto1;
1137           snat_session_t * s0 = 0, * s1 = 0;
1138           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1139
1140           /* Prefetch next iteration. */
1141           {
1142             vlib_buffer_t * p2, * p3;
1143
1144             p2 = vlib_get_buffer (vm, from[2]);
1145             p3 = vlib_get_buffer (vm, from[3]);
1146
1147             vlib_prefetch_buffer_header (p2, LOAD);
1148             vlib_prefetch_buffer_header (p3, LOAD);
1149
1150             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1151             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1152           }
1153
1154           /* speculatively enqueue b0 and b1 to the current next frame */
1155           to_next[0] = bi0 = from[0];
1156           to_next[1] = bi1 = from[1];
1157           from += 2;
1158           to_next += 2;
1159           n_left_from -= 2;
1160           n_left_to_next -= 2;
1161
1162           b0 = vlib_get_buffer (vm, bi0);
1163           b1 = vlib_get_buffer (vm, bi1);
1164
1165           vnet_buffer (b0)->snat.flags = 0;
1166           vnet_buffer (b1)->snat.flags = 0;
1167
1168           ip0 = vlib_buffer_get_current (b0);
1169           udp0 = ip4_next_header (ip0);
1170           tcp0 = (tcp_header_t *) udp0;
1171           icmp0 = (icmp46_header_t *) udp0;
1172
1173           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1174           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1175                                    sw_if_index0);
1176
1177           if (PREDICT_FALSE(ip0->ttl == 1))
1178             {
1179               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1180               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1181                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1182                                            0);
1183               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1184               goto trace0;
1185             }
1186
1187           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1188
1189           if (PREDICT_FALSE (proto0 == ~0))
1190             {
1191               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1192                                              thread_index, now, vm, node);
1193               if (!sm->forwarding_enabled)
1194                 if (!s0)
1195                   next0 = SNAT_OUT2IN_NEXT_DROP;
1196               goto trace0;
1197             }
1198
1199           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1200             {
1201               next0 = icmp_out2in_slow_path
1202                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1203                  next0, now, thread_index, &s0);
1204               goto trace0;
1205             }
1206
1207           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1208             {
1209               next0 = SNAT_OUT2IN_NEXT_REASS;
1210               goto trace0;
1211             }
1212
1213           key0.addr = ip0->dst_address;
1214           key0.port = udp0->dst_port;
1215           key0.protocol = proto0;
1216           key0.fib_index = rx_fib_index0;
1217
1218           kv0.key = key0.as_u64;
1219
1220           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1221                                       &kv0, &value0))
1222             {
1223               /* Try to match static mapping by external address and port,
1224                  destination address and port in packet */
1225               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1226                 {
1227                   /*
1228                    * Send DHCP packets to the ipv4 stack, or we won't
1229                    * be able to use dhcp client on the outside interface
1230                    */
1231                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1232                       && (udp0->dst_port ==
1233                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1234                     {
1235                       vnet_feature_next
1236                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1237                       goto trace0;
1238                     }
1239
1240                   if (!sm->forwarding_enabled)
1241                     {
1242                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1243                       next0 = SNAT_OUT2IN_NEXT_DROP;
1244                       goto trace0;
1245                     }
1246                   else
1247                     {
1248                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1249                         {
1250                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1251                           goto trace0;
1252                         }
1253                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1254                       goto trace0;
1255                     }
1256                 }
1257
1258               /* Create session initiated by host from external network */
1259               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1260                                                      thread_index);
1261               if (!s0)
1262                 {
1263                   next0 = SNAT_OUT2IN_NEXT_DROP;
1264                   goto trace0;
1265                 }
1266             }
1267           else
1268             {
1269               if (PREDICT_FALSE (value0.value == ~0ULL))
1270                 {
1271                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1272                                       now, vm, node);
1273                   if (!s0)
1274                     next0 = SNAT_OUT2IN_NEXT_DROP;
1275                   goto trace0;
1276                 }
1277               else
1278                 {
1279                   s0 = pool_elt_at_index (
1280                     sm->per_thread_data[thread_index].sessions,
1281                     value0.value);
1282                 }
1283             }
1284
1285           old_addr0 = ip0->dst_address.as_u32;
1286           ip0->dst_address = s0->in2out.addr;
1287           new_addr0 = ip0->dst_address.as_u32;
1288           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1289
1290           sum0 = ip0->checksum;
1291           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1292                                  ip4_header_t,
1293                                  dst_address /* changed member */);
1294           ip0->checksum = ip_csum_fold (sum0);
1295
1296           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1297             {
1298               old_port0 = tcp0->dst_port;
1299               tcp0->dst_port = s0->in2out.port;
1300               new_port0 = tcp0->dst_port;
1301
1302               sum0 = tcp0->checksum;
1303               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1304                                      ip4_header_t,
1305                                      dst_address /* changed member */);
1306
1307               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1308                                      ip4_header_t /* cheat */,
1309                                      length /* changed member */);
1310               tcp0->checksum = ip_csum_fold(sum0);
1311               if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1312                 goto trace0;
1313             }
1314           else
1315             {
1316               old_port0 = udp0->dst_port;
1317               udp0->dst_port = s0->in2out.port;
1318               udp0->checksum = 0;
1319             }
1320
1321           /* Accounting */
1322           s0->last_heard = now;
1323           s0->total_pkts++;
1324           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1325           /* Per-user LRU list maintenance */
1326           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1327                              s0->per_user_index);
1328           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1329                               s0->per_user_list_head_index,
1330                               s0->per_user_index);
1331         trace0:
1332
1333           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1334                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1335             {
1336               snat_out2in_trace_t *t =
1337                  vlib_add_trace (vm, node, b0, sizeof (*t));
1338               t->sw_if_index = sw_if_index0;
1339               t->next_index = next0;
1340               t->session_index = ~0;
1341               if (s0)
1342                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1343             }
1344
1345           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1346
1347
1348           ip1 = vlib_buffer_get_current (b1);
1349           udp1 = ip4_next_header (ip1);
1350           tcp1 = (tcp_header_t *) udp1;
1351           icmp1 = (icmp46_header_t *) udp1;
1352
1353           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1354           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1355                                    sw_if_index1);
1356
1357           if (PREDICT_FALSE(ip1->ttl == 1))
1358             {
1359               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1360               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1361                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1362                                            0);
1363               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1364               goto trace1;
1365             }
1366
1367           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1368
1369           if (PREDICT_FALSE (proto1 == ~0))
1370             {
1371               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1372                                              thread_index, now, vm, node);
1373               if (!sm->forwarding_enabled)
1374                 if (!s1)
1375                   next1 = SNAT_OUT2IN_NEXT_DROP;
1376               goto trace1;
1377             }
1378
1379           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1380             {
1381               next1 = icmp_out2in_slow_path
1382                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1383                  next1, now, thread_index, &s1);
1384               goto trace1;
1385             }
1386
1387           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1388             {
1389               next1 = SNAT_OUT2IN_NEXT_REASS;
1390               goto trace1;
1391             }
1392
1393           key1.addr = ip1->dst_address;
1394           key1.port = udp1->dst_port;
1395           key1.protocol = proto1;
1396           key1.fib_index = rx_fib_index1;
1397
1398           kv1.key = key1.as_u64;
1399
1400           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1401                                       &kv1, &value1))
1402             {
1403               /* Try to match static mapping by external address and port,
1404                  destination address and port in packet */
1405               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1406                 {
1407                   /*
1408                    * Send DHCP packets to the ipv4 stack, or we won't
1409                    * be able to use dhcp client on the outside interface
1410                    */
1411                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1412                       && (udp1->dst_port ==
1413                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1414                     {
1415                       vnet_feature_next
1416                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1417                       goto trace1;
1418                     }
1419
1420                   if (!sm->forwarding_enabled)
1421                     {
1422                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1423                       next1 = SNAT_OUT2IN_NEXT_DROP;
1424                       goto trace1;
1425                     }
1426                   else
1427                     {
1428                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1429                         {
1430                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1431                           goto trace1;
1432                         }
1433                       create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1434                       goto trace1;
1435                     }
1436                 }
1437
1438               /* Create session initiated by host from external network */
1439               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1440                                                      thread_index);
1441               if (!s1)
1442                 {
1443                   next1 = SNAT_OUT2IN_NEXT_DROP;
1444                   goto trace1;
1445                 }
1446             }
1447           else
1448             {
1449               if (PREDICT_FALSE (value1.value == ~0ULL))
1450                 {
1451                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1452                                       now, vm, node);
1453                   if (!s1)
1454                     next1 = SNAT_OUT2IN_NEXT_DROP;
1455                   goto trace1;
1456                 }
1457               else
1458                 {
1459                   s1 = pool_elt_at_index (
1460                     sm->per_thread_data[thread_index].sessions,
1461                     value1.value);
1462                 }
1463             }
1464
1465           old_addr1 = ip1->dst_address.as_u32;
1466           ip1->dst_address = s1->in2out.addr;
1467           new_addr1 = ip1->dst_address.as_u32;
1468           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1469
1470           sum1 = ip1->checksum;
1471           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1472                                  ip4_header_t,
1473                                  dst_address /* changed member */);
1474           ip1->checksum = ip_csum_fold (sum1);
1475
1476           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1477             {
1478               old_port1 = tcp1->dst_port;
1479               tcp1->dst_port = s1->in2out.port;
1480               new_port1 = tcp1->dst_port;
1481
1482               sum1 = tcp1->checksum;
1483               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1484                                      ip4_header_t,
1485                                      dst_address /* changed member */);
1486
1487               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1488                                      ip4_header_t /* cheat */,
1489                                      length /* changed member */);
1490               tcp1->checksum = ip_csum_fold(sum1);
1491               if (nat44_set_tcp_session_state (sm, s1, tcp1, thread_index))
1492                 goto trace1;
1493             }
1494           else
1495             {
1496               old_port1 = udp1->dst_port;
1497               udp1->dst_port = s1->in2out.port;
1498               udp1->checksum = 0;
1499             }
1500
1501           /* Accounting */
1502           s1->last_heard = now;
1503           s1->total_pkts++;
1504           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1505           /* Per-user LRU list maintenance */
1506           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1507                              s1->per_user_index);
1508           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1509                               s1->per_user_list_head_index,
1510                               s1->per_user_index);
1511         trace1:
1512
1513           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1514                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1515             {
1516               snat_out2in_trace_t *t =
1517                  vlib_add_trace (vm, node, b1, sizeof (*t));
1518               t->sw_if_index = sw_if_index1;
1519               t->next_index = next1;
1520               t->session_index = ~0;
1521               if (s1)
1522                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1523             }
1524
1525           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1526
1527           /* verify speculative enqueues, maybe switch current next frame */
1528           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1529                                            to_next, n_left_to_next,
1530                                            bi0, bi1, next0, next1);
1531         }
1532
1533       while (n_left_from > 0 && n_left_to_next > 0)
1534         {
1535           u32 bi0;
1536           vlib_buffer_t * b0;
1537           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1538           u32 sw_if_index0;
1539           ip4_header_t * ip0;
1540           ip_csum_t sum0;
1541           u32 new_addr0, old_addr0;
1542           u16 new_port0, old_port0;
1543           udp_header_t * udp0;
1544           tcp_header_t * tcp0;
1545           icmp46_header_t * icmp0;
1546           snat_session_key_t key0, sm0;
1547           u32 rx_fib_index0;
1548           u32 proto0;
1549           snat_session_t * s0 = 0;
1550           clib_bihash_kv_8_8_t kv0, value0;
1551
1552           /* speculatively enqueue b0 to the current next frame */
1553           bi0 = from[0];
1554           to_next[0] = bi0;
1555           from += 1;
1556           to_next += 1;
1557           n_left_from -= 1;
1558           n_left_to_next -= 1;
1559
1560           b0 = vlib_get_buffer (vm, bi0);
1561
1562           vnet_buffer (b0)->snat.flags = 0;
1563
1564           ip0 = vlib_buffer_get_current (b0);
1565           udp0 = ip4_next_header (ip0);
1566           tcp0 = (tcp_header_t *) udp0;
1567           icmp0 = (icmp46_header_t *) udp0;
1568
1569           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1570           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1571                                    sw_if_index0);
1572
1573           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1574
1575           if (PREDICT_FALSE (proto0 == ~0))
1576             {
1577               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1578                                              thread_index, now, vm, node);
1579               if (!sm->forwarding_enabled)
1580                 if (!s0)
1581                   next0 = SNAT_OUT2IN_NEXT_DROP;
1582               goto trace00;
1583             }
1584
1585           if (PREDICT_FALSE(ip0->ttl == 1))
1586             {
1587               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1588               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1589                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1590                                            0);
1591               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1592               goto trace00;
1593             }
1594
1595           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1596             {
1597               next0 = icmp_out2in_slow_path
1598                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1599                  next0, now, thread_index, &s0);
1600               goto trace00;
1601             }
1602
1603           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1604             {
1605               next0 = SNAT_OUT2IN_NEXT_REASS;
1606               goto trace00;
1607             }
1608
1609           key0.addr = ip0->dst_address;
1610           key0.port = udp0->dst_port;
1611           key0.protocol = proto0;
1612           key0.fib_index = rx_fib_index0;
1613
1614           kv0.key = key0.as_u64;
1615
1616           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1617                                       &kv0, &value0))
1618             {
1619               /* Try to match static mapping by external address and port,
1620                  destination address and port in packet */
1621               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1622                 {
1623                   /*
1624                    * Send DHCP packets to the ipv4 stack, or we won't
1625                    * be able to use dhcp client on the outside interface
1626                    */
1627                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1628                       && (udp0->dst_port ==
1629                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1630                     {
1631                       vnet_feature_next
1632                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1633                       goto trace00;
1634                     }
1635
1636                   if (!sm->forwarding_enabled)
1637                     {
1638                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1639                       next0 = SNAT_OUT2IN_NEXT_DROP;
1640                       goto trace00;
1641                     }
1642                   else
1643                     {
1644                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1645                         {
1646                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1647                           goto trace00;
1648                         }
1649                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1650                       goto trace00;
1651                     }
1652                 }
1653
1654               /* Create session initiated by host from external network */
1655               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1656                                                      thread_index);
1657               if (!s0)
1658                 {
1659                   next0 = SNAT_OUT2IN_NEXT_DROP;
1660                   goto trace00;
1661                 }
1662             }
1663           else
1664             {
1665               if (PREDICT_FALSE (value0.value == ~0ULL))
1666                 {
1667                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1668                                       now, vm, node);
1669                   if (!s0)
1670                     next0 = SNAT_OUT2IN_NEXT_DROP;
1671                   goto trace00;
1672                 }
1673               else
1674                 {
1675                   s0 = pool_elt_at_index (
1676                     sm->per_thread_data[thread_index].sessions,
1677                     value0.value);
1678                 }
1679             }
1680
1681           old_addr0 = ip0->dst_address.as_u32;
1682           ip0->dst_address = s0->in2out.addr;
1683           new_addr0 = ip0->dst_address.as_u32;
1684           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1685
1686           sum0 = ip0->checksum;
1687           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1688                                  ip4_header_t,
1689                                  dst_address /* changed member */);
1690           ip0->checksum = ip_csum_fold (sum0);
1691
1692           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1693             {
1694               old_port0 = tcp0->dst_port;
1695               tcp0->dst_port = s0->in2out.port;
1696               new_port0 = tcp0->dst_port;
1697
1698               sum0 = tcp0->checksum;
1699               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1700                                      ip4_header_t,
1701                                      dst_address /* changed member */);
1702
1703               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1704                                      ip4_header_t /* cheat */,
1705                                      length /* changed member */);
1706               tcp0->checksum = ip_csum_fold(sum0);
1707               if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1708                 goto trace00;
1709             }
1710           else
1711             {
1712               old_port0 = udp0->dst_port;
1713               udp0->dst_port = s0->in2out.port;
1714               udp0->checksum = 0;
1715             }
1716
1717           /* Accounting */
1718           s0->last_heard = now;
1719           s0->total_pkts++;
1720           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1721           /* Per-user LRU list maintenance */
1722           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1723                              s0->per_user_index);
1724           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1725                               s0->per_user_list_head_index,
1726                               s0->per_user_index);
1727         trace00:
1728
1729           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1730                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1731             {
1732               snat_out2in_trace_t *t =
1733                  vlib_add_trace (vm, node, b0, sizeof (*t));
1734               t->sw_if_index = sw_if_index0;
1735               t->next_index = next0;
1736               t->session_index = ~0;
1737               if (s0)
1738                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1739             }
1740
1741           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1742
1743           /* verify speculative enqueue, maybe switch current next frame */
1744           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1745                                            to_next, n_left_to_next,
1746                                            bi0, next0);
1747         }
1748
1749       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1750     }
1751
1752   vlib_node_increment_counter (vm, snat_out2in_node.index,
1753                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1754                                pkts_processed);
1755   return frame->n_vectors;
1756 }
1757
1758 VLIB_REGISTER_NODE (snat_out2in_node) = {
1759   .function = snat_out2in_node_fn,
1760   .name = "nat44-out2in",
1761   .vector_size = sizeof (u32),
1762   .format_trace = format_snat_out2in_trace,
1763   .type = VLIB_NODE_TYPE_INTERNAL,
1764
1765   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1766   .error_strings = snat_out2in_error_strings,
1767
1768   .runtime_data_bytes = sizeof (snat_runtime_t),
1769
1770   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1771
1772   /* edit / add dispositions here */
1773   .next_nodes = {
1774     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1775     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1776     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1777     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1778     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1779   },
1780 };
1781 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1782
1783 static uword
1784 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1785                             vlib_node_runtime_t * node,
1786                             vlib_frame_t * frame)
1787 {
1788   u32 n_left_from, *from, *to_next;
1789   snat_out2in_next_t next_index;
1790   u32 pkts_processed = 0;
1791   snat_main_t *sm = &snat_main;
1792   f64 now = vlib_time_now (vm);
1793   u32 thread_index = vlib_get_thread_index ();
1794   snat_main_per_thread_data_t *per_thread_data =
1795     &sm->per_thread_data[thread_index];
1796   u32 *fragments_to_drop = 0;
1797   u32 *fragments_to_loopback = 0;
1798
1799   from = vlib_frame_vector_args (frame);
1800   n_left_from = frame->n_vectors;
1801   next_index = node->cached_next_index;
1802
1803   while (n_left_from > 0)
1804     {
1805       u32 n_left_to_next;
1806
1807       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1808
1809       while (n_left_from > 0 && n_left_to_next > 0)
1810        {
1811           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1812           vlib_buffer_t *b0;
1813           u32 next0;
1814           u8 cached0 = 0;
1815           ip4_header_t *ip0;
1816           nat_reass_ip4_t *reass0;
1817           udp_header_t * udp0;
1818           tcp_header_t * tcp0;
1819           snat_session_key_t key0, sm0;
1820           clib_bihash_kv_8_8_t kv0, value0;
1821           snat_session_t * s0 = 0;
1822           u16 old_port0, new_port0;
1823           ip_csum_t sum0;
1824
1825           /* speculatively enqueue b0 to the current next frame */
1826           bi0 = from[0];
1827           to_next[0] = bi0;
1828           from += 1;
1829           to_next += 1;
1830           n_left_from -= 1;
1831           n_left_to_next -= 1;
1832
1833           b0 = vlib_get_buffer (vm, bi0);
1834           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1835
1836           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1837           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1838                                                                sw_if_index0);
1839
1840           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1841             {
1842               next0 = SNAT_OUT2IN_NEXT_DROP;
1843               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1844               goto trace0;
1845             }
1846
1847           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1848           udp0 = ip4_next_header (ip0);
1849           tcp0 = (tcp_header_t *) udp0;
1850           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1851
1852           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1853                                                  ip0->dst_address,
1854                                                  ip0->fragment_id,
1855                                                  ip0->protocol,
1856                                                  1,
1857                                                  &fragments_to_drop);
1858
1859           if (PREDICT_FALSE (!reass0))
1860             {
1861               next0 = SNAT_OUT2IN_NEXT_DROP;
1862               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1863               goto trace0;
1864             }
1865
1866           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1867             {
1868               key0.addr = ip0->dst_address;
1869               key0.port = udp0->dst_port;
1870               key0.protocol = proto0;
1871               key0.fib_index = rx_fib_index0;
1872               kv0.key = key0.as_u64;
1873
1874               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1875                 {
1876                   /* Try to match static mapping by external address and port,
1877                      destination address and port in packet */
1878                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1879                     {
1880                       /*
1881                        * Send DHCP packets to the ipv4 stack, or we won't
1882                        * be able to use dhcp client on the outside interface
1883                        */
1884                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1885                           && (udp0->dst_port
1886                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1887                         {
1888                           vnet_feature_next
1889                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1890                              &next0, b0);
1891                           goto trace0;
1892                         }
1893
1894                       if (!sm->forwarding_enabled)
1895                         {
1896                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1897                           next0 = SNAT_OUT2IN_NEXT_DROP;
1898                           goto trace0;
1899                         }
1900                       else
1901                         {
1902                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1903                             {
1904                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1905                               goto trace0;
1906                             }
1907                           create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1908                           goto trace0;
1909                         }
1910                     }
1911
1912                   /* Create session initiated by host from external network */
1913                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1914                                                          thread_index);
1915                   if (!s0)
1916                     {
1917                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1918                       next0 = SNAT_OUT2IN_NEXT_DROP;
1919                       goto trace0;
1920                     }
1921                   reass0->sess_index = s0 - per_thread_data->sessions;
1922                   reass0->thread_index = thread_index;
1923                 }
1924               else
1925                 {
1926                   s0 = pool_elt_at_index (per_thread_data->sessions,
1927                                           value0.value);
1928                   reass0->sess_index = value0.value;
1929                 }
1930               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1931             }
1932           else
1933             {
1934               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1935                 {
1936                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1937                     {
1938                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1939                       next0 = SNAT_OUT2IN_NEXT_DROP;
1940                       goto trace0;
1941                     }
1942                   cached0 = 1;
1943                   goto trace0;
1944                 }
1945               s0 = pool_elt_at_index (per_thread_data->sessions,
1946                                       reass0->sess_index);
1947             }
1948
1949           old_addr0 = ip0->dst_address.as_u32;
1950           ip0->dst_address = s0->in2out.addr;
1951           new_addr0 = ip0->dst_address.as_u32;
1952           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1953
1954           sum0 = ip0->checksum;
1955           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1956                                  ip4_header_t,
1957                                  dst_address /* changed member */);
1958           ip0->checksum = ip_csum_fold (sum0);
1959
1960           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1961             {
1962               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1963                 {
1964                   old_port0 = tcp0->dst_port;
1965                   tcp0->dst_port = s0->in2out.port;
1966                   new_port0 = tcp0->dst_port;
1967
1968                   sum0 = tcp0->checksum;
1969                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1970                                          ip4_header_t,
1971                                          dst_address /* changed member */);
1972
1973                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1974                                          ip4_header_t /* cheat */,
1975                                          length /* changed member */);
1976                   tcp0->checksum = ip_csum_fold(sum0);
1977                   if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1978                     goto trace0;
1979                 }
1980               else
1981                 {
1982                   old_port0 = udp0->dst_port;
1983                   udp0->dst_port = s0->in2out.port;
1984                   udp0->checksum = 0;
1985                 }
1986             }
1987
1988           /* Accounting */
1989           s0->last_heard = now;
1990           s0->total_pkts++;
1991           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1992           /* Per-user LRU list maintenance */
1993           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1994                              s0->per_user_index);
1995           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1996                               s0->per_user_list_head_index,
1997                               s0->per_user_index);
1998
1999         trace0:
2000           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2001                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2002             {
2003               nat44_out2in_reass_trace_t *t =
2004                  vlib_add_trace (vm, node, b0, sizeof (*t));
2005               t->cached = cached0;
2006               t->sw_if_index = sw_if_index0;
2007               t->next_index = next0;
2008             }
2009
2010           if (cached0)
2011             {
2012               n_left_to_next++;
2013               to_next--;
2014             }
2015           else
2016             {
2017               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2018
2019               /* verify speculative enqueue, maybe switch current next frame */
2020               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2021                                                to_next, n_left_to_next,
2022                                                bi0, next0);
2023             }
2024
2025           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2026             {
2027               from = vlib_frame_vector_args (frame);
2028               u32 len = vec_len (fragments_to_loopback);
2029               if (len <= VLIB_FRAME_SIZE)
2030                 {
2031                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2032                   n_left_from = len;
2033                   vec_reset_length (fragments_to_loopback);
2034                 }
2035               else
2036                 {
2037                   clib_memcpy (from,
2038                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2039                                sizeof (u32) * VLIB_FRAME_SIZE);
2040                   n_left_from = VLIB_FRAME_SIZE;
2041                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2042                 }
2043             }
2044        }
2045
2046       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2047     }
2048
2049   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2050                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2051                                pkts_processed);
2052
2053   nat_send_all_to_node (vm, fragments_to_drop, node,
2054                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2055                         SNAT_OUT2IN_NEXT_DROP);
2056
2057   vec_free (fragments_to_drop);
2058   vec_free (fragments_to_loopback);
2059   return frame->n_vectors;
2060 }
2061
2062 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2063   .function = nat44_out2in_reass_node_fn,
2064   .name = "nat44-out2in-reass",
2065   .vector_size = sizeof (u32),
2066   .format_trace = format_nat44_out2in_reass_trace,
2067   .type = VLIB_NODE_TYPE_INTERNAL,
2068
2069   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2070   .error_strings = snat_out2in_error_strings,
2071
2072   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2073
2074   /* edit / add dispositions here */
2075   .next_nodes = {
2076     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2077     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2078     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2079     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2080     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2081   },
2082 };
2083 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2084                               nat44_out2in_reass_node_fn);
2085
2086 /**************************/
2087 /*** deterministic mode ***/
2088 /**************************/
2089 static uword
2090 snat_det_out2in_node_fn (vlib_main_t * vm,
2091                          vlib_node_runtime_t * node,
2092                          vlib_frame_t * frame)
2093 {
2094   u32 n_left_from, * from, * to_next;
2095   snat_out2in_next_t next_index;
2096   u32 pkts_processed = 0;
2097   snat_main_t * sm = &snat_main;
2098   u32 thread_index = vlib_get_thread_index ();
2099
2100   from = vlib_frame_vector_args (frame);
2101   n_left_from = frame->n_vectors;
2102   next_index = node->cached_next_index;
2103
2104   while (n_left_from > 0)
2105     {
2106       u32 n_left_to_next;
2107
2108       vlib_get_next_frame (vm, node, next_index,
2109                            to_next, n_left_to_next);
2110
2111       while (n_left_from >= 4 && n_left_to_next >= 2)
2112         {
2113           u32 bi0, bi1;
2114           vlib_buffer_t * b0, * b1;
2115           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2116           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2117           u32 sw_if_index0, sw_if_index1;
2118           ip4_header_t * ip0, * ip1;
2119           ip_csum_t sum0, sum1;
2120           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2121           u16 new_port0, old_port0, old_port1, new_port1;
2122           udp_header_t * udp0, * udp1;
2123           tcp_header_t * tcp0, * tcp1;
2124           u32 proto0, proto1;
2125           snat_det_out_key_t key0, key1;
2126           snat_det_map_t * dm0, * dm1;
2127           snat_det_session_t * ses0 = 0, * ses1 = 0;
2128           u32 rx_fib_index0, rx_fib_index1;
2129           icmp46_header_t * icmp0, * icmp1;
2130
2131           /* Prefetch next iteration. */
2132           {
2133             vlib_buffer_t * p2, * p3;
2134
2135             p2 = vlib_get_buffer (vm, from[2]);
2136             p3 = vlib_get_buffer (vm, from[3]);
2137
2138             vlib_prefetch_buffer_header (p2, LOAD);
2139             vlib_prefetch_buffer_header (p3, LOAD);
2140
2141             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2142             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2143           }
2144
2145           /* speculatively enqueue b0 and b1 to the current next frame */
2146           to_next[0] = bi0 = from[0];
2147           to_next[1] = bi1 = from[1];
2148           from += 2;
2149           to_next += 2;
2150           n_left_from -= 2;
2151           n_left_to_next -= 2;
2152
2153           b0 = vlib_get_buffer (vm, bi0);
2154           b1 = vlib_get_buffer (vm, bi1);
2155
2156           ip0 = vlib_buffer_get_current (b0);
2157           udp0 = ip4_next_header (ip0);
2158           tcp0 = (tcp_header_t *) udp0;
2159
2160           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2161
2162           if (PREDICT_FALSE(ip0->ttl == 1))
2163             {
2164               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2165               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2166                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2167                                            0);
2168               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2169               goto trace0;
2170             }
2171
2172           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2173
2174           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2175             {
2176               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2177               icmp0 = (icmp46_header_t *) udp0;
2178
2179               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2180                                   rx_fib_index0, node, next0, thread_index,
2181                                   &ses0, &dm0);
2182               goto trace0;
2183             }
2184
2185           key0.ext_host_addr = ip0->src_address;
2186           key0.ext_host_port = tcp0->src;
2187           key0.out_port = tcp0->dst;
2188
2189           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2190           if (PREDICT_FALSE(!dm0))
2191             {
2192               clib_warning("unknown dst address:  %U",
2193                            format_ip4_address, &ip0->dst_address);
2194               next0 = SNAT_OUT2IN_NEXT_DROP;
2195               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2196               goto trace0;
2197             }
2198
2199           snat_det_reverse(dm0, &ip0->dst_address,
2200                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2201
2202           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2203           if (PREDICT_FALSE(!ses0))
2204             {
2205               clib_warning("no match src %U:%d dst %U:%d for user %U",
2206                            format_ip4_address, &ip0->src_address,
2207                            clib_net_to_host_u16 (tcp0->src),
2208                            format_ip4_address, &ip0->dst_address,
2209                            clib_net_to_host_u16 (tcp0->dst),
2210                            format_ip4_address, &new_addr0);
2211               next0 = SNAT_OUT2IN_NEXT_DROP;
2212               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2213               goto trace0;
2214             }
2215           new_port0 = ses0->in_port;
2216
2217           old_addr0 = ip0->dst_address;
2218           ip0->dst_address = new_addr0;
2219           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2220
2221           sum0 = ip0->checksum;
2222           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2223                                  ip4_header_t,
2224                                  dst_address /* changed member */);
2225           ip0->checksum = ip_csum_fold (sum0);
2226
2227           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2228             {
2229               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2230                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2231               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2232                 snat_det_ses_close(dm0, ses0);
2233
2234               old_port0 = tcp0->dst;
2235               tcp0->dst = new_port0;
2236
2237               sum0 = tcp0->checksum;
2238               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2239                                      ip4_header_t,
2240                                      dst_address /* changed member */);
2241
2242               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2243                                      ip4_header_t /* cheat */,
2244                                      length /* changed member */);
2245               tcp0->checksum = ip_csum_fold(sum0);
2246             }
2247           else
2248             {
2249               old_port0 = udp0->dst_port;
2250               udp0->dst_port = new_port0;
2251               udp0->checksum = 0;
2252             }
2253
2254         trace0:
2255
2256           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2257                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2258             {
2259               snat_out2in_trace_t *t =
2260                  vlib_add_trace (vm, node, b0, sizeof (*t));
2261               t->sw_if_index = sw_if_index0;
2262               t->next_index = next0;
2263               t->session_index = ~0;
2264               if (ses0)
2265                 t->session_index = ses0 - dm0->sessions;
2266             }
2267
2268           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2269
2270           b1 = vlib_get_buffer (vm, bi1);
2271
2272           ip1 = vlib_buffer_get_current (b1);
2273           udp1 = ip4_next_header (ip1);
2274           tcp1 = (tcp_header_t *) udp1;
2275
2276           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2277
2278           if (PREDICT_FALSE(ip1->ttl == 1))
2279             {
2280               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2281               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2282                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2283                                            0);
2284               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2285               goto trace1;
2286             }
2287
2288           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2289
2290           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2291             {
2292               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2293               icmp1 = (icmp46_header_t *) udp1;
2294
2295               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2296                                   rx_fib_index1, node, next1, thread_index,
2297                                   &ses1, &dm1);
2298               goto trace1;
2299             }
2300
2301           key1.ext_host_addr = ip1->src_address;
2302           key1.ext_host_port = tcp1->src;
2303           key1.out_port = tcp1->dst;
2304
2305           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2306           if (PREDICT_FALSE(!dm1))
2307             {
2308               clib_warning("unknown dst address:  %U",
2309                            format_ip4_address, &ip1->dst_address);
2310               next1 = SNAT_OUT2IN_NEXT_DROP;
2311               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2312               goto trace1;
2313             }
2314
2315           snat_det_reverse(dm1, &ip1->dst_address,
2316                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2317
2318           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2319           if (PREDICT_FALSE(!ses1))
2320             {
2321               clib_warning("no match src %U:%d dst %U:%d for user %U",
2322                            format_ip4_address, &ip1->src_address,
2323                            clib_net_to_host_u16 (tcp1->src),
2324                            format_ip4_address, &ip1->dst_address,
2325                            clib_net_to_host_u16 (tcp1->dst),
2326                            format_ip4_address, &new_addr1);
2327               next1 = SNAT_OUT2IN_NEXT_DROP;
2328               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2329               goto trace1;
2330             }
2331           new_port1 = ses1->in_port;
2332
2333           old_addr1 = ip1->dst_address;
2334           ip1->dst_address = new_addr1;
2335           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2336
2337           sum1 = ip1->checksum;
2338           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2339                                  ip4_header_t,
2340                                  dst_address /* changed member */);
2341           ip1->checksum = ip_csum_fold (sum1);
2342
2343           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2344             {
2345               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2346                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2347               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2348                 snat_det_ses_close(dm1, ses1);
2349
2350               old_port1 = tcp1->dst;
2351               tcp1->dst = new_port1;
2352
2353               sum1 = tcp1->checksum;
2354               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2355                                      ip4_header_t,
2356                                      dst_address /* changed member */);
2357
2358               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2359                                      ip4_header_t /* cheat */,
2360                                      length /* changed member */);
2361               tcp1->checksum = ip_csum_fold(sum1);
2362             }
2363           else
2364             {
2365               old_port1 = udp1->dst_port;
2366               udp1->dst_port = new_port1;
2367               udp1->checksum = 0;
2368             }
2369
2370         trace1:
2371
2372           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2373                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2374             {
2375               snat_out2in_trace_t *t =
2376                  vlib_add_trace (vm, node, b1, sizeof (*t));
2377               t->sw_if_index = sw_if_index1;
2378               t->next_index = next1;
2379               t->session_index = ~0;
2380               if (ses1)
2381                 t->session_index = ses1 - dm1->sessions;
2382             }
2383
2384           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2385
2386           /* verify speculative enqueues, maybe switch current next frame */
2387           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2388                                            to_next, n_left_to_next,
2389                                            bi0, bi1, next0, next1);
2390          }
2391
2392       while (n_left_from > 0 && n_left_to_next > 0)
2393         {
2394           u32 bi0;
2395           vlib_buffer_t * b0;
2396           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2397           u32 sw_if_index0;
2398           ip4_header_t * ip0;
2399           ip_csum_t sum0;
2400           ip4_address_t new_addr0, old_addr0;
2401           u16 new_port0, old_port0;
2402           udp_header_t * udp0;
2403           tcp_header_t * tcp0;
2404           u32 proto0;
2405           snat_det_out_key_t key0;
2406           snat_det_map_t * dm0;
2407           snat_det_session_t * ses0 = 0;
2408           u32 rx_fib_index0;
2409           icmp46_header_t * icmp0;
2410
2411           /* speculatively enqueue b0 to the current next frame */
2412           bi0 = from[0];
2413           to_next[0] = bi0;
2414           from += 1;
2415           to_next += 1;
2416           n_left_from -= 1;
2417           n_left_to_next -= 1;
2418
2419           b0 = vlib_get_buffer (vm, bi0);
2420
2421           ip0 = vlib_buffer_get_current (b0);
2422           udp0 = ip4_next_header (ip0);
2423           tcp0 = (tcp_header_t *) udp0;
2424
2425           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2426
2427           if (PREDICT_FALSE(ip0->ttl == 1))
2428             {
2429               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2430               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2431                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2432                                            0);
2433               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2434               goto trace00;
2435             }
2436
2437           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2438
2439           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2440             {
2441               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2442               icmp0 = (icmp46_header_t *) udp0;
2443
2444               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2445                                   rx_fib_index0, node, next0, thread_index,
2446                                   &ses0, &dm0);
2447               goto trace00;
2448             }
2449
2450           key0.ext_host_addr = ip0->src_address;
2451           key0.ext_host_port = tcp0->src;
2452           key0.out_port = tcp0->dst;
2453
2454           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2455           if (PREDICT_FALSE(!dm0))
2456             {
2457               clib_warning("unknown dst address:  %U",
2458                            format_ip4_address, &ip0->dst_address);
2459               next0 = SNAT_OUT2IN_NEXT_DROP;
2460               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2461               goto trace00;
2462             }
2463
2464           snat_det_reverse(dm0, &ip0->dst_address,
2465                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2466
2467           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2468           if (PREDICT_FALSE(!ses0))
2469             {
2470               clib_warning("no match src %U:%d dst %U:%d for user %U",
2471                            format_ip4_address, &ip0->src_address,
2472                            clib_net_to_host_u16 (tcp0->src),
2473                            format_ip4_address, &ip0->dst_address,
2474                            clib_net_to_host_u16 (tcp0->dst),
2475                            format_ip4_address, &new_addr0);
2476               next0 = SNAT_OUT2IN_NEXT_DROP;
2477               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2478               goto trace00;
2479             }
2480           new_port0 = ses0->in_port;
2481
2482           old_addr0 = ip0->dst_address;
2483           ip0->dst_address = new_addr0;
2484           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2485
2486           sum0 = ip0->checksum;
2487           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2488                                  ip4_header_t,
2489                                  dst_address /* changed member */);
2490           ip0->checksum = ip_csum_fold (sum0);
2491
2492           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2493             {
2494               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2495                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2496               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2497                 snat_det_ses_close(dm0, ses0);
2498
2499               old_port0 = tcp0->dst;
2500               tcp0->dst = new_port0;
2501
2502               sum0 = tcp0->checksum;
2503               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2504                                      ip4_header_t,
2505                                      dst_address /* changed member */);
2506
2507               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2508                                      ip4_header_t /* cheat */,
2509                                      length /* changed member */);
2510               tcp0->checksum = ip_csum_fold(sum0);
2511             }
2512           else
2513             {
2514               old_port0 = udp0->dst_port;
2515               udp0->dst_port = new_port0;
2516               udp0->checksum = 0;
2517             }
2518
2519         trace00:
2520
2521           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2522                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2523             {
2524               snat_out2in_trace_t *t =
2525                  vlib_add_trace (vm, node, b0, sizeof (*t));
2526               t->sw_if_index = sw_if_index0;
2527               t->next_index = next0;
2528               t->session_index = ~0;
2529               if (ses0)
2530                 t->session_index = ses0 - dm0->sessions;
2531             }
2532
2533           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2534
2535           /* verify speculative enqueue, maybe switch current next frame */
2536           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2537                                            to_next, n_left_to_next,
2538                                            bi0, next0);
2539         }
2540
2541       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2542     }
2543
2544   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2545                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2546                                pkts_processed);
2547   return frame->n_vectors;
2548 }
2549
2550 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2551   .function = snat_det_out2in_node_fn,
2552   .name = "nat44-det-out2in",
2553   .vector_size = sizeof (u32),
2554   .format_trace = format_snat_out2in_trace,
2555   .type = VLIB_NODE_TYPE_INTERNAL,
2556
2557   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2558   .error_strings = snat_out2in_error_strings,
2559
2560   .runtime_data_bytes = sizeof (snat_runtime_t),
2561
2562   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2563
2564   /* edit / add dispositions here */
2565   .next_nodes = {
2566     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2567     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2568     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2569     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2570     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2571   },
2572 };
2573 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2574
2575 /**
2576  * Get address and port values to be used for ICMP packet translation
2577  * and create session if needed
2578  *
2579  * @param[in,out] sm             NAT main
2580  * @param[in,out] node           NAT node runtime
2581  * @param[in] thread_index       thread index
2582  * @param[in,out] b0             buffer containing packet to be translated
2583  * @param[out] p_proto           protocol used for matching
2584  * @param[out] p_value           address and port after NAT translation
2585  * @param[out] p_dont_translate  if packet should not be translated
2586  * @param d                      optional parameter
2587  * @param e                      optional parameter
2588  */
2589 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2590                           u32 thread_index, vlib_buffer_t *b0,
2591                           ip4_header_t *ip0, u8 *p_proto,
2592                           snat_session_key_t *p_value,
2593                           u8 *p_dont_translate, void *d, void *e)
2594 {
2595   icmp46_header_t *icmp0;
2596   u32 sw_if_index0;
2597   u8 protocol;
2598   snat_det_out_key_t key0;
2599   u8 dont_translate = 0;
2600   u32 next0 = ~0;
2601   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2602   ip4_header_t *inner_ip0;
2603   void *l4_header = 0;
2604   icmp46_header_t *inner_icmp0;
2605   snat_det_map_t * dm0 = 0;
2606   ip4_address_t new_addr0 = {{0}};
2607   snat_det_session_t * ses0 = 0;
2608   ip4_address_t out_addr;
2609
2610   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2611   echo0 = (icmp_echo_header_t *)(icmp0+1);
2612   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2613
2614   if (!icmp_is_error_message (icmp0))
2615     {
2616       protocol = SNAT_PROTOCOL_ICMP;
2617       key0.ext_host_addr = ip0->src_address;
2618       key0.ext_host_port = 0;
2619       key0.out_port = echo0->identifier;
2620       out_addr = ip0->dst_address;
2621     }
2622   else
2623     {
2624       inner_ip0 = (ip4_header_t *)(echo0+1);
2625       l4_header = ip4_next_header (inner_ip0);
2626       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2627       key0.ext_host_addr = inner_ip0->dst_address;
2628       out_addr = inner_ip0->src_address;
2629       switch (protocol)
2630         {
2631         case SNAT_PROTOCOL_ICMP:
2632           inner_icmp0 = (icmp46_header_t*)l4_header;
2633           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2634           key0.ext_host_port = 0;
2635           key0.out_port = inner_echo0->identifier;
2636           break;
2637         case SNAT_PROTOCOL_UDP:
2638         case SNAT_PROTOCOL_TCP:
2639           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2640           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2641           break;
2642         default:
2643           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2644           next0 = SNAT_OUT2IN_NEXT_DROP;
2645           goto out;
2646         }
2647     }
2648
2649   dm0 = snat_det_map_by_out(sm, &out_addr);
2650   if (PREDICT_FALSE(!dm0))
2651     {
2652       /* Don't NAT packet aimed at the intfc address */
2653       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2654                                           ip0->dst_address.as_u32)))
2655         {
2656           dont_translate = 1;
2657           goto out;
2658         }
2659       clib_warning("unknown dst address:  %U",
2660                    format_ip4_address, &ip0->dst_address);
2661       goto out;
2662     }
2663
2664   snat_det_reverse(dm0, &ip0->dst_address,
2665                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2666
2667   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2668   if (PREDICT_FALSE(!ses0))
2669     {
2670       /* Don't NAT packet aimed at the intfc address */
2671       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2672                                           ip0->dst_address.as_u32)))
2673         {
2674           dont_translate = 1;
2675           goto out;
2676         }
2677       clib_warning("no match src %U:%d dst %U:%d for user %U",
2678                    format_ip4_address, &key0.ext_host_addr,
2679                    clib_net_to_host_u16 (key0.ext_host_port),
2680                    format_ip4_address, &out_addr,
2681                    clib_net_to_host_u16 (key0.out_port),
2682                    format_ip4_address, &new_addr0);
2683       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2684       next0 = SNAT_OUT2IN_NEXT_DROP;
2685       goto out;
2686     }
2687
2688   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2689                     !icmp_is_error_message (icmp0)))
2690     {
2691       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2692       next0 = SNAT_OUT2IN_NEXT_DROP;
2693       goto out;
2694     }
2695
2696   goto out;
2697
2698 out:
2699   *p_proto = protocol;
2700   if (ses0)
2701     {
2702       p_value->addr = new_addr0;
2703       p_value->fib_index = sm->inside_fib_index;
2704       p_value->port = ses0->in_port;
2705     }
2706   *p_dont_translate = dont_translate;
2707   if (d)
2708     *(snat_det_session_t**)d = ses0;
2709   if (e)
2710     *(snat_det_map_t**)e = dm0;
2711   return next0;
2712 }
2713
2714 /**********************/
2715 /*** worker handoff ***/
2716 /**********************/
2717 static uword
2718 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2719                                vlib_node_runtime_t * node,
2720                                vlib_frame_t * frame)
2721 {
2722   snat_main_t *sm = &snat_main;
2723   vlib_thread_main_t *tm = vlib_get_thread_main ();
2724   u32 n_left_from, *from, *to_next = 0;
2725   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2726   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2727     = 0;
2728   vlib_frame_queue_elt_t *hf = 0;
2729   vlib_frame_t *f = 0;
2730   int i;
2731   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2732   u32 next_worker_index = 0;
2733   u32 current_worker_index = ~0;
2734   u32 thread_index = vlib_get_thread_index ();
2735
2736   ASSERT (vec_len (sm->workers));
2737
2738   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2739     {
2740       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2741
2742       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2743                                sm->first_worker_index + sm->num_workers - 1,
2744                                (vlib_frame_queue_t *) (~0));
2745     }
2746
2747   from = vlib_frame_vector_args (frame);
2748   n_left_from = frame->n_vectors;
2749
2750   while (n_left_from > 0)
2751     {
2752       u32 bi0;
2753       vlib_buffer_t *b0;
2754       u32 sw_if_index0;
2755       u32 rx_fib_index0;
2756       ip4_header_t * ip0;
2757       u8 do_handoff;
2758
2759       bi0 = from[0];
2760       from += 1;
2761       n_left_from -= 1;
2762
2763       b0 = vlib_get_buffer (vm, bi0);
2764
2765       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2766       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2767
2768       ip0 = vlib_buffer_get_current (b0);
2769
2770       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2771
2772       if (PREDICT_FALSE (next_worker_index != thread_index))
2773         {
2774           do_handoff = 1;
2775
2776           if (next_worker_index != current_worker_index)
2777             {
2778               if (hf)
2779                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2780
2781               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2782                                                       next_worker_index,
2783                                                       handoff_queue_elt_by_worker_index);
2784
2785               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2786               to_next_worker = &hf->buffer_index[hf->n_vectors];
2787               current_worker_index = next_worker_index;
2788             }
2789
2790           /* enqueue to correct worker thread */
2791           to_next_worker[0] = bi0;
2792           to_next_worker++;
2793           n_left_to_next_worker--;
2794
2795           if (n_left_to_next_worker == 0)
2796             {
2797               hf->n_vectors = VLIB_FRAME_SIZE;
2798               vlib_put_frame_queue_elt (hf);
2799               current_worker_index = ~0;
2800               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2801               hf = 0;
2802             }
2803         }
2804       else
2805         {
2806           do_handoff = 0;
2807           /* if this is 1st frame */
2808           if (!f)
2809             {
2810               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2811               to_next = vlib_frame_vector_args (f);
2812             }
2813
2814           to_next[0] = bi0;
2815           to_next += 1;
2816           f->n_vectors++;
2817         }
2818
2819       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2820                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2821         {
2822           snat_out2in_worker_handoff_trace_t *t =
2823             vlib_add_trace (vm, node, b0, sizeof (*t));
2824           t->next_worker_index = next_worker_index;
2825           t->do_handoff = do_handoff;
2826         }
2827     }
2828
2829   if (f)
2830     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2831
2832   if (hf)
2833     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2834
2835   /* Ship frames to the worker nodes */
2836   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2837     {
2838       if (handoff_queue_elt_by_worker_index[i])
2839         {
2840           hf = handoff_queue_elt_by_worker_index[i];
2841           /*
2842            * It works better to let the handoff node
2843            * rate-adapt, always ship the handoff queue element.
2844            */
2845           if (1 || hf->n_vectors == hf->last_n_vectors)
2846             {
2847               vlib_put_frame_queue_elt (hf);
2848               handoff_queue_elt_by_worker_index[i] = 0;
2849             }
2850           else
2851             hf->last_n_vectors = hf->n_vectors;
2852         }
2853       congested_handoff_queue_by_worker_index[i] =
2854         (vlib_frame_queue_t *) (~0);
2855     }
2856   hf = 0;
2857   current_worker_index = ~0;
2858   return frame->n_vectors;
2859 }
2860
2861 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2862   .function = snat_out2in_worker_handoff_fn,
2863   .name = "nat44-out2in-worker-handoff",
2864   .vector_size = sizeof (u32),
2865   .format_trace = format_snat_out2in_worker_handoff_trace,
2866   .type = VLIB_NODE_TYPE_INTERNAL,
2867
2868   .n_next_nodes = 1,
2869
2870   .next_nodes = {
2871     [0] = "error-drop",
2872   },
2873 };
2874
2875 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2876
2877 static uword
2878 snat_out2in_fast_node_fn (vlib_main_t * vm,
2879                           vlib_node_runtime_t * node,
2880                           vlib_frame_t * frame)
2881 {
2882   u32 n_left_from, * from, * to_next;
2883   snat_out2in_next_t next_index;
2884   u32 pkts_processed = 0;
2885   snat_main_t * sm = &snat_main;
2886
2887   from = vlib_frame_vector_args (frame);
2888   n_left_from = frame->n_vectors;
2889   next_index = node->cached_next_index;
2890
2891   while (n_left_from > 0)
2892     {
2893       u32 n_left_to_next;
2894
2895       vlib_get_next_frame (vm, node, next_index,
2896                            to_next, n_left_to_next);
2897
2898       while (n_left_from > 0 && n_left_to_next > 0)
2899         {
2900           u32 bi0;
2901           vlib_buffer_t * b0;
2902           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2903           u32 sw_if_index0;
2904           ip4_header_t * ip0;
2905           ip_csum_t sum0;
2906           u32 new_addr0, old_addr0;
2907           u16 new_port0, old_port0;
2908           udp_header_t * udp0;
2909           tcp_header_t * tcp0;
2910           icmp46_header_t * icmp0;
2911           snat_session_key_t key0, sm0;
2912           u32 proto0;
2913           u32 rx_fib_index0;
2914
2915           /* speculatively enqueue b0 to the current next frame */
2916           bi0 = from[0];
2917           to_next[0] = bi0;
2918           from += 1;
2919           to_next += 1;
2920           n_left_from -= 1;
2921           n_left_to_next -= 1;
2922
2923           b0 = vlib_get_buffer (vm, bi0);
2924
2925           ip0 = vlib_buffer_get_current (b0);
2926           udp0 = ip4_next_header (ip0);
2927           tcp0 = (tcp_header_t *) udp0;
2928           icmp0 = (icmp46_header_t *) udp0;
2929
2930           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2931           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2932
2933           vnet_feature_next (sw_if_index0, &next0, b0);
2934
2935           if (PREDICT_FALSE(ip0->ttl == 1))
2936             {
2937               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2938               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2939                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2940                                            0);
2941               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2942               goto trace00;
2943             }
2944
2945           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2946
2947           if (PREDICT_FALSE (proto0 == ~0))
2948               goto trace00;
2949
2950           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2951             {
2952               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2953                                   rx_fib_index0, node, next0, ~0, 0, 0);
2954               goto trace00;
2955             }
2956
2957           key0.addr = ip0->dst_address;
2958           key0.port = udp0->dst_port;
2959           key0.fib_index = rx_fib_index0;
2960
2961           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2962             {
2963               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2964               goto trace00;
2965             }
2966
2967           new_addr0 = sm0.addr.as_u32;
2968           new_port0 = sm0.port;
2969           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2970           old_addr0 = ip0->dst_address.as_u32;
2971           ip0->dst_address.as_u32 = new_addr0;
2972
2973           sum0 = ip0->checksum;
2974           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2975                                  ip4_header_t,
2976                                  dst_address /* changed member */);
2977           ip0->checksum = ip_csum_fold (sum0);
2978
2979           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2980             {
2981                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2982                 {
2983                   old_port0 = tcp0->dst_port;
2984                   tcp0->dst_port = new_port0;
2985
2986                   sum0 = tcp0->checksum;
2987                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2988                                          ip4_header_t,
2989                                          dst_address /* changed member */);
2990
2991                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2992                                          ip4_header_t /* cheat */,
2993                                          length /* changed member */);
2994                   tcp0->checksum = ip_csum_fold(sum0);
2995                 }
2996               else
2997                 {
2998                   old_port0 = udp0->dst_port;
2999                   udp0->dst_port = new_port0;
3000                   udp0->checksum = 0;
3001                 }
3002             }
3003           else
3004             {
3005               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3006                 {
3007                   sum0 = tcp0->checksum;
3008                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3009                                          ip4_header_t,
3010                                          dst_address /* changed member */);
3011
3012                   tcp0->checksum = ip_csum_fold(sum0);
3013                 }
3014             }
3015
3016         trace00:
3017
3018           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3019                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3020             {
3021               snat_out2in_trace_t *t =
3022                  vlib_add_trace (vm, node, b0, sizeof (*t));
3023               t->sw_if_index = sw_if_index0;
3024               t->next_index = next0;
3025             }
3026
3027           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3028
3029           /* verify speculative enqueue, maybe switch current next frame */
3030           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3031                                            to_next, n_left_to_next,
3032                                            bi0, next0);
3033         }
3034
3035       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3036     }
3037
3038   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3039                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3040                                pkts_processed);
3041   return frame->n_vectors;
3042 }
3043
3044 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3045   .function = snat_out2in_fast_node_fn,
3046   .name = "nat44-out2in-fast",
3047   .vector_size = sizeof (u32),
3048   .format_trace = format_snat_out2in_fast_trace,
3049   .type = VLIB_NODE_TYPE_INTERNAL,
3050
3051   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3052   .error_strings = snat_out2in_error_strings,
3053
3054   .runtime_data_bytes = sizeof (snat_runtime_t),
3055
3056   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3057
3058   /* edit / add dispositions here */
3059   .next_nodes = {
3060     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3061     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3062     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3063     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3064     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3065   },
3066 };
3067 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);