src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   user_session_increment (sm, u, 1 /* static */);
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 337                       u32 thread_index)
 338 {
 339   nat_ed_ses_key_t key;
 340   clib_bihash_kv_16_8_t kv, value;
 341   udp_header_t *udp;
 342   snat_user_t *u;
 343   snat_session_t *s = 0;
 344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 345   f64 now = vlib_time_now (sm->vlib_main);
 346
 347   if (ip->protocol == IP_PROTOCOL_ICMP)
 348     {
 349       if (icmp_get_ed_key (ip, &key))
 350         return;
 351     }
 352   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 353     {
 354       udp = ip4_next_header(ip);
 355       key.r_addr = ip->src_address;
 356       key.l_addr = ip->dst_address;
 357       key.proto = ip->protocol;
 358       key.l_port = udp->dst_port;
 359       key.r_port = udp->src_port;
 360     }
 361   else
 362     {
 363       key.r_addr = ip->src_address;
 364       key.l_addr = ip->dst_address;
 365       key.proto = ip->protocol;
 366       key.l_port = key.r_port = 0;
 367     }
 368   key.fib_index = 0;
 369   kv.key[0] = key.as_u64[0];
 370   kv.key[1] = key.as_u64[1];
 371
 372   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 373     {
 374       s = pool_elt_at_index (tsm->sessions, value.value);
 375     }
 376   else
 377     {
 378       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 379         return;
 380
 381       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
 382       if (!u)
 383         {
 384           clib_warning ("create NAT user failed");
 385           return;
 386         }
 387
 388       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 389       if (!s)
 390         {
 391           clib_warning ("create NAT session failed");
 392           return;
 393         }
 394
 395       s->ext_host_addr = key.r_addr;
 396       s->ext_host_port = key.r_port;
 397       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 398       s->outside_address_index = ~0;
 399       s->out2in.addr = key.l_addr;
 400       s->out2in.port = key.l_port;
 401       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 402       s->out2in.fib_index = 0;
 403       s->in2out = s->out2in;
 404       user_session_increment (sm, u, 0);
 405
 406       kv.value = s - tsm->sessions;
 407       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 408         clib_warning ("in2out_ed key add failed");
 409     }
 410
 411   if (ip->protocol == IP_PROTOCOL_TCP)
 412     {
 413       tcp_header_t *tcp = ip4_next_header(ip);
 414       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
 415         return;
 416     }
 417   /* Per-user LRU list maintenance */
 418   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 419   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 420                       s->per_user_index);
 421   /* Accounting */
 422   s->last_heard = now;
 423   s->total_pkts++;
 424 }
 425
 426 /**
 427  * Get address and port values to be used for ICMP packet translation
 428  * and create session if needed
 429  *
 430  * @param[in,out] sm             NAT main
 431  * @param[in,out] node           NAT node runtime
 432  * @param[in] thread_index       thread index
 433  * @param[in,out] b0             buffer containing packet to be translated
 434  * @param[out] p_proto           protocol used for matching
 435  * @param[out] p_value           address and port after NAT translation
 436  * @param[out] p_dont_translate  if packet should not be translated
 437  * @param d                      optional parameter
 438  * @param e                      optional parameter
 439  */
 440 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 441                            u32 thread_index, vlib_buffer_t *b0,
 442                            ip4_header_t *ip0, u8 *p_proto,
 443                            snat_session_key_t *p_value,
 444                            u8 *p_dont_translate, void *d, void *e)
 445 {
 446   icmp46_header_t *icmp0;
 447   u32 sw_if_index0;
 448   u32 rx_fib_index0;
 449   snat_session_key_t key0;
 450   snat_session_key_t sm0;
 451   snat_session_t *s0 = 0;
 452   u8 dont_translate = 0;
 453   clib_bihash_kv_8_8_t kv0, value0;
 454   u8 is_addr_only;
 455   u32 next0 = ~0;
 456   int err;
 457
 458   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 459   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 460   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 461
 462   key0.protocol = 0;
 463
 464   err = icmp_get_key (ip0, &key0);
 465   if (err != -1)
 466     {
 467       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 468       next0 = SNAT_OUT2IN_NEXT_DROP;
 469       goto out;
 470     }
 471   key0.fib_index = rx_fib_index0;
 472
 473   kv0.key = key0.as_u64;
 474
 475   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 476                               &value0))
 477     {
 478       /* Try to match static mapping by external address and port,
 479          destination address and port in packet */
 480       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 481         {
 482           if (!sm->forwarding_enabled)
 483             {
 484               /* Don't NAT packet aimed at the intfc address */
 485               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 486                                                   ip0->dst_address.as_u32)))
 487                 {
 488                   dont_translate = 1;
 489                   goto out;
 490                 }
 491               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 492               next0 = SNAT_OUT2IN_NEXT_DROP;
 493               goto out;
 494             }
 495           else
 496             {
 497               dont_translate = 1;
 498               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 499                 {
 500                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 501                   goto out;
 502                 }
 503               create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
 504               goto out;
 505             }
 506         }
 507
 508       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 509                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 510         {
 511           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 512           next0 = SNAT_OUT2IN_NEXT_DROP;
 513           goto out;
 514         }
 515
 516       /* Create session initiated by host from external network */
 517       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 518                                              node, thread_index);
 519
 520       if (!s0)
 521         {
 522           next0 = SNAT_OUT2IN_NEXT_DROP;
 523           goto out;
 524         }
 525     }
 526   else
 527     {
 528       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 529                         icmp0->type != ICMP4_echo_request &&
 530                         !icmp_is_error_message (icmp0)))
 531         {
 532           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 533           next0 = SNAT_OUT2IN_NEXT_DROP;
 534           goto out;
 535         }
 536
 537       if (PREDICT_FALSE (value0.value == ~0ULL))
 538         {
 539           nat_ed_ses_key_t key;
 540           clib_bihash_kv_16_8_t s_kv, s_value;
 541
 542           key.as_u64[0] = 0;
 543           key.as_u64[1] = 0;
 544           if (icmp_get_ed_key (ip0, &key))
 545             {
 546               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 547               next0 = SNAT_OUT2IN_NEXT_DROP;
 548               goto out;
 549             }
 550           key.fib_index = rx_fib_index0;
 551           s_kv.key[0] = key.as_u64[0];
 552           s_kv.key[1] = key.as_u64[1];
 553           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 554             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 555                                     s_value.value);
 556           else
 557            {
 558               next0 = SNAT_OUT2IN_NEXT_DROP;
 559               goto out;
 560            }
 561         }
 562       else
 563         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 564                                 value0.value);
 565     }
 566
 567 out:
 568   *p_proto = key0.protocol;
 569   if (s0)
 570     *p_value = s0->in2out;
 571   *p_dont_translate = dont_translate;
 572   if (d)
 573     *(snat_session_t**)d = s0;
 574   return next0;
 575 }
 576
 577 /**
 578  * Get address and port values to be used for ICMP packet translation
 579  *
 580  * @param[in] sm                 NAT main
 581  * @param[in,out] node           NAT node runtime
 582  * @param[in] thread_index       thread index
 583  * @param[in,out] b0             buffer containing packet to be translated
 584  * @param[out] p_proto           protocol used for matching
 585  * @param[out] p_value           address and port after NAT translation
 586  * @param[out] p_dont_translate  if packet should not be translated
 587  * @param d                      optional parameter
 588  * @param e                      optional parameter
 589  */
 590 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 591                            u32 thread_index, vlib_buffer_t *b0,
 592                            ip4_header_t *ip0, u8 *p_proto,
 593                            snat_session_key_t *p_value,
 594                            u8 *p_dont_translate, void *d, void *e)
 595 {
 596   icmp46_header_t *icmp0;
 597   u32 sw_if_index0;
 598   u32 rx_fib_index0;
 599   snat_session_key_t key0;
 600   snat_session_key_t sm0;
 601   u8 dont_translate = 0;
 602   u8 is_addr_only;
 603   u32 next0 = ~0;
 604   int err;
 605
 606   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 607   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 608   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 609
 610   err = icmp_get_key (ip0, &key0);
 611   if (err != -1)
 612     {
 613       b0->error = node->errors[err];
 614       next0 = SNAT_OUT2IN_NEXT_DROP;
 615       goto out2;
 616     }
 617   key0.fib_index = rx_fib_index0;
 618
 619   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 620     {
 621       /* Don't NAT packet aimed at the intfc address */
 622       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 623         {
 624           dont_translate = 1;
 625           goto out;
 626         }
 627       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 628       next0 = SNAT_OUT2IN_NEXT_DROP;
 629       goto out;
 630     }
 631
 632   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 633                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 634                     !icmp_is_error_message (icmp0)))
 635     {
 636       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 637       next0 = SNAT_OUT2IN_NEXT_DROP;
 638       goto out;
 639     }
 640
 641 out:
 642   *p_value = sm0;
 643 out2:
 644   *p_proto = key0.protocol;
 645   *p_dont_translate = dont_translate;
 646   return next0;
 647 }
 648
 649 static inline u32 icmp_out2in (snat_main_t *sm,
 650                                vlib_buffer_t * b0,
 651                                ip4_header_t * ip0,
 652                                icmp46_header_t * icmp0,
 653                                u32 sw_if_index0,
 654                                u32 rx_fib_index0,
 655                                vlib_node_runtime_t * node,
 656                                u32 next0,
 657                                u32 thread_index,
 658                                void *d,
 659                                void *e)
 660 {
 661   snat_session_key_t sm0;
 662   u8 protocol;
 663   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 664   ip4_header_t *inner_ip0 = 0;
 665   void *l4_header = 0;
 666   icmp46_header_t *inner_icmp0;
 667   u8 dont_translate;
 668   u32 new_addr0, old_addr0;
 669   u16 old_id0, new_id0;
 670   ip_csum_t sum0;
 671   u16 checksum0;
 672   u32 next0_tmp;
 673
 674   echo0 = (icmp_echo_header_t *)(icmp0+1);
 675
 676   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 677                                        &protocol, &sm0, &dont_translate, d, e);
 678   if (next0_tmp != ~0)
 679     next0 = next0_tmp;
 680   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 681     goto out;
 682
 683   sum0 = ip_incremental_checksum (0, icmp0,
 684                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 685   checksum0 = ~ip_csum_fold (sum0);
 686   if (checksum0 != 0 && checksum0 != 0xffff)
 687     {
 688       next0 = SNAT_OUT2IN_NEXT_DROP;
 689       goto out;
 690     }
 691
 692   old_addr0 = ip0->dst_address.as_u32;
 693   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 694   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 695
 696   sum0 = ip0->checksum;
 697   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 698                          dst_address /* changed member */);
 699   ip0->checksum = ip_csum_fold (sum0);
 700
 701   if (icmp0->checksum == 0)
 702     icmp0->checksum = 0xffff;
 703
 704   if (!icmp_is_error_message (icmp0))
 705     {
 706       new_id0 = sm0.port;
 707       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 708         {
 709           old_id0 = echo0->identifier;
 710           new_id0 = sm0.port;
 711           echo0->identifier = new_id0;
 712
 713           sum0 = icmp0->checksum;
 714           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 715                                  identifier /* changed member */);
 716           icmp0->checksum = ip_csum_fold (sum0);
 717         }
 718     }
 719   else
 720     {
 721       inner_ip0 = (ip4_header_t *)(echo0+1);
 722       l4_header = ip4_next_header (inner_ip0);
 723
 724       if (!ip4_header_checksum_is_valid (inner_ip0))
 725         {
 726           next0 = SNAT_OUT2IN_NEXT_DROP;
 727           goto out;
 728         }
 729
 730       old_addr0 = inner_ip0->src_address.as_u32;
 731       inner_ip0->src_address = sm0.addr;
 732       new_addr0 = inner_ip0->src_address.as_u32;
 733
 734       sum0 = icmp0->checksum;
 735       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 736                              src_address /* changed member */);
 737       icmp0->checksum = ip_csum_fold (sum0);
 738
 739       switch (protocol)
 740         {
 741         case SNAT_PROTOCOL_ICMP:
 742           inner_icmp0 = (icmp46_header_t*)l4_header;
 743           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 744
 745           old_id0 = inner_echo0->identifier;
 746           new_id0 = sm0.port;
 747           inner_echo0->identifier = new_id0;
 748
 749           sum0 = icmp0->checksum;
 750           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 751                                  identifier);
 752           icmp0->checksum = ip_csum_fold (sum0);
 753           break;
 754         case SNAT_PROTOCOL_UDP:
 755         case SNAT_PROTOCOL_TCP:
 756           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 757           new_id0 = sm0.port;
 758           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 759
 760           sum0 = icmp0->checksum;
 761           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 762                                  src_port);
 763           icmp0->checksum = ip_csum_fold (sum0);
 764           break;
 765         default:
 766           ASSERT(0);
 767         }
 768     }
 769
 770 out:
 771   return next0;
 772 }
 773
 774
 775 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 776                                          vlib_buffer_t * b0,
 777                                          ip4_header_t * ip0,
 778                                          icmp46_header_t * icmp0,
 779                                          u32 sw_if_index0,
 780                                          u32 rx_fib_index0,
 781                                          vlib_node_runtime_t * node,
 782                                          u32 next0, f64 now,
 783                                          u32 thread_index,
 784                                          snat_session_t ** p_s0)
 785 {
 786   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 787                       next0, thread_index, p_s0, 0);
 788   snat_session_t * s0 = *p_s0;
 789   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 790     {
 791       /* Accounting */
 792       s0->last_heard = now;
 793       s0->total_pkts++;
 794       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 795       /* Per-user LRU list maintenance */
 796       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 797                          s0->per_user_index);
 798       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 799                           s0->per_user_list_head_index,
 800                           s0->per_user_index);
 801     }
 802   return next0;
 803 }
 804
 805 static snat_session_t *
 806 snat_out2in_unknown_proto (snat_main_t *sm,
 807                            vlib_buffer_t * b,
 808                            ip4_header_t * ip,
 809                            u32 rx_fib_index,
 810                            u32 thread_index,
 811                            f64 now,
 812                            vlib_main_t * vm,
 813                            vlib_node_runtime_t * node)
 814 {
 815   clib_bihash_kv_8_8_t kv, value;
 816   clib_bihash_kv_16_8_t s_kv, s_value;
 817   snat_static_mapping_t *m;
 818   snat_session_key_t m_key;
 819   u32 old_addr, new_addr;
 820   ip_csum_t sum;
 821   nat_ed_ses_key_t key;
 822   snat_session_t * s;
 823   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 824   snat_user_t *u;
 825
 826   old_addr = ip->dst_address.as_u32;
 827
 828   key.l_addr = ip->dst_address;
 829   key.r_addr = ip->src_address;
 830   key.fib_index = rx_fib_index;
 831   key.proto = ip->protocol;
 832   key.r_port = 0;
 833   key.l_port = 0;
 834   s_kv.key[0] = key.as_u64[0];
 835   s_kv.key[1] = key.as_u64[1];
 836
 837   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 838     {
 839       s = pool_elt_at_index (tsm->sessions, s_value.value);
 840       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 841     }
 842   else
 843     {
 844       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 845         {
 846           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 847           return 0;
 848         }
 849
 850       m_key.addr = ip->dst_address;
 851       m_key.port = 0;
 852       m_key.protocol = 0;
 853       m_key.fib_index = rx_fib_index;
 854       kv.key = m_key.as_u64;
 855       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 856         {
 857           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 858           return 0;
 859         }
 860
 861       m = pool_elt_at_index (sm->static_mappings, value.value);
 862
 863       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 864
 865       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 866                                   thread_index);
 867       if (!u)
 868         {
 869           clib_warning ("create NAT user failed");
 870           return 0;
 871         }
 872
 873       /* Create a new session */
 874       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 875       if (!s)
 876         {
 877           clib_warning ("create NAT session failed");
 878           return 0;
 879         }
 880
 881       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 882       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 883       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 884       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 885       s->outside_address_index = ~0;
 886       s->out2in.addr.as_u32 = old_addr;
 887       s->out2in.fib_index = rx_fib_index;
 888       s->in2out.addr.as_u32 = new_addr;
 889       s->in2out.fib_index = m->fib_index;
 890       s->in2out.port = s->out2in.port = ip->protocol;
 891       user_session_increment (sm, u, 1 /* static */);
 892
 893       /* Add to lookup tables */
 894       s_kv.value = s - tsm->sessions;
 895       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 896         clib_warning ("out2in key add failed");
 897
 898       key.l_addr = ip->dst_address;
 899       key.fib_index = m->fib_index;
 900       s_kv.key[0] = key.as_u64[0];
 901       s_kv.key[1] = key.as_u64[1];
 902       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 903         clib_warning ("in2out key add failed");
 904    }
 905
 906   /* Update IP checksum */
 907   sum = ip->checksum;
 908   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 909   ip->checksum = ip_csum_fold (sum);
 910
 911   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 912
 913   /* Accounting */
 914   s->last_heard = now;
 915   s->total_pkts++;
 916   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 917   /* Per-user LRU list maintenance */
 918   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 919   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 920                       s->per_user_index);
 921
 922   return s;
 923 }
 924
 925 static snat_session_t *
 926 snat_out2in_lb (snat_main_t *sm,
 927                 vlib_buffer_t * b,
 928                 ip4_header_t * ip,
 929                 u32 rx_fib_index,
 930                 u32 thread_index,
 931                 f64 now,
 932                 vlib_main_t * vm,
 933                 vlib_node_runtime_t * node)
 934 {
 935   nat_ed_ses_key_t key;
 936   clib_bihash_kv_16_8_t s_kv, s_value;
 937   udp_header_t *udp = ip4_next_header (ip);
 938   tcp_header_t *tcp = (tcp_header_t *) udp;
 939   snat_session_t *s = 0;
 940   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 941   snat_session_key_t e_key, l_key;
 942   u32 old_addr, new_addr;
 943   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 944   u16 new_port, old_port;
 945   ip_csum_t sum;
 946   snat_user_t *u;
 947   u32 address_index;
 948   snat_session_key_t eh_key;
 949   twice_nat_type_t twice_nat;
 950   u8 lb;
 951
 952   old_addr = ip->dst_address.as_u32;
 953
 954   key.l_addr = ip->dst_address;
 955   key.r_addr = ip->src_address;
 956   key.fib_index = rx_fib_index;
 957   key.proto = ip->protocol;
 958   key.r_port = udp->src_port;
 959   key.l_port = udp->dst_port;
 960   s_kv.key[0] = key.as_u64[0];
 961   s_kv.key[1] = key.as_u64[1];
 962
 963   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 964     {
 965       s = pool_elt_at_index (tsm->sessions, s_value.value);
 966     }
 967   else
 968     {
 969       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 970         {
 971           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 972           return 0;
 973         }
 974
 975       e_key.addr = ip->dst_address;
 976       e_key.port = udp->dst_port;
 977       e_key.protocol = proto;
 978       e_key.fib_index = rx_fib_index;
 979       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 980         return 0;
 981
 982       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 983                                   thread_index);
 984       if (!u)
 985       {
 986         clib_warning ("create NAT user failed");
 987         return 0;
 988       }
 989
 990       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 991       if (!s)
 992         {
 993           clib_warning ("create NAT session failed");
 994           return 0;
 995         }
 996
 997       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 998       s->ext_host_port = udp->src_port;
 999       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1000       if (lb)
1001         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1002       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1003       s->outside_address_index = ~0;
1004       s->out2in = e_key;
1005       s->in2out = l_key;
1006       user_session_increment (sm, u, 1 /* static */);
1007
1008       /* Add to lookup tables */
1009       s_kv.value = s - tsm->sessions;
1010       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1011         clib_warning ("out2in-ed key add failed");
1012
1013       if (twice_nat == TWICE_NAT ||
1014           (twice_nat == TWICE_NAT_SELF &&
1015            ip->src_address.as_u32 == l_key.addr.as_u32))
1016         {
1017           eh_key.protocol = proto;
1018           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1019                                                    thread_index, &eh_key,
1020                                                    &address_index,
1021                                                    sm->port_per_thread,
1022                                                    sm->per_thread_data[thread_index].snat_thread_index))
1023             {
1024               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1025               return 0;
1026             }
1027           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1028           key.r_port = s->ext_host_nat_port = eh_key.port;
1029           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1030         }
1031       key.l_addr = l_key.addr;
1032       key.fib_index = l_key.fib_index;
1033       key.l_port = l_key.port;
1034       s_kv.key[0] = key.as_u64[0];
1035       s_kv.key[1] = key.as_u64[1];
1036       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1037         clib_warning ("in2out-ed key add failed");
1038     }
1039
1040   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1041
1042   /* Update IP checksum */
1043   sum = ip->checksum;
1044   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1045   if (is_twice_nat_session (s))
1046     sum = ip_csum_update (sum, ip->src_address.as_u32,
1047                           s->ext_host_nat_addr.as_u32, ip4_header_t,
1048                           src_address);
1049   ip->checksum = ip_csum_fold (sum);
1050
1051   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1052
1053   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1054     {
1055       old_port = tcp->dst_port;
1056       tcp->dst_port = s->in2out.port;
1057       new_port = tcp->dst_port;
1058
1059       sum = tcp->checksum;
1060       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1061       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1062       if (is_twice_nat_session (s))
1063         {
1064           sum = ip_csum_update (sum, ip->src_address.as_u32,
1065                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1066                                 dst_address);
1067           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1068                                 ip4_header_t, length);
1069           tcp->src_port = s->ext_host_nat_port;
1070           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1071         }
1072       tcp->checksum = ip_csum_fold(sum);
1073       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1074         return s;
1075     }
1076   else
1077     {
1078       udp->dst_port = s->in2out.port;
1079       if (is_twice_nat_session (s))
1080         {
1081           udp->src_port = s->ext_host_nat_port;
1082           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1083         }
1084       udp->checksum = 0;
1085     }
1086
1087   /* Accounting */
1088   s->last_heard = now;
1089   s->total_pkts++;
1090   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1091   /* Per-user LRU list maintenance */
1092   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1093   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1094                       s->per_user_index);
1095
1096   return s;
1097 }
1098
1099 static uword
1100 snat_out2in_node_fn (vlib_main_t * vm,
1101                   vlib_node_runtime_t * node,
1102                   vlib_frame_t * frame)
1103 {
1104   u32 n_left_from, * from, * to_next;
1105   snat_out2in_next_t next_index;
1106   u32 pkts_processed = 0;
1107   snat_main_t * sm = &snat_main;
1108   f64 now = vlib_time_now (vm);
1109   u32 thread_index = vlib_get_thread_index ();
1110
1111   from = vlib_frame_vector_args (frame);
1112   n_left_from = frame->n_vectors;
1113   next_index = node->cached_next_index;
1114
1115   while (n_left_from > 0)
1116     {
1117       u32 n_left_to_next;
1118
1119       vlib_get_next_frame (vm, node, next_index,
1120                            to_next, n_left_to_next);
1121
1122       while (n_left_from >= 4 && n_left_to_next >= 2)
1123         {
1124           u32 bi0, bi1;
1125           vlib_buffer_t * b0, * b1;
1126           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1127           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1128           u32 sw_if_index0, sw_if_index1;
1129           ip4_header_t * ip0, *ip1;
1130           ip_csum_t sum0, sum1;
1131           u32 new_addr0, old_addr0;
1132           u16 new_port0, old_port0;
1133           u32 new_addr1, old_addr1;
1134           u16 new_port1, old_port1;
1135           udp_header_t * udp0, * udp1;
1136           tcp_header_t * tcp0, * tcp1;
1137           icmp46_header_t * icmp0, * icmp1;
1138           snat_session_key_t key0, key1, sm0, sm1;
1139           u32 rx_fib_index0, rx_fib_index1;
1140           u32 proto0, proto1;
1141           snat_session_t * s0 = 0, * s1 = 0;
1142           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1143
1144           /* Prefetch next iteration. */
1145           {
1146             vlib_buffer_t * p2, * p3;
1147
1148             p2 = vlib_get_buffer (vm, from[2]);
1149             p3 = vlib_get_buffer (vm, from[3]);
1150
1151             vlib_prefetch_buffer_header (p2, LOAD);
1152             vlib_prefetch_buffer_header (p3, LOAD);
1153
1154             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1155             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1156           }
1157
1158           /* speculatively enqueue b0 and b1 to the current next frame */
1159           to_next[0] = bi0 = from[0];
1160           to_next[1] = bi1 = from[1];
1161           from += 2;
1162           to_next += 2;
1163           n_left_from -= 2;
1164           n_left_to_next -= 2;
1165
1166           b0 = vlib_get_buffer (vm, bi0);
1167           b1 = vlib_get_buffer (vm, bi1);
1168
1169           vnet_buffer (b0)->snat.flags = 0;
1170           vnet_buffer (b1)->snat.flags = 0;
1171
1172           ip0 = vlib_buffer_get_current (b0);
1173           udp0 = ip4_next_header (ip0);
1174           tcp0 = (tcp_header_t *) udp0;
1175           icmp0 = (icmp46_header_t *) udp0;
1176
1177           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1178           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1179                                    sw_if_index0);
1180
1181           if (PREDICT_FALSE(ip0->ttl == 1))
1182             {
1183               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1184               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1185                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1186                                            0);
1187               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1188               goto trace0;
1189             }
1190
1191           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1192
1193           if (PREDICT_FALSE (proto0 == ~0))
1194             {
1195               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1196                                              thread_index, now, vm, node);
1197               if (!sm->forwarding_enabled)
1198                 if (!s0)
1199                   next0 = SNAT_OUT2IN_NEXT_DROP;
1200               goto trace0;
1201             }
1202
1203           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1204             {
1205               next0 = icmp_out2in_slow_path
1206                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1207                  next0, now, thread_index, &s0);
1208               goto trace0;
1209             }
1210
1211           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1212             {
1213               next0 = SNAT_OUT2IN_NEXT_REASS;
1214               goto trace0;
1215             }
1216
1217           key0.addr = ip0->dst_address;
1218           key0.port = udp0->dst_port;
1219           key0.protocol = proto0;
1220           key0.fib_index = rx_fib_index0;
1221
1222           kv0.key = key0.as_u64;
1223
1224           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1225                                       &kv0, &value0))
1226             {
1227               /* Try to match static mapping by external address and port,
1228                  destination address and port in packet */
1229               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1230                 {
1231                   /*
1232                    * Send DHCP packets to the ipv4 stack, or we won't
1233                    * be able to use dhcp client on the outside interface
1234                    */
1235                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1236                       && (udp0->dst_port ==
1237                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1238                     {
1239                       vnet_feature_next
1240                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1241                       goto trace0;
1242                     }
1243
1244                   if (!sm->forwarding_enabled)
1245                     {
1246                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1247                       next0 = SNAT_OUT2IN_NEXT_DROP;
1248                       goto trace0;
1249                     }
1250                   else
1251                     {
1252                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1253                         {
1254                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1255                           goto trace0;
1256                         }
1257                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1258                       goto trace0;
1259                     }
1260                 }
1261
1262               /* Create session initiated by host from external network */
1263               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1264                                                      thread_index);
1265               if (!s0)
1266                 {
1267                   next0 = SNAT_OUT2IN_NEXT_DROP;
1268                   goto trace0;
1269                 }
1270             }
1271           else
1272             {
1273               if (PREDICT_FALSE (value0.value == ~0ULL))
1274                 {
1275                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1276                                       now, vm, node);
1277                   if (!s0)
1278                     next0 = SNAT_OUT2IN_NEXT_DROP;
1279                   goto trace0;
1280                 }
1281               else
1282                 {
1283                   s0 = pool_elt_at_index (
1284                     sm->per_thread_data[thread_index].sessions,
1285                     value0.value);
1286                 }
1287             }
1288
1289           old_addr0 = ip0->dst_address.as_u32;
1290           ip0->dst_address = s0->in2out.addr;
1291           new_addr0 = ip0->dst_address.as_u32;
1292           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1293
1294           sum0 = ip0->checksum;
1295           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1296                                  ip4_header_t,
1297                                  dst_address /* changed member */);
1298           ip0->checksum = ip_csum_fold (sum0);
1299
1300           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1301             {
1302               old_port0 = tcp0->dst_port;
1303               tcp0->dst_port = s0->in2out.port;
1304               new_port0 = tcp0->dst_port;
1305
1306               sum0 = tcp0->checksum;
1307               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1308                                      ip4_header_t,
1309                                      dst_address /* changed member */);
1310
1311               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1312                                      ip4_header_t /* cheat */,
1313                                      length /* changed member */);
1314               tcp0->checksum = ip_csum_fold(sum0);
1315             }
1316           else
1317             {
1318               old_port0 = udp0->dst_port;
1319               udp0->dst_port = s0->in2out.port;
1320               udp0->checksum = 0;
1321             }
1322
1323           /* Accounting */
1324           s0->last_heard = now;
1325           s0->total_pkts++;
1326           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1327           /* Per-user LRU list maintenance */
1328           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1329                              s0->per_user_index);
1330           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1331                               s0->per_user_list_head_index,
1332                               s0->per_user_index);
1333         trace0:
1334
1335           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1336                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1337             {
1338               snat_out2in_trace_t *t =
1339                  vlib_add_trace (vm, node, b0, sizeof (*t));
1340               t->sw_if_index = sw_if_index0;
1341               t->next_index = next0;
1342               t->session_index = ~0;
1343               if (s0)
1344                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1345             }
1346
1347           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1348
1349
1350           ip1 = vlib_buffer_get_current (b1);
1351           udp1 = ip4_next_header (ip1);
1352           tcp1 = (tcp_header_t *) udp1;
1353           icmp1 = (icmp46_header_t *) udp1;
1354
1355           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1356           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1357                                    sw_if_index1);
1358
1359           if (PREDICT_FALSE(ip1->ttl == 1))
1360             {
1361               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1362               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1363                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1364                                            0);
1365               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1366               goto trace1;
1367             }
1368
1369           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1370
1371           if (PREDICT_FALSE (proto1 == ~0))
1372             {
1373               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1374                                              thread_index, now, vm, node);
1375               if (!sm->forwarding_enabled)
1376                 if (!s1)
1377                   next1 = SNAT_OUT2IN_NEXT_DROP;
1378               goto trace1;
1379             }
1380
1381           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1382             {
1383               next1 = icmp_out2in_slow_path
1384                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1385                  next1, now, thread_index, &s1);
1386               goto trace1;
1387             }
1388
1389           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1390             {
1391               next1 = SNAT_OUT2IN_NEXT_REASS;
1392               goto trace1;
1393             }
1394
1395           key1.addr = ip1->dst_address;
1396           key1.port = udp1->dst_port;
1397           key1.protocol = proto1;
1398           key1.fib_index = rx_fib_index1;
1399
1400           kv1.key = key1.as_u64;
1401
1402           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1403                                       &kv1, &value1))
1404             {
1405               /* Try to match static mapping by external address and port,
1406                  destination address and port in packet */
1407               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1408                 {
1409                   /*
1410                    * Send DHCP packets to the ipv4 stack, or we won't
1411                    * be able to use dhcp client on the outside interface
1412                    */
1413                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1414                       && (udp1->dst_port ==
1415                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1416                     {
1417                       vnet_feature_next
1418                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1419                       goto trace1;
1420                     }
1421
1422                   if (!sm->forwarding_enabled)
1423                     {
1424                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1425                       next1 = SNAT_OUT2IN_NEXT_DROP;
1426                       goto trace1;
1427                     }
1428                   else
1429                     {
1430                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1431                         {
1432                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1433                           goto trace1;
1434                         }
1435                       create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1436                       goto trace1;
1437                     }
1438                 }
1439
1440               /* Create session initiated by host from external network */
1441               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1442                                                      thread_index);
1443               if (!s1)
1444                 {
1445                   next1 = SNAT_OUT2IN_NEXT_DROP;
1446                   goto trace1;
1447                 }
1448             }
1449           else
1450             {
1451               if (PREDICT_FALSE (value1.value == ~0ULL))
1452                 {
1453                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1454                                       now, vm, node);
1455                   if (!s1)
1456                     next1 = SNAT_OUT2IN_NEXT_DROP;
1457                   goto trace1;
1458                 }
1459               else
1460                 {
1461                   s1 = pool_elt_at_index (
1462                     sm->per_thread_data[thread_index].sessions,
1463                     value1.value);
1464                 }
1465             }
1466
1467           old_addr1 = ip1->dst_address.as_u32;
1468           ip1->dst_address = s1->in2out.addr;
1469           new_addr1 = ip1->dst_address.as_u32;
1470           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1471
1472           sum1 = ip1->checksum;
1473           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1474                                  ip4_header_t,
1475                                  dst_address /* changed member */);
1476           ip1->checksum = ip_csum_fold (sum1);
1477
1478           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1479             {
1480               old_port1 = tcp1->dst_port;
1481               tcp1->dst_port = s1->in2out.port;
1482               new_port1 = tcp1->dst_port;
1483
1484               sum1 = tcp1->checksum;
1485               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1486                                      ip4_header_t,
1487                                      dst_address /* changed member */);
1488
1489               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1490                                      ip4_header_t /* cheat */,
1491                                      length /* changed member */);
1492               tcp1->checksum = ip_csum_fold(sum1);
1493             }
1494           else
1495             {
1496               old_port1 = udp1->dst_port;
1497               udp1->dst_port = s1->in2out.port;
1498               udp1->checksum = 0;
1499             }
1500
1501           /* Accounting */
1502           s1->last_heard = now;
1503           s1->total_pkts++;
1504           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1505           /* Per-user LRU list maintenance */
1506           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1507                              s1->per_user_index);
1508           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1509                               s1->per_user_list_head_index,
1510                               s1->per_user_index);
1511         trace1:
1512
1513           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1514                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1515             {
1516               snat_out2in_trace_t *t =
1517                  vlib_add_trace (vm, node, b1, sizeof (*t));
1518               t->sw_if_index = sw_if_index1;
1519               t->next_index = next1;
1520               t->session_index = ~0;
1521               if (s1)
1522                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1523             }
1524
1525           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1526
1527           /* verify speculative enqueues, maybe switch current next frame */
1528           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1529                                            to_next, n_left_to_next,
1530                                            bi0, bi1, next0, next1);
1531         }
1532
1533       while (n_left_from > 0 && n_left_to_next > 0)
1534         {
1535           u32 bi0;
1536           vlib_buffer_t * b0;
1537           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1538           u32 sw_if_index0;
1539           ip4_header_t * ip0;
1540           ip_csum_t sum0;
1541           u32 new_addr0, old_addr0;
1542           u16 new_port0, old_port0;
1543           udp_header_t * udp0;
1544           tcp_header_t * tcp0;
1545           icmp46_header_t * icmp0;
1546           snat_session_key_t key0, sm0;
1547           u32 rx_fib_index0;
1548           u32 proto0;
1549           snat_session_t * s0 = 0;
1550           clib_bihash_kv_8_8_t kv0, value0;
1551
1552           /* speculatively enqueue b0 to the current next frame */
1553           bi0 = from[0];
1554           to_next[0] = bi0;
1555           from += 1;
1556           to_next += 1;
1557           n_left_from -= 1;
1558           n_left_to_next -= 1;
1559
1560           b0 = vlib_get_buffer (vm, bi0);
1561
1562           vnet_buffer (b0)->snat.flags = 0;
1563
1564           ip0 = vlib_buffer_get_current (b0);
1565           udp0 = ip4_next_header (ip0);
1566           tcp0 = (tcp_header_t *) udp0;
1567           icmp0 = (icmp46_header_t *) udp0;
1568
1569           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1570           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1571                                    sw_if_index0);
1572
1573           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1574
1575           if (PREDICT_FALSE (proto0 == ~0))
1576             {
1577               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1578                                              thread_index, now, vm, node);
1579               if (!sm->forwarding_enabled)
1580                 if (!s0)
1581                   next0 = SNAT_OUT2IN_NEXT_DROP;
1582               goto trace00;
1583             }
1584
1585           if (PREDICT_FALSE(ip0->ttl == 1))
1586             {
1587               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1588               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1589                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1590                                            0);
1591               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1592               goto trace00;
1593             }
1594
1595           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1596             {
1597               next0 = icmp_out2in_slow_path
1598                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1599                  next0, now, thread_index, &s0);
1600               goto trace00;
1601             }
1602
1603           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1604             {
1605               next0 = SNAT_OUT2IN_NEXT_REASS;
1606               goto trace00;
1607             }
1608
1609           key0.addr = ip0->dst_address;
1610           key0.port = udp0->dst_port;
1611           key0.protocol = proto0;
1612           key0.fib_index = rx_fib_index0;
1613
1614           kv0.key = key0.as_u64;
1615
1616           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1617                                       &kv0, &value0))
1618             {
1619               /* Try to match static mapping by external address and port,
1620                  destination address and port in packet */
1621               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1622                 {
1623                   /*
1624                    * Send DHCP packets to the ipv4 stack, or we won't
1625                    * be able to use dhcp client on the outside interface
1626                    */
1627                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1628                       && (udp0->dst_port ==
1629                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1630                     {
1631                       vnet_feature_next
1632                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1633                       goto trace00;
1634                     }
1635
1636                   if (!sm->forwarding_enabled)
1637                     {
1638                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1639                       next0 = SNAT_OUT2IN_NEXT_DROP;
1640                       goto trace00;
1641                     }
1642                   else
1643                     {
1644                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1645                         {
1646                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1647                           goto trace00;
1648                         }
1649                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1650                       goto trace00;
1651                     }
1652                 }
1653
1654               /* Create session initiated by host from external network */
1655               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1656                                                      thread_index);
1657               if (!s0)
1658                 {
1659                   next0 = SNAT_OUT2IN_NEXT_DROP;
1660                   goto trace00;
1661                 }
1662             }
1663           else
1664             {
1665               if (PREDICT_FALSE (value0.value == ~0ULL))
1666                 {
1667                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1668                                       now, vm, node);
1669                   if (!s0)
1670                     next0 = SNAT_OUT2IN_NEXT_DROP;
1671                   goto trace00;
1672                 }
1673               else
1674                 {
1675                   s0 = pool_elt_at_index (
1676                     sm->per_thread_data[thread_index].sessions,
1677                     value0.value);
1678                 }
1679             }
1680
1681           old_addr0 = ip0->dst_address.as_u32;
1682           ip0->dst_address = s0->in2out.addr;
1683           new_addr0 = ip0->dst_address.as_u32;
1684           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1685
1686           sum0 = ip0->checksum;
1687           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1688                                  ip4_header_t,
1689                                  dst_address /* changed member */);
1690           ip0->checksum = ip_csum_fold (sum0);
1691
1692           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1693             {
1694               old_port0 = tcp0->dst_port;
1695               tcp0->dst_port = s0->in2out.port;
1696               new_port0 = tcp0->dst_port;
1697
1698               sum0 = tcp0->checksum;
1699               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1700                                      ip4_header_t,
1701                                      dst_address /* changed member */);
1702
1703               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1704                                      ip4_header_t /* cheat */,
1705                                      length /* changed member */);
1706               tcp0->checksum = ip_csum_fold(sum0);
1707             }
1708           else
1709             {
1710               old_port0 = udp0->dst_port;
1711               udp0->dst_port = s0->in2out.port;
1712               udp0->checksum = 0;
1713             }
1714
1715           /* Accounting */
1716           s0->last_heard = now;
1717           s0->total_pkts++;
1718           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1719           /* Per-user LRU list maintenance */
1720           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1721                              s0->per_user_index);
1722           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1723                               s0->per_user_list_head_index,
1724                               s0->per_user_index);
1725         trace00:
1726
1727           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1728                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1729             {
1730               snat_out2in_trace_t *t =
1731                  vlib_add_trace (vm, node, b0, sizeof (*t));
1732               t->sw_if_index = sw_if_index0;
1733               t->next_index = next0;
1734               t->session_index = ~0;
1735               if (s0)
1736                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1737             }
1738
1739           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1740
1741           /* verify speculative enqueue, maybe switch current next frame */
1742           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1743                                            to_next, n_left_to_next,
1744                                            bi0, next0);
1745         }
1746
1747       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1748     }
1749
1750   vlib_node_increment_counter (vm, snat_out2in_node.index,
1751                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1752                                pkts_processed);
1753   return frame->n_vectors;
1754 }
1755
1756 VLIB_REGISTER_NODE (snat_out2in_node) = {
1757   .function = snat_out2in_node_fn,
1758   .name = "nat44-out2in",
1759   .vector_size = sizeof (u32),
1760   .format_trace = format_snat_out2in_trace,
1761   .type = VLIB_NODE_TYPE_INTERNAL,
1762
1763   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1764   .error_strings = snat_out2in_error_strings,
1765
1766   .runtime_data_bytes = sizeof (snat_runtime_t),
1767
1768   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1769
1770   /* edit / add dispositions here */
1771   .next_nodes = {
1772     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1773     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1774     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1775     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1776     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1777   },
1778 };
1779 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1780
1781 static uword
1782 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1783                             vlib_node_runtime_t * node,
1784                             vlib_frame_t * frame)
1785 {
1786   u32 n_left_from, *from, *to_next;
1787   snat_out2in_next_t next_index;
1788   u32 pkts_processed = 0;
1789   snat_main_t *sm = &snat_main;
1790   f64 now = vlib_time_now (vm);
1791   u32 thread_index = vlib_get_thread_index ();
1792   snat_main_per_thread_data_t *per_thread_data =
1793     &sm->per_thread_data[thread_index];
1794   u32 *fragments_to_drop = 0;
1795   u32 *fragments_to_loopback = 0;
1796
1797   from = vlib_frame_vector_args (frame);
1798   n_left_from = frame->n_vectors;
1799   next_index = node->cached_next_index;
1800
1801   while (n_left_from > 0)
1802     {
1803       u32 n_left_to_next;
1804
1805       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1806
1807       while (n_left_from > 0 && n_left_to_next > 0)
1808        {
1809           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1810           vlib_buffer_t *b0;
1811           u32 next0;
1812           u8 cached0 = 0;
1813           ip4_header_t *ip0;
1814           nat_reass_ip4_t *reass0;
1815           udp_header_t * udp0;
1816           tcp_header_t * tcp0;
1817           snat_session_key_t key0, sm0;
1818           clib_bihash_kv_8_8_t kv0, value0;
1819           snat_session_t * s0 = 0;
1820           u16 old_port0, new_port0;
1821           ip_csum_t sum0;
1822
1823           /* speculatively enqueue b0 to the current next frame */
1824           bi0 = from[0];
1825           to_next[0] = bi0;
1826           from += 1;
1827           to_next += 1;
1828           n_left_from -= 1;
1829           n_left_to_next -= 1;
1830
1831           b0 = vlib_get_buffer (vm, bi0);
1832           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1833
1834           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1835           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1836                                                                sw_if_index0);
1837
1838           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1839             {
1840               next0 = SNAT_OUT2IN_NEXT_DROP;
1841               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1842               goto trace0;
1843             }
1844
1845           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1846           udp0 = ip4_next_header (ip0);
1847           tcp0 = (tcp_header_t *) udp0;
1848           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1849
1850           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1851                                                  ip0->dst_address,
1852                                                  ip0->fragment_id,
1853                                                  ip0->protocol,
1854                                                  1,
1855                                                  &fragments_to_drop);
1856
1857           if (PREDICT_FALSE (!reass0))
1858             {
1859               next0 = SNAT_OUT2IN_NEXT_DROP;
1860               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1861               goto trace0;
1862             }
1863
1864           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1865             {
1866               key0.addr = ip0->dst_address;
1867               key0.port = udp0->dst_port;
1868               key0.protocol = proto0;
1869               key0.fib_index = rx_fib_index0;
1870               kv0.key = key0.as_u64;
1871
1872               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1873                 {
1874                   /* Try to match static mapping by external address and port,
1875                      destination address and port in packet */
1876                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1877                     {
1878                       /*
1879                        * Send DHCP packets to the ipv4 stack, or we won't
1880                        * be able to use dhcp client on the outside interface
1881                        */
1882                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1883                           && (udp0->dst_port
1884                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1885                         {
1886                           vnet_feature_next
1887                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1888                              &next0, b0);
1889                           goto trace0;
1890                         }
1891
1892                       if (!sm->forwarding_enabled)
1893                         {
1894                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1895                           next0 = SNAT_OUT2IN_NEXT_DROP;
1896                           goto trace0;
1897                         }
1898                       else
1899                         {
1900                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1901                             {
1902                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1903                               goto trace0;
1904                             }
1905                           create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1906                           goto trace0;
1907                         }
1908                     }
1909
1910                   /* Create session initiated by host from external network */
1911                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1912                                                          thread_index);
1913                   if (!s0)
1914                     {
1915                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1916                       next0 = SNAT_OUT2IN_NEXT_DROP;
1917                       goto trace0;
1918                     }
1919                   reass0->sess_index = s0 - per_thread_data->sessions;
1920                   reass0->thread_index = thread_index;
1921                 }
1922               else
1923                 {
1924                   s0 = pool_elt_at_index (per_thread_data->sessions,
1925                                           value0.value);
1926                   reass0->sess_index = value0.value;
1927                 }
1928               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1929             }
1930           else
1931             {
1932               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1933                 {
1934                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1935                     {
1936                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1937                       next0 = SNAT_OUT2IN_NEXT_DROP;
1938                       goto trace0;
1939                     }
1940                   cached0 = 1;
1941                   goto trace0;
1942                 }
1943               s0 = pool_elt_at_index (per_thread_data->sessions,
1944                                       reass0->sess_index);
1945             }
1946
1947           old_addr0 = ip0->dst_address.as_u32;
1948           ip0->dst_address = s0->in2out.addr;
1949           new_addr0 = ip0->dst_address.as_u32;
1950           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1951
1952           sum0 = ip0->checksum;
1953           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1954                                  ip4_header_t,
1955                                  dst_address /* changed member */);
1956           ip0->checksum = ip_csum_fold (sum0);
1957
1958           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1959             {
1960               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1961                 {
1962                   old_port0 = tcp0->dst_port;
1963                   tcp0->dst_port = s0->in2out.port;
1964                   new_port0 = tcp0->dst_port;
1965
1966                   sum0 = tcp0->checksum;
1967                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1968                                          ip4_header_t,
1969                                          dst_address /* changed member */);
1970
1971                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1972                                          ip4_header_t /* cheat */,
1973                                          length /* changed member */);
1974                   tcp0->checksum = ip_csum_fold(sum0);
1975                 }
1976               else
1977                 {
1978                   old_port0 = udp0->dst_port;
1979                   udp0->dst_port = s0->in2out.port;
1980                   udp0->checksum = 0;
1981                 }
1982             }
1983
1984           /* Accounting */
1985           s0->last_heard = now;
1986           s0->total_pkts++;
1987           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1988           /* Per-user LRU list maintenance */
1989           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1990                              s0->per_user_index);
1991           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1992                               s0->per_user_list_head_index,
1993                               s0->per_user_index);
1994
1995         trace0:
1996           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1997                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1998             {
1999               nat44_out2in_reass_trace_t *t =
2000                  vlib_add_trace (vm, node, b0, sizeof (*t));
2001               t->cached = cached0;
2002               t->sw_if_index = sw_if_index0;
2003               t->next_index = next0;
2004             }
2005
2006           if (cached0)
2007             {
2008               n_left_to_next++;
2009               to_next--;
2010             }
2011           else
2012             {
2013               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2014
2015               /* verify speculative enqueue, maybe switch current next frame */
2016               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2017                                                to_next, n_left_to_next,
2018                                                bi0, next0);
2019             }
2020
2021           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2022             {
2023               from = vlib_frame_vector_args (frame);
2024               u32 len = vec_len (fragments_to_loopback);
2025               if (len <= VLIB_FRAME_SIZE)
2026                 {
2027                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2028                   n_left_from = len;
2029                   vec_reset_length (fragments_to_loopback);
2030                 }
2031               else
2032                 {
2033                   clib_memcpy (from,
2034                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2035                                sizeof (u32) * VLIB_FRAME_SIZE);
2036                   n_left_from = VLIB_FRAME_SIZE;
2037                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2038                 }
2039             }
2040        }
2041
2042       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2043     }
2044
2045   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2046                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2047                                pkts_processed);
2048
2049   nat_send_all_to_node (vm, fragments_to_drop, node,
2050                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2051                         SNAT_OUT2IN_NEXT_DROP);
2052
2053   vec_free (fragments_to_drop);
2054   vec_free (fragments_to_loopback);
2055   return frame->n_vectors;
2056 }
2057
2058 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2059   .function = nat44_out2in_reass_node_fn,
2060   .name = "nat44-out2in-reass",
2061   .vector_size = sizeof (u32),
2062   .format_trace = format_nat44_out2in_reass_trace,
2063   .type = VLIB_NODE_TYPE_INTERNAL,
2064
2065   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2066   .error_strings = snat_out2in_error_strings,
2067
2068   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2069
2070   /* edit / add dispositions here */
2071   .next_nodes = {
2072     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2073     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2074     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2075     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2076     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2077   },
2078 };
2079 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2080                               nat44_out2in_reass_node_fn);
2081
2082 /**************************/
2083 /*** deterministic mode ***/
2084 /**************************/
2085 static uword
2086 snat_det_out2in_node_fn (vlib_main_t * vm,
2087                          vlib_node_runtime_t * node,
2088                          vlib_frame_t * frame)
2089 {
2090   u32 n_left_from, * from, * to_next;
2091   snat_out2in_next_t next_index;
2092   u32 pkts_processed = 0;
2093   snat_main_t * sm = &snat_main;
2094   u32 thread_index = vlib_get_thread_index ();
2095
2096   from = vlib_frame_vector_args (frame);
2097   n_left_from = frame->n_vectors;
2098   next_index = node->cached_next_index;
2099
2100   while (n_left_from > 0)
2101     {
2102       u32 n_left_to_next;
2103
2104       vlib_get_next_frame (vm, node, next_index,
2105                            to_next, n_left_to_next);
2106
2107       while (n_left_from >= 4 && n_left_to_next >= 2)
2108         {
2109           u32 bi0, bi1;
2110           vlib_buffer_t * b0, * b1;
2111           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2112           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2113           u32 sw_if_index0, sw_if_index1;
2114           ip4_header_t * ip0, * ip1;
2115           ip_csum_t sum0, sum1;
2116           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2117           u16 new_port0, old_port0, old_port1, new_port1;
2118           udp_header_t * udp0, * udp1;
2119           tcp_header_t * tcp0, * tcp1;
2120           u32 proto0, proto1;
2121           snat_det_out_key_t key0, key1;
2122           snat_det_map_t * dm0, * dm1;
2123           snat_det_session_t * ses0 = 0, * ses1 = 0;
2124           u32 rx_fib_index0, rx_fib_index1;
2125           icmp46_header_t * icmp0, * icmp1;
2126
2127           /* Prefetch next iteration. */
2128           {
2129             vlib_buffer_t * p2, * p3;
2130
2131             p2 = vlib_get_buffer (vm, from[2]);
2132             p3 = vlib_get_buffer (vm, from[3]);
2133
2134             vlib_prefetch_buffer_header (p2, LOAD);
2135             vlib_prefetch_buffer_header (p3, LOAD);
2136
2137             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2138             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2139           }
2140
2141           /* speculatively enqueue b0 and b1 to the current next frame */
2142           to_next[0] = bi0 = from[0];
2143           to_next[1] = bi1 = from[1];
2144           from += 2;
2145           to_next += 2;
2146           n_left_from -= 2;
2147           n_left_to_next -= 2;
2148
2149           b0 = vlib_get_buffer (vm, bi0);
2150           b1 = vlib_get_buffer (vm, bi1);
2151
2152           ip0 = vlib_buffer_get_current (b0);
2153           udp0 = ip4_next_header (ip0);
2154           tcp0 = (tcp_header_t *) udp0;
2155
2156           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2157
2158           if (PREDICT_FALSE(ip0->ttl == 1))
2159             {
2160               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2161               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2162                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2163                                            0);
2164               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2165               goto trace0;
2166             }
2167
2168           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2169
2170           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2171             {
2172               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2173               icmp0 = (icmp46_header_t *) udp0;
2174
2175               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2176                                   rx_fib_index0, node, next0, thread_index,
2177                                   &ses0, &dm0);
2178               goto trace0;
2179             }
2180
2181           key0.ext_host_addr = ip0->src_address;
2182           key0.ext_host_port = tcp0->src;
2183           key0.out_port = tcp0->dst;
2184
2185           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2186           if (PREDICT_FALSE(!dm0))
2187             {
2188               clib_warning("unknown dst address:  %U",
2189                            format_ip4_address, &ip0->dst_address);
2190               next0 = SNAT_OUT2IN_NEXT_DROP;
2191               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2192               goto trace0;
2193             }
2194
2195           snat_det_reverse(dm0, &ip0->dst_address,
2196                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2197
2198           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2199           if (PREDICT_FALSE(!ses0))
2200             {
2201               clib_warning("no match src %U:%d dst %U:%d for user %U",
2202                            format_ip4_address, &ip0->src_address,
2203                            clib_net_to_host_u16 (tcp0->src),
2204                            format_ip4_address, &ip0->dst_address,
2205                            clib_net_to_host_u16 (tcp0->dst),
2206                            format_ip4_address, &new_addr0);
2207               next0 = SNAT_OUT2IN_NEXT_DROP;
2208               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2209               goto trace0;
2210             }
2211           new_port0 = ses0->in_port;
2212
2213           old_addr0 = ip0->dst_address;
2214           ip0->dst_address = new_addr0;
2215           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2216
2217           sum0 = ip0->checksum;
2218           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2219                                  ip4_header_t,
2220                                  dst_address /* changed member */);
2221           ip0->checksum = ip_csum_fold (sum0);
2222
2223           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2224             {
2225               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2226                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2227               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2228                 snat_det_ses_close(dm0, ses0);
2229
2230               old_port0 = tcp0->dst;
2231               tcp0->dst = new_port0;
2232
2233               sum0 = tcp0->checksum;
2234               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2235                                      ip4_header_t,
2236                                      dst_address /* changed member */);
2237
2238               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2239                                      ip4_header_t /* cheat */,
2240                                      length /* changed member */);
2241               tcp0->checksum = ip_csum_fold(sum0);
2242             }
2243           else
2244             {
2245               old_port0 = udp0->dst_port;
2246               udp0->dst_port = new_port0;
2247               udp0->checksum = 0;
2248             }
2249
2250         trace0:
2251
2252           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2253                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2254             {
2255               snat_out2in_trace_t *t =
2256                  vlib_add_trace (vm, node, b0, sizeof (*t));
2257               t->sw_if_index = sw_if_index0;
2258               t->next_index = next0;
2259               t->session_index = ~0;
2260               if (ses0)
2261                 t->session_index = ses0 - dm0->sessions;
2262             }
2263
2264           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2265
2266           b1 = vlib_get_buffer (vm, bi1);
2267
2268           ip1 = vlib_buffer_get_current (b1);
2269           udp1 = ip4_next_header (ip1);
2270           tcp1 = (tcp_header_t *) udp1;
2271
2272           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2273
2274           if (PREDICT_FALSE(ip1->ttl == 1))
2275             {
2276               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2277               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2278                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2279                                            0);
2280               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2281               goto trace1;
2282             }
2283
2284           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2285
2286           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2287             {
2288               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2289               icmp1 = (icmp46_header_t *) udp1;
2290
2291               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2292                                   rx_fib_index1, node, next1, thread_index,
2293                                   &ses1, &dm1);
2294               goto trace1;
2295             }
2296
2297           key1.ext_host_addr = ip1->src_address;
2298           key1.ext_host_port = tcp1->src;
2299           key1.out_port = tcp1->dst;
2300
2301           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2302           if (PREDICT_FALSE(!dm1))
2303             {
2304               clib_warning("unknown dst address:  %U",
2305                            format_ip4_address, &ip1->dst_address);
2306               next1 = SNAT_OUT2IN_NEXT_DROP;
2307               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2308               goto trace1;
2309             }
2310
2311           snat_det_reverse(dm1, &ip1->dst_address,
2312                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2313
2314           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2315           if (PREDICT_FALSE(!ses1))
2316             {
2317               clib_warning("no match src %U:%d dst %U:%d for user %U",
2318                            format_ip4_address, &ip1->src_address,
2319                            clib_net_to_host_u16 (tcp1->src),
2320                            format_ip4_address, &ip1->dst_address,
2321                            clib_net_to_host_u16 (tcp1->dst),
2322                            format_ip4_address, &new_addr1);
2323               next1 = SNAT_OUT2IN_NEXT_DROP;
2324               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2325               goto trace1;
2326             }
2327           new_port1 = ses1->in_port;
2328
2329           old_addr1 = ip1->dst_address;
2330           ip1->dst_address = new_addr1;
2331           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2332
2333           sum1 = ip1->checksum;
2334           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2335                                  ip4_header_t,
2336                                  dst_address /* changed member */);
2337           ip1->checksum = ip_csum_fold (sum1);
2338
2339           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2340             {
2341               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2342                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2343               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2344                 snat_det_ses_close(dm1, ses1);
2345
2346               old_port1 = tcp1->dst;
2347               tcp1->dst = new_port1;
2348
2349               sum1 = tcp1->checksum;
2350               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2351                                      ip4_header_t,
2352                                      dst_address /* changed member */);
2353
2354               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2355                                      ip4_header_t /* cheat */,
2356                                      length /* changed member */);
2357               tcp1->checksum = ip_csum_fold(sum1);
2358             }
2359           else
2360             {
2361               old_port1 = udp1->dst_port;
2362               udp1->dst_port = new_port1;
2363               udp1->checksum = 0;
2364             }
2365
2366         trace1:
2367
2368           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2369                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2370             {
2371               snat_out2in_trace_t *t =
2372                  vlib_add_trace (vm, node, b1, sizeof (*t));
2373               t->sw_if_index = sw_if_index1;
2374               t->next_index = next1;
2375               t->session_index = ~0;
2376               if (ses1)
2377                 t->session_index = ses1 - dm1->sessions;
2378             }
2379
2380           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2381
2382           /* verify speculative enqueues, maybe switch current next frame */
2383           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2384                                            to_next, n_left_to_next,
2385                                            bi0, bi1, next0, next1);
2386          }
2387
2388       while (n_left_from > 0 && n_left_to_next > 0)
2389         {
2390           u32 bi0;
2391           vlib_buffer_t * b0;
2392           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2393           u32 sw_if_index0;
2394           ip4_header_t * ip0;
2395           ip_csum_t sum0;
2396           ip4_address_t new_addr0, old_addr0;
2397           u16 new_port0, old_port0;
2398           udp_header_t * udp0;
2399           tcp_header_t * tcp0;
2400           u32 proto0;
2401           snat_det_out_key_t key0;
2402           snat_det_map_t * dm0;
2403           snat_det_session_t * ses0 = 0;
2404           u32 rx_fib_index0;
2405           icmp46_header_t * icmp0;
2406
2407           /* speculatively enqueue b0 to the current next frame */
2408           bi0 = from[0];
2409           to_next[0] = bi0;
2410           from += 1;
2411           to_next += 1;
2412           n_left_from -= 1;
2413           n_left_to_next -= 1;
2414
2415           b0 = vlib_get_buffer (vm, bi0);
2416
2417           ip0 = vlib_buffer_get_current (b0);
2418           udp0 = ip4_next_header (ip0);
2419           tcp0 = (tcp_header_t *) udp0;
2420
2421           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2422
2423           if (PREDICT_FALSE(ip0->ttl == 1))
2424             {
2425               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2426               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2427                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2428                                            0);
2429               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2430               goto trace00;
2431             }
2432
2433           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2434
2435           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2436             {
2437               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2438               icmp0 = (icmp46_header_t *) udp0;
2439
2440               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2441                                   rx_fib_index0, node, next0, thread_index,
2442                                   &ses0, &dm0);
2443               goto trace00;
2444             }
2445
2446           key0.ext_host_addr = ip0->src_address;
2447           key0.ext_host_port = tcp0->src;
2448           key0.out_port = tcp0->dst;
2449
2450           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2451           if (PREDICT_FALSE(!dm0))
2452             {
2453               clib_warning("unknown dst address:  %U",
2454                            format_ip4_address, &ip0->dst_address);
2455               next0 = SNAT_OUT2IN_NEXT_DROP;
2456               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2457               goto trace00;
2458             }
2459
2460           snat_det_reverse(dm0, &ip0->dst_address,
2461                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2462
2463           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2464           if (PREDICT_FALSE(!ses0))
2465             {
2466               clib_warning("no match src %U:%d dst %U:%d for user %U",
2467                            format_ip4_address, &ip0->src_address,
2468                            clib_net_to_host_u16 (tcp0->src),
2469                            format_ip4_address, &ip0->dst_address,
2470                            clib_net_to_host_u16 (tcp0->dst),
2471                            format_ip4_address, &new_addr0);
2472               next0 = SNAT_OUT2IN_NEXT_DROP;
2473               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2474               goto trace00;
2475             }
2476           new_port0 = ses0->in_port;
2477
2478           old_addr0 = ip0->dst_address;
2479           ip0->dst_address = new_addr0;
2480           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2481
2482           sum0 = ip0->checksum;
2483           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2484                                  ip4_header_t,
2485                                  dst_address /* changed member */);
2486           ip0->checksum = ip_csum_fold (sum0);
2487
2488           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2489             {
2490               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2491                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2492               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2493                 snat_det_ses_close(dm0, ses0);
2494
2495               old_port0 = tcp0->dst;
2496               tcp0->dst = new_port0;
2497
2498               sum0 = tcp0->checksum;
2499               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2500                                      ip4_header_t,
2501                                      dst_address /* changed member */);
2502
2503               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2504                                      ip4_header_t /* cheat */,
2505                                      length /* changed member */);
2506               tcp0->checksum = ip_csum_fold(sum0);
2507             }
2508           else
2509             {
2510               old_port0 = udp0->dst_port;
2511               udp0->dst_port = new_port0;
2512               udp0->checksum = 0;
2513             }
2514
2515         trace00:
2516
2517           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2518                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2519             {
2520               snat_out2in_trace_t *t =
2521                  vlib_add_trace (vm, node, b0, sizeof (*t));
2522               t->sw_if_index = sw_if_index0;
2523               t->next_index = next0;
2524               t->session_index = ~0;
2525               if (ses0)
2526                 t->session_index = ses0 - dm0->sessions;
2527             }
2528
2529           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2530
2531           /* verify speculative enqueue, maybe switch current next frame */
2532           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2533                                            to_next, n_left_to_next,
2534                                            bi0, next0);
2535         }
2536
2537       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2538     }
2539
2540   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2541                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2542                                pkts_processed);
2543   return frame->n_vectors;
2544 }
2545
2546 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2547   .function = snat_det_out2in_node_fn,
2548   .name = "nat44-det-out2in",
2549   .vector_size = sizeof (u32),
2550   .format_trace = format_snat_out2in_trace,
2551   .type = VLIB_NODE_TYPE_INTERNAL,
2552
2553   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2554   .error_strings = snat_out2in_error_strings,
2555
2556   .runtime_data_bytes = sizeof (snat_runtime_t),
2557
2558   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2559
2560   /* edit / add dispositions here */
2561   .next_nodes = {
2562     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2563     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2564     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2565     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2566     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2567   },
2568 };
2569 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2570
2571 /**
2572  * Get address and port values to be used for ICMP packet translation
2573  * and create session if needed
2574  *
2575  * @param[in,out] sm             NAT main
2576  * @param[in,out] node           NAT node runtime
2577  * @param[in] thread_index       thread index
2578  * @param[in,out] b0             buffer containing packet to be translated
2579  * @param[out] p_proto           protocol used for matching
2580  * @param[out] p_value           address and port after NAT translation
2581  * @param[out] p_dont_translate  if packet should not be translated
2582  * @param d                      optional parameter
2583  * @param e                      optional parameter
2584  */
2585 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2586                           u32 thread_index, vlib_buffer_t *b0,
2587                           ip4_header_t *ip0, u8 *p_proto,
2588                           snat_session_key_t *p_value,
2589                           u8 *p_dont_translate, void *d, void *e)
2590 {
2591   icmp46_header_t *icmp0;
2592   u32 sw_if_index0;
2593   u8 protocol;
2594   snat_det_out_key_t key0;
2595   u8 dont_translate = 0;
2596   u32 next0 = ~0;
2597   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2598   ip4_header_t *inner_ip0;
2599   void *l4_header = 0;
2600   icmp46_header_t *inner_icmp0;
2601   snat_det_map_t * dm0 = 0;
2602   ip4_address_t new_addr0 = {{0}};
2603   snat_det_session_t * ses0 = 0;
2604   ip4_address_t out_addr;
2605
2606   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2607   echo0 = (icmp_echo_header_t *)(icmp0+1);
2608   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2609
2610   if (!icmp_is_error_message (icmp0))
2611     {
2612       protocol = SNAT_PROTOCOL_ICMP;
2613       key0.ext_host_addr = ip0->src_address;
2614       key0.ext_host_port = 0;
2615       key0.out_port = echo0->identifier;
2616       out_addr = ip0->dst_address;
2617     }
2618   else
2619     {
2620       inner_ip0 = (ip4_header_t *)(echo0+1);
2621       l4_header = ip4_next_header (inner_ip0);
2622       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2623       key0.ext_host_addr = inner_ip0->dst_address;
2624       out_addr = inner_ip0->src_address;
2625       switch (protocol)
2626         {
2627         case SNAT_PROTOCOL_ICMP:
2628           inner_icmp0 = (icmp46_header_t*)l4_header;
2629           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2630           key0.ext_host_port = 0;
2631           key0.out_port = inner_echo0->identifier;
2632           break;
2633         case SNAT_PROTOCOL_UDP:
2634         case SNAT_PROTOCOL_TCP:
2635           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2636           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2637           break;
2638         default:
2639           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2640           next0 = SNAT_OUT2IN_NEXT_DROP;
2641           goto out;
2642         }
2643     }
2644
2645   dm0 = snat_det_map_by_out(sm, &out_addr);
2646   if (PREDICT_FALSE(!dm0))
2647     {
2648       /* Don't NAT packet aimed at the intfc address */
2649       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2650                                           ip0->dst_address.as_u32)))
2651         {
2652           dont_translate = 1;
2653           goto out;
2654         }
2655       clib_warning("unknown dst address:  %U",
2656                    format_ip4_address, &ip0->dst_address);
2657       goto out;
2658     }
2659
2660   snat_det_reverse(dm0, &ip0->dst_address,
2661                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2662
2663   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2664   if (PREDICT_FALSE(!ses0))
2665     {
2666       /* Don't NAT packet aimed at the intfc address */
2667       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2668                                           ip0->dst_address.as_u32)))
2669         {
2670           dont_translate = 1;
2671           goto out;
2672         }
2673       clib_warning("no match src %U:%d dst %U:%d for user %U",
2674                    format_ip4_address, &key0.ext_host_addr,
2675                    clib_net_to_host_u16 (key0.ext_host_port),
2676                    format_ip4_address, &out_addr,
2677                    clib_net_to_host_u16 (key0.out_port),
2678                    format_ip4_address, &new_addr0);
2679       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2680       next0 = SNAT_OUT2IN_NEXT_DROP;
2681       goto out;
2682     }
2683
2684   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2685                     !icmp_is_error_message (icmp0)))
2686     {
2687       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2688       next0 = SNAT_OUT2IN_NEXT_DROP;
2689       goto out;
2690     }
2691
2692   goto out;
2693
2694 out:
2695   *p_proto = protocol;
2696   if (ses0)
2697     {
2698       p_value->addr = new_addr0;
2699       p_value->fib_index = sm->inside_fib_index;
2700       p_value->port = ses0->in_port;
2701     }
2702   *p_dont_translate = dont_translate;
2703   if (d)
2704     *(snat_det_session_t**)d = ses0;
2705   if (e)
2706     *(snat_det_map_t**)e = dm0;
2707   return next0;
2708 }
2709
2710 /**********************/
2711 /*** worker handoff ***/
2712 /**********************/
2713 static uword
2714 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2715                                vlib_node_runtime_t * node,
2716                                vlib_frame_t * frame)
2717 {
2718   snat_main_t *sm = &snat_main;
2719   vlib_thread_main_t *tm = vlib_get_thread_main ();
2720   u32 n_left_from, *from, *to_next = 0;
2721   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2722   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2723     = 0;
2724   vlib_frame_queue_elt_t *hf = 0;
2725   vlib_frame_t *f = 0;
2726   int i;
2727   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2728   u32 next_worker_index = 0;
2729   u32 current_worker_index = ~0;
2730   u32 thread_index = vlib_get_thread_index ();
2731
2732   ASSERT (vec_len (sm->workers));
2733
2734   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2735     {
2736       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2737
2738       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2739                                sm->first_worker_index + sm->num_workers - 1,
2740                                (vlib_frame_queue_t *) (~0));
2741     }
2742
2743   from = vlib_frame_vector_args (frame);
2744   n_left_from = frame->n_vectors;
2745
2746   while (n_left_from > 0)
2747     {
2748       u32 bi0;
2749       vlib_buffer_t *b0;
2750       u32 sw_if_index0;
2751       u32 rx_fib_index0;
2752       ip4_header_t * ip0;
2753       u8 do_handoff;
2754
2755       bi0 = from[0];
2756       from += 1;
2757       n_left_from -= 1;
2758
2759       b0 = vlib_get_buffer (vm, bi0);
2760
2761       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2762       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2763
2764       ip0 = vlib_buffer_get_current (b0);
2765
2766       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2767
2768       if (PREDICT_FALSE (next_worker_index != thread_index))
2769         {
2770           do_handoff = 1;
2771
2772           if (next_worker_index != current_worker_index)
2773             {
2774               if (hf)
2775                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2776
2777               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2778                                                       next_worker_index,
2779                                                       handoff_queue_elt_by_worker_index);
2780
2781               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2782               to_next_worker = &hf->buffer_index[hf->n_vectors];
2783               current_worker_index = next_worker_index;
2784             }
2785
2786           /* enqueue to correct worker thread */
2787           to_next_worker[0] = bi0;
2788           to_next_worker++;
2789           n_left_to_next_worker--;
2790
2791           if (n_left_to_next_worker == 0)
2792             {
2793               hf->n_vectors = VLIB_FRAME_SIZE;
2794               vlib_put_frame_queue_elt (hf);
2795               current_worker_index = ~0;
2796               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2797               hf = 0;
2798             }
2799         }
2800       else
2801         {
2802           do_handoff = 0;
2803           /* if this is 1st frame */
2804           if (!f)
2805             {
2806               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2807               to_next = vlib_frame_vector_args (f);
2808             }
2809
2810           to_next[0] = bi0;
2811           to_next += 1;
2812           f->n_vectors++;
2813         }
2814
2815       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2816                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2817         {
2818           snat_out2in_worker_handoff_trace_t *t =
2819             vlib_add_trace (vm, node, b0, sizeof (*t));
2820           t->next_worker_index = next_worker_index;
2821           t->do_handoff = do_handoff;
2822         }
2823     }
2824
2825   if (f)
2826     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2827
2828   if (hf)
2829     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2830
2831   /* Ship frames to the worker nodes */
2832   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2833     {
2834       if (handoff_queue_elt_by_worker_index[i])
2835         {
2836           hf = handoff_queue_elt_by_worker_index[i];
2837           /*
2838            * It works better to let the handoff node
2839            * rate-adapt, always ship the handoff queue element.
2840            */
2841           if (1 || hf->n_vectors == hf->last_n_vectors)
2842             {
2843               vlib_put_frame_queue_elt (hf);
2844               handoff_queue_elt_by_worker_index[i] = 0;
2845             }
2846           else
2847             hf->last_n_vectors = hf->n_vectors;
2848         }
2849       congested_handoff_queue_by_worker_index[i] =
2850         (vlib_frame_queue_t *) (~0);
2851     }
2852   hf = 0;
2853   current_worker_index = ~0;
2854   return frame->n_vectors;
2855 }
2856
2857 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2858   .function = snat_out2in_worker_handoff_fn,
2859   .name = "nat44-out2in-worker-handoff",
2860   .vector_size = sizeof (u32),
2861   .format_trace = format_snat_out2in_worker_handoff_trace,
2862   .type = VLIB_NODE_TYPE_INTERNAL,
2863
2864   .n_next_nodes = 1,
2865
2866   .next_nodes = {
2867     [0] = "error-drop",
2868   },
2869 };
2870
2871 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2872
2873 static uword
2874 snat_out2in_fast_node_fn (vlib_main_t * vm,
2875                           vlib_node_runtime_t * node,
2876                           vlib_frame_t * frame)
2877 {
2878   u32 n_left_from, * from, * to_next;
2879   snat_out2in_next_t next_index;
2880   u32 pkts_processed = 0;
2881   snat_main_t * sm = &snat_main;
2882
2883   from = vlib_frame_vector_args (frame);
2884   n_left_from = frame->n_vectors;
2885   next_index = node->cached_next_index;
2886
2887   while (n_left_from > 0)
2888     {
2889       u32 n_left_to_next;
2890
2891       vlib_get_next_frame (vm, node, next_index,
2892                            to_next, n_left_to_next);
2893
2894       while (n_left_from > 0 && n_left_to_next > 0)
2895         {
2896           u32 bi0;
2897           vlib_buffer_t * b0;
2898           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2899           u32 sw_if_index0;
2900           ip4_header_t * ip0;
2901           ip_csum_t sum0;
2902           u32 new_addr0, old_addr0;
2903           u16 new_port0, old_port0;
2904           udp_header_t * udp0;
2905           tcp_header_t * tcp0;
2906           icmp46_header_t * icmp0;
2907           snat_session_key_t key0, sm0;
2908           u32 proto0;
2909           u32 rx_fib_index0;
2910
2911           /* speculatively enqueue b0 to the current next frame */
2912           bi0 = from[0];
2913           to_next[0] = bi0;
2914           from += 1;
2915           to_next += 1;
2916           n_left_from -= 1;
2917           n_left_to_next -= 1;
2918
2919           b0 = vlib_get_buffer (vm, bi0);
2920
2921           ip0 = vlib_buffer_get_current (b0);
2922           udp0 = ip4_next_header (ip0);
2923           tcp0 = (tcp_header_t *) udp0;
2924           icmp0 = (icmp46_header_t *) udp0;
2925
2926           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2927           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2928
2929           vnet_feature_next (sw_if_index0, &next0, b0);
2930
2931           if (PREDICT_FALSE(ip0->ttl == 1))
2932             {
2933               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2934               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2935                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2936                                            0);
2937               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2938               goto trace00;
2939             }
2940
2941           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2942
2943           if (PREDICT_FALSE (proto0 == ~0))
2944               goto trace00;
2945
2946           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2947             {
2948               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2949                                   rx_fib_index0, node, next0, ~0, 0, 0);
2950               goto trace00;
2951             }
2952
2953           key0.addr = ip0->dst_address;
2954           key0.port = udp0->dst_port;
2955           key0.fib_index = rx_fib_index0;
2956
2957           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2958             {
2959               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2960               goto trace00;
2961             }
2962
2963           new_addr0 = sm0.addr.as_u32;
2964           new_port0 = sm0.port;
2965           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2966           old_addr0 = ip0->dst_address.as_u32;
2967           ip0->dst_address.as_u32 = new_addr0;
2968
2969           sum0 = ip0->checksum;
2970           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2971                                  ip4_header_t,
2972                                  dst_address /* changed member */);
2973           ip0->checksum = ip_csum_fold (sum0);
2974
2975           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2976             {
2977                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2978                 {
2979                   old_port0 = tcp0->dst_port;
2980                   tcp0->dst_port = new_port0;
2981
2982                   sum0 = tcp0->checksum;
2983                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2984                                          ip4_header_t,
2985                                          dst_address /* changed member */);
2986
2987                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2988                                          ip4_header_t /* cheat */,
2989                                          length /* changed member */);
2990                   tcp0->checksum = ip_csum_fold(sum0);
2991                 }
2992               else
2993                 {
2994                   old_port0 = udp0->dst_port;
2995                   udp0->dst_port = new_port0;
2996                   udp0->checksum = 0;
2997                 }
2998             }
2999           else
3000             {
3001               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3002                 {
3003                   sum0 = tcp0->checksum;
3004                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3005                                          ip4_header_t,
3006                                          dst_address /* changed member */);
3007
3008                   tcp0->checksum = ip_csum_fold(sum0);
3009                 }
3010             }
3011
3012         trace00:
3013
3014           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3015                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3016             {
3017               snat_out2in_trace_t *t =
3018                  vlib_add_trace (vm, node, b0, sizeof (*t));
3019               t->sw_if_index = sw_if_index0;
3020               t->next_index = next0;
3021             }
3022
3023           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3024
3025           /* verify speculative enqueue, maybe switch current next frame */
3026           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3027                                            to_next, n_left_to_next,
3028                                            bi0, next0);
3029         }
3030
3031       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3032     }
3033
3034   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3035                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3036                                pkts_processed);
3037   return frame->n_vectors;
3038 }
3039
3040 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3041   .function = snat_out2in_fast_node_fn,
3042   .name = "nat44-out2in-fast",
3043   .vector_size = sizeof (u32),
3044   .format_trace = format_snat_out2in_fast_trace,
3045   .type = VLIB_NODE_TYPE_INTERNAL,
3046
3047   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3048   .error_strings = snat_out2in_error_strings,
3049
3050   .runtime_data_bytes = sizeof (snat_runtime_t),
3051
3052   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3053
3054   /* edit / add dispositions here */
3055   .next_nodes = {
3056     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3057     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3058     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3059     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3060     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3061   },
3062 };
3063 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);