src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   user_session_increment (sm, u, 1 /* static */);
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 337                       u32 thread_index)
 338 {
 339   nat_ed_ses_key_t key;
 340   clib_bihash_kv_16_8_t kv, value;
 341   udp_header_t *udp;
 342   snat_user_t *u;
 343   snat_session_t *s = 0;
 344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 345
 346   if (ip->protocol == IP_PROTOCOL_ICMP)
 347     {
 348       if (icmp_get_ed_key (ip, &key))
 349         return;
 350     }
 351   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 352     {
 353       udp = ip4_next_header(ip);
 354       key.r_addr = ip->src_address;
 355       key.l_addr = ip->dst_address;
 356       key.proto = ip->protocol;
 357       key.l_port = udp->dst_port;
 358       key.r_port = udp->src_port;
 359     }
 360   else
 361     {
 362       key.r_addr = ip->src_address;
 363       key.l_addr = ip->dst_address;
 364       key.proto = ip->protocol;
 365       key.l_port = key.r_port = 0;
 366     }
 367   key.fib_index = 0;
 368   kv.key[0] = key.as_u64[0];
 369   kv.key[1] = key.as_u64[1];
 370
 371   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 372     {
 373       s = pool_elt_at_index (tsm->sessions, value.value);
 374     }
 375   else
 376     {
 377       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 378         return;
 379
 380       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
 381       if (!u)
 382         {
 383           clib_warning ("create NAT user failed");
 384           return;
 385         }
 386
 387       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 388       if (!s)
 389         {
 390           clib_warning ("create NAT session failed");
 391           return;
 392         }
 393
 394       s->ext_host_addr = key.r_addr;
 395       s->ext_host_port = key.r_port;
 396       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 397       s->outside_address_index = ~0;
 398       s->out2in.addr = key.l_addr;
 399       s->out2in.port = key.l_port;
 400       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 401       s->out2in.fib_index = 0;
 402       s->in2out = s->out2in;
 403       user_session_increment (sm, u, 0);
 404
 405       kv.value = s - tsm->sessions;
 406       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 407         clib_warning ("in2out_ed key add failed");
 408     }
 409
 410   /* Per-user LRU list maintenance */
 411   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 412   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 413                       s->per_user_index);
 414 }
 415
 416 /**
 417  * Get address and port values to be used for ICMP packet translation
 418  * and create session if needed
 419  *
 420  * @param[in,out] sm             NAT main
 421  * @param[in,out] node           NAT node runtime
 422  * @param[in] thread_index       thread index
 423  * @param[in,out] b0             buffer containing packet to be translated
 424  * @param[out] p_proto           protocol used for matching
 425  * @param[out] p_value           address and port after NAT translation
 426  * @param[out] p_dont_translate  if packet should not be translated
 427  * @param d                      optional parameter
 428  * @param e                      optional parameter
 429  */
 430 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 431                            u32 thread_index, vlib_buffer_t *b0,
 432                            ip4_header_t *ip0, u8 *p_proto,
 433                            snat_session_key_t *p_value,
 434                            u8 *p_dont_translate, void *d, void *e)
 435 {
 436   icmp46_header_t *icmp0;
 437   u32 sw_if_index0;
 438   u32 rx_fib_index0;
 439   snat_session_key_t key0;
 440   snat_session_key_t sm0;
 441   snat_session_t *s0 = 0;
 442   u8 dont_translate = 0;
 443   clib_bihash_kv_8_8_t kv0, value0;
 444   u8 is_addr_only;
 445   u32 next0 = ~0;
 446   int err;
 447
 448   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 449   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 450   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 451
 452   key0.protocol = 0;
 453
 454   err = icmp_get_key (ip0, &key0);
 455   if (err != -1)
 456     {
 457       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 458       next0 = SNAT_OUT2IN_NEXT_DROP;
 459       goto out;
 460     }
 461   key0.fib_index = rx_fib_index0;
 462
 463   kv0.key = key0.as_u64;
 464
 465   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 466                               &value0))
 467     {
 468       /* Try to match static mapping by external address and port,
 469          destination address and port in packet */
 470       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 471         {
 472           if (!sm->forwarding_enabled)
 473             {
 474               /* Don't NAT packet aimed at the intfc address */
 475               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 476                                                   ip0->dst_address.as_u32)))
 477                 {
 478                   dont_translate = 1;
 479                   goto out;
 480                 }
 481               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 482               next0 = SNAT_OUT2IN_NEXT_DROP;
 483               goto out;
 484             }
 485           else
 486             {
 487               dont_translate = 1;
 488               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 489                 {
 490                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 491                   goto out;
 492                 }
 493               create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
 494               goto out;
 495             }
 496         }
 497
 498       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 499                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 500         {
 501           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 502           next0 = SNAT_OUT2IN_NEXT_DROP;
 503           goto out;
 504         }
 505
 506       /* Create session initiated by host from external network */
 507       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 508                                              node, thread_index);
 509
 510       if (!s0)
 511         {
 512           next0 = SNAT_OUT2IN_NEXT_DROP;
 513           goto out;
 514         }
 515     }
 516   else
 517     {
 518       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 519                         icmp0->type != ICMP4_echo_request &&
 520                         !icmp_is_error_message (icmp0)))
 521         {
 522           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 523           next0 = SNAT_OUT2IN_NEXT_DROP;
 524           goto out;
 525         }
 526
 527       if (PREDICT_FALSE (value0.value == ~0ULL))
 528         {
 529           nat_ed_ses_key_t key;
 530           clib_bihash_kv_16_8_t s_kv, s_value;
 531
 532           key.as_u64[0] = 0;
 533           key.as_u64[1] = 0;
 534           if (icmp_get_ed_key (ip0, &key))
 535             {
 536               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 537               next0 = SNAT_OUT2IN_NEXT_DROP;
 538               goto out;
 539             }
 540           key.fib_index = rx_fib_index0;
 541           s_kv.key[0] = key.as_u64[0];
 542           s_kv.key[1] = key.as_u64[1];
 543           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 544             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 545                                     s_value.value);
 546           else
 547            {
 548               next0 = SNAT_OUT2IN_NEXT_DROP;
 549               goto out;
 550            }
 551         }
 552       else
 553         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 554                                 value0.value);
 555     }
 556
 557 out:
 558   *p_proto = key0.protocol;
 559   if (s0)
 560     *p_value = s0->in2out;
 561   *p_dont_translate = dont_translate;
 562   if (d)
 563     *(snat_session_t**)d = s0;
 564   return next0;
 565 }
 566
 567 /**
 568  * Get address and port values to be used for ICMP packet translation
 569  *
 570  * @param[in] sm                 NAT main
 571  * @param[in,out] node           NAT node runtime
 572  * @param[in] thread_index       thread index
 573  * @param[in,out] b0             buffer containing packet to be translated
 574  * @param[out] p_proto           protocol used for matching
 575  * @param[out] p_value           address and port after NAT translation
 576  * @param[out] p_dont_translate  if packet should not be translated
 577  * @param d                      optional parameter
 578  * @param e                      optional parameter
 579  */
 580 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 581                            u32 thread_index, vlib_buffer_t *b0,
 582                            ip4_header_t *ip0, u8 *p_proto,
 583                            snat_session_key_t *p_value,
 584                            u8 *p_dont_translate, void *d, void *e)
 585 {
 586   icmp46_header_t *icmp0;
 587   u32 sw_if_index0;
 588   u32 rx_fib_index0;
 589   snat_session_key_t key0;
 590   snat_session_key_t sm0;
 591   u8 dont_translate = 0;
 592   u8 is_addr_only;
 593   u32 next0 = ~0;
 594   int err;
 595
 596   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 597   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 598   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 599
 600   err = icmp_get_key (ip0, &key0);
 601   if (err != -1)
 602     {
 603       b0->error = node->errors[err];
 604       next0 = SNAT_OUT2IN_NEXT_DROP;
 605       goto out2;
 606     }
 607   key0.fib_index = rx_fib_index0;
 608
 609   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 610     {
 611       /* Don't NAT packet aimed at the intfc address */
 612       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 613         {
 614           dont_translate = 1;
 615           goto out;
 616         }
 617       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 618       next0 = SNAT_OUT2IN_NEXT_DROP;
 619       goto out;
 620     }
 621
 622   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 623                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 624                     !icmp_is_error_message (icmp0)))
 625     {
 626       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 627       next0 = SNAT_OUT2IN_NEXT_DROP;
 628       goto out;
 629     }
 630
 631 out:
 632   *p_value = sm0;
 633 out2:
 634   *p_proto = key0.protocol;
 635   *p_dont_translate = dont_translate;
 636   return next0;
 637 }
 638
 639 static inline u32 icmp_out2in (snat_main_t *sm,
 640                                vlib_buffer_t * b0,
 641                                ip4_header_t * ip0,
 642                                icmp46_header_t * icmp0,
 643                                u32 sw_if_index0,
 644                                u32 rx_fib_index0,
 645                                vlib_node_runtime_t * node,
 646                                u32 next0,
 647                                u32 thread_index,
 648                                void *d,
 649                                void *e)
 650 {
 651   snat_session_key_t sm0;
 652   u8 protocol;
 653   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 654   ip4_header_t *inner_ip0 = 0;
 655   void *l4_header = 0;
 656   icmp46_header_t *inner_icmp0;
 657   u8 dont_translate;
 658   u32 new_addr0, old_addr0;
 659   u16 old_id0, new_id0;
 660   ip_csum_t sum0;
 661   u16 checksum0;
 662   u32 next0_tmp;
 663
 664   echo0 = (icmp_echo_header_t *)(icmp0+1);
 665
 666   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 667                                        &protocol, &sm0, &dont_translate, d, e);
 668   if (next0_tmp != ~0)
 669     next0 = next0_tmp;
 670   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 671     goto out;
 672
 673   sum0 = ip_incremental_checksum (0, icmp0,
 674                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 675   checksum0 = ~ip_csum_fold (sum0);
 676   if (checksum0 != 0 && checksum0 != 0xffff)
 677     {
 678       next0 = SNAT_OUT2IN_NEXT_DROP;
 679       goto out;
 680     }
 681
 682   old_addr0 = ip0->dst_address.as_u32;
 683   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 684   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 685
 686   sum0 = ip0->checksum;
 687   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 688                          dst_address /* changed member */);
 689   ip0->checksum = ip_csum_fold (sum0);
 690
 691   if (icmp0->checksum == 0)
 692     icmp0->checksum = 0xffff;
 693
 694   if (!icmp_is_error_message (icmp0))
 695     {
 696       new_id0 = sm0.port;
 697       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 698         {
 699           old_id0 = echo0->identifier;
 700           new_id0 = sm0.port;
 701           echo0->identifier = new_id0;
 702
 703           sum0 = icmp0->checksum;
 704           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 705                                  identifier /* changed member */);
 706           icmp0->checksum = ip_csum_fold (sum0);
 707         }
 708     }
 709   else
 710     {
 711       inner_ip0 = (ip4_header_t *)(echo0+1);
 712       l4_header = ip4_next_header (inner_ip0);
 713
 714       if (!ip4_header_checksum_is_valid (inner_ip0))
 715         {
 716           next0 = SNAT_OUT2IN_NEXT_DROP;
 717           goto out;
 718         }
 719
 720       old_addr0 = inner_ip0->src_address.as_u32;
 721       inner_ip0->src_address = sm0.addr;
 722       new_addr0 = inner_ip0->src_address.as_u32;
 723
 724       sum0 = icmp0->checksum;
 725       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 726                              src_address /* changed member */);
 727       icmp0->checksum = ip_csum_fold (sum0);
 728
 729       switch (protocol)
 730         {
 731         case SNAT_PROTOCOL_ICMP:
 732           inner_icmp0 = (icmp46_header_t*)l4_header;
 733           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 734
 735           old_id0 = inner_echo0->identifier;
 736           new_id0 = sm0.port;
 737           inner_echo0->identifier = new_id0;
 738
 739           sum0 = icmp0->checksum;
 740           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 741                                  identifier);
 742           icmp0->checksum = ip_csum_fold (sum0);
 743           break;
 744         case SNAT_PROTOCOL_UDP:
 745         case SNAT_PROTOCOL_TCP:
 746           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 747           new_id0 = sm0.port;
 748           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 749
 750           sum0 = icmp0->checksum;
 751           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 752                                  src_port);
 753           icmp0->checksum = ip_csum_fold (sum0);
 754           break;
 755         default:
 756           ASSERT(0);
 757         }
 758     }
 759
 760 out:
 761   return next0;
 762 }
 763
 764
 765 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 766                                          vlib_buffer_t * b0,
 767                                          ip4_header_t * ip0,
 768                                          icmp46_header_t * icmp0,
 769                                          u32 sw_if_index0,
 770                                          u32 rx_fib_index0,
 771                                          vlib_node_runtime_t * node,
 772                                          u32 next0, f64 now,
 773                                          u32 thread_index,
 774                                          snat_session_t ** p_s0)
 775 {
 776   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 777                       next0, thread_index, p_s0, 0);
 778   snat_session_t * s0 = *p_s0;
 779   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 780     {
 781       /* Accounting */
 782       s0->last_heard = now;
 783       s0->total_pkts++;
 784       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 785       /* Per-user LRU list maintenance */
 786       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 787                          s0->per_user_index);
 788       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 789                           s0->per_user_list_head_index,
 790                           s0->per_user_index);
 791     }
 792   return next0;
 793 }
 794
 795 static snat_session_t *
 796 snat_out2in_unknown_proto (snat_main_t *sm,
 797                            vlib_buffer_t * b,
 798                            ip4_header_t * ip,
 799                            u32 rx_fib_index,
 800                            u32 thread_index,
 801                            f64 now,
 802                            vlib_main_t * vm,
 803                            vlib_node_runtime_t * node)
 804 {
 805   clib_bihash_kv_8_8_t kv, value;
 806   clib_bihash_kv_16_8_t s_kv, s_value;
 807   snat_static_mapping_t *m;
 808   snat_session_key_t m_key;
 809   u32 old_addr, new_addr;
 810   ip_csum_t sum;
 811   nat_ed_ses_key_t key;
 812   snat_session_t * s;
 813   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 814   snat_user_t *u;
 815
 816   old_addr = ip->dst_address.as_u32;
 817
 818   key.l_addr = ip->dst_address;
 819   key.r_addr = ip->src_address;
 820   key.fib_index = rx_fib_index;
 821   key.proto = ip->protocol;
 822   key.r_port = 0;
 823   key.l_port = 0;
 824   s_kv.key[0] = key.as_u64[0];
 825   s_kv.key[1] = key.as_u64[1];
 826
 827   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 828     {
 829       s = pool_elt_at_index (tsm->sessions, s_value.value);
 830       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 831     }
 832   else
 833     {
 834       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 835         {
 836           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 837           return 0;
 838         }
 839
 840       m_key.addr = ip->dst_address;
 841       m_key.port = 0;
 842       m_key.protocol = 0;
 843       m_key.fib_index = rx_fib_index;
 844       kv.key = m_key.as_u64;
 845       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 846         {
 847           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 848           return 0;
 849         }
 850
 851       m = pool_elt_at_index (sm->static_mappings, value.value);
 852
 853       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 854
 855       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 856                                   thread_index);
 857       if (!u)
 858         {
 859           clib_warning ("create NAT user failed");
 860           return 0;
 861         }
 862
 863       /* Create a new session */
 864       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 865       if (!s)
 866         {
 867           clib_warning ("create NAT session failed");
 868           return 0;
 869         }
 870
 871       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 872       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 873       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 874       s->outside_address_index = ~0;
 875       s->out2in.addr.as_u32 = old_addr;
 876       s->out2in.fib_index = rx_fib_index;
 877       s->in2out.addr.as_u32 = new_addr;
 878       s->in2out.fib_index = m->fib_index;
 879       s->in2out.port = s->out2in.port = ip->protocol;
 880       user_session_increment (sm, u, 1 /* static */);
 881
 882       /* Add to lookup tables */
 883       s_kv.value = s - tsm->sessions;
 884       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 885         clib_warning ("out2in key add failed");
 886
 887       key.l_addr = ip->dst_address;
 888       key.fib_index = m->fib_index;
 889       s_kv.key[0] = key.as_u64[0];
 890       s_kv.key[1] = key.as_u64[1];
 891       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 892         clib_warning ("in2out key add failed");
 893    }
 894
 895   /* Update IP checksum */
 896   sum = ip->checksum;
 897   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 898   ip->checksum = ip_csum_fold (sum);
 899
 900   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 901
 902   /* Accounting */
 903   s->last_heard = now;
 904   s->total_pkts++;
 905   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 906   /* Per-user LRU list maintenance */
 907   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 908   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 909                       s->per_user_index);
 910
 911   return s;
 912 }
 913
 914 static snat_session_t *
 915 snat_out2in_lb (snat_main_t *sm,
 916                 vlib_buffer_t * b,
 917                 ip4_header_t * ip,
 918                 u32 rx_fib_index,
 919                 u32 thread_index,
 920                 f64 now,
 921                 vlib_main_t * vm,
 922                 vlib_node_runtime_t * node)
 923 {
 924   nat_ed_ses_key_t key;
 925   clib_bihash_kv_16_8_t s_kv, s_value;
 926   udp_header_t *udp = ip4_next_header (ip);
 927   tcp_header_t *tcp = (tcp_header_t *) udp;
 928   snat_session_t *s = 0;
 929   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 930   snat_session_key_t e_key, l_key;
 931   u32 old_addr, new_addr;
 932   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 933   u16 new_port, old_port;
 934   ip_csum_t sum;
 935   snat_user_t *u;
 936   u32 address_index;
 937   snat_session_key_t eh_key;
 938   u8 twice_nat, lb;
 939
 940   old_addr = ip->dst_address.as_u32;
 941
 942   key.l_addr = ip->dst_address;
 943   key.r_addr = ip->src_address;
 944   key.fib_index = rx_fib_index;
 945   key.proto = ip->protocol;
 946   key.r_port = udp->src_port;
 947   key.l_port = udp->dst_port;
 948   s_kv.key[0] = key.as_u64[0];
 949   s_kv.key[1] = key.as_u64[1];
 950
 951   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 952     {
 953       s = pool_elt_at_index (tsm->sessions, s_value.value);
 954     }
 955   else
 956     {
 957       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 958         {
 959           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 960           return 0;
 961         }
 962
 963       e_key.addr = ip->dst_address;
 964       e_key.port = udp->dst_port;
 965       e_key.protocol = proto;
 966       e_key.fib_index = rx_fib_index;
 967       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 968         return 0;
 969
 970       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 971                                   thread_index);
 972       if (!u)
 973       {
 974         clib_warning ("create NAT user failed");
 975         return 0;
 976       }
 977
 978       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 979       if (!s)
 980         {
 981           clib_warning ("create NAT session failed");
 982           return 0;
 983         }
 984
 985       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 986       s->ext_host_port = udp->src_port;
 987       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 988       if (lb)
 989         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 990       s->outside_address_index = ~0;
 991       s->out2in = e_key;
 992       s->in2out = l_key;
 993       user_session_increment (sm, u, 1 /* static */);
 994
 995       /* Add to lookup tables */
 996       s_kv.value = s - tsm->sessions;
 997       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 998         clib_warning ("out2in-ed key add failed");
 999
1000       if (twice_nat)
1001         {
1002           eh_key.protocol = proto;
1003           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1004                                                    thread_index, &eh_key,
1005                                                    &address_index,
1006                                                    sm->port_per_thread,
1007                                                    sm->per_thread_data[thread_index].snat_thread_index))
1008             {
1009               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1010               return 0;
1011             }
1012           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1013           key.r_port = s->ext_host_nat_port = eh_key.port;
1014           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1015         }
1016       key.l_addr = l_key.addr;
1017       key.fib_index = l_key.fib_index;
1018       key.l_port = l_key.port;
1019       s_kv.key[0] = key.as_u64[0];
1020       s_kv.key[1] = key.as_u64[1];
1021       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1022         clib_warning ("in2out-ed key add failed");
1023     }
1024
1025   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1026
1027   /* Update IP checksum */
1028   sum = ip->checksum;
1029   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1030   if (is_twice_nat_session (s))
1031     sum = ip_csum_update (sum, ip->src_address.as_u32,
1032                           s->ext_host_nat_addr.as_u32, ip4_header_t,
1033                           src_address);
1034   ip->checksum = ip_csum_fold (sum);
1035
1036   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1037     {
1038       old_port = tcp->dst_port;
1039       tcp->dst_port = s->in2out.port;
1040       new_port = tcp->dst_port;
1041
1042       sum = tcp->checksum;
1043       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1044       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1045       if (is_twice_nat_session (s))
1046         {
1047           sum = ip_csum_update (sum, ip->src_address.as_u32,
1048                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1049                                 dst_address);
1050           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1051                                 ip4_header_t, length);
1052           tcp->src_port = s->ext_host_nat_port;
1053           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1054         }
1055       tcp->checksum = ip_csum_fold(sum);
1056     }
1057   else
1058     {
1059       udp->dst_port = s->in2out.port;
1060       if (is_twice_nat_session (s))
1061         {
1062           udp->src_port = s->ext_host_nat_port;
1063           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1064         }
1065       udp->checksum = 0;
1066     }
1067
1068   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1069
1070   /* Accounting */
1071   s->last_heard = now;
1072   s->total_pkts++;
1073   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1074   /* Per-user LRU list maintenance */
1075   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1076   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1077                       s->per_user_index);
1078
1079   return s;
1080 }
1081
1082 static uword
1083 snat_out2in_node_fn (vlib_main_t * vm,
1084                   vlib_node_runtime_t * node,
1085                   vlib_frame_t * frame)
1086 {
1087   u32 n_left_from, * from, * to_next;
1088   snat_out2in_next_t next_index;
1089   u32 pkts_processed = 0;
1090   snat_main_t * sm = &snat_main;
1091   f64 now = vlib_time_now (vm);
1092   u32 thread_index = vlib_get_thread_index ();
1093
1094   from = vlib_frame_vector_args (frame);
1095   n_left_from = frame->n_vectors;
1096   next_index = node->cached_next_index;
1097
1098   while (n_left_from > 0)
1099     {
1100       u32 n_left_to_next;
1101
1102       vlib_get_next_frame (vm, node, next_index,
1103                            to_next, n_left_to_next);
1104
1105       while (n_left_from >= 4 && n_left_to_next >= 2)
1106         {
1107           u32 bi0, bi1;
1108           vlib_buffer_t * b0, * b1;
1109           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1110           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1111           u32 sw_if_index0, sw_if_index1;
1112           ip4_header_t * ip0, *ip1;
1113           ip_csum_t sum0, sum1;
1114           u32 new_addr0, old_addr0;
1115           u16 new_port0, old_port0;
1116           u32 new_addr1, old_addr1;
1117           u16 new_port1, old_port1;
1118           udp_header_t * udp0, * udp1;
1119           tcp_header_t * tcp0, * tcp1;
1120           icmp46_header_t * icmp0, * icmp1;
1121           snat_session_key_t key0, key1, sm0, sm1;
1122           u32 rx_fib_index0, rx_fib_index1;
1123           u32 proto0, proto1;
1124           snat_session_t * s0 = 0, * s1 = 0;
1125           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1126
1127           /* Prefetch next iteration. */
1128           {
1129             vlib_buffer_t * p2, * p3;
1130
1131             p2 = vlib_get_buffer (vm, from[2]);
1132             p3 = vlib_get_buffer (vm, from[3]);
1133
1134             vlib_prefetch_buffer_header (p2, LOAD);
1135             vlib_prefetch_buffer_header (p3, LOAD);
1136
1137             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1138             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1139           }
1140
1141           /* speculatively enqueue b0 and b1 to the current next frame */
1142           to_next[0] = bi0 = from[0];
1143           to_next[1] = bi1 = from[1];
1144           from += 2;
1145           to_next += 2;
1146           n_left_from -= 2;
1147           n_left_to_next -= 2;
1148
1149           b0 = vlib_get_buffer (vm, bi0);
1150           b1 = vlib_get_buffer (vm, bi1);
1151
1152           vnet_buffer (b0)->snat.flags = 0;
1153           vnet_buffer (b1)->snat.flags = 0;
1154
1155           ip0 = vlib_buffer_get_current (b0);
1156           udp0 = ip4_next_header (ip0);
1157           tcp0 = (tcp_header_t *) udp0;
1158           icmp0 = (icmp46_header_t *) udp0;
1159
1160           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1161           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1162                                    sw_if_index0);
1163
1164           if (PREDICT_FALSE(ip0->ttl == 1))
1165             {
1166               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1167               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1168                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1169                                            0);
1170               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1171               goto trace0;
1172             }
1173
1174           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1175
1176           if (PREDICT_FALSE (proto0 == ~0))
1177             {
1178               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1179                                              thread_index, now, vm, node);
1180               if (!sm->forwarding_enabled)
1181                 if (!s0)
1182                   next0 = SNAT_OUT2IN_NEXT_DROP;
1183               goto trace0;
1184             }
1185
1186           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1187             {
1188               next0 = icmp_out2in_slow_path
1189                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1190                  next0, now, thread_index, &s0);
1191               goto trace0;
1192             }
1193
1194           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1195             {
1196               next0 = SNAT_OUT2IN_NEXT_REASS;
1197               goto trace0;
1198             }
1199
1200           key0.addr = ip0->dst_address;
1201           key0.port = udp0->dst_port;
1202           key0.protocol = proto0;
1203           key0.fib_index = rx_fib_index0;
1204
1205           kv0.key = key0.as_u64;
1206
1207           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1208                                       &kv0, &value0))
1209             {
1210               /* Try to match static mapping by external address and port,
1211                  destination address and port in packet */
1212               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1213                 {
1214                   /*
1215                    * Send DHCP packets to the ipv4 stack, or we won't
1216                    * be able to use dhcp client on the outside interface
1217                    */
1218                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1219                       && (udp0->dst_port ==
1220                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1221                     {
1222                       vnet_feature_next
1223                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1224                       goto trace0;
1225                     }
1226
1227                   if (!sm->forwarding_enabled)
1228                     {
1229                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1230                       next0 = SNAT_OUT2IN_NEXT_DROP;
1231                       goto trace0;
1232                     }
1233                   else
1234                     {
1235                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1236                         {
1237                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1238                           goto trace0;
1239                         }
1240                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1241                       goto trace0;
1242                     }
1243                 }
1244
1245               /* Create session initiated by host from external network */
1246               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1247                                                      thread_index);
1248               if (!s0)
1249                 {
1250                   next0 = SNAT_OUT2IN_NEXT_DROP;
1251                   goto trace0;
1252                 }
1253             }
1254           else
1255             {
1256               if (PREDICT_FALSE (value0.value == ~0ULL))
1257                 {
1258                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1259                                       now, vm, node);
1260                   if (!s0)
1261                     next0 = SNAT_OUT2IN_NEXT_DROP;
1262                   goto trace0;
1263                 }
1264               else
1265                 {
1266                   s0 = pool_elt_at_index (
1267                     sm->per_thread_data[thread_index].sessions,
1268                     value0.value);
1269                 }
1270             }
1271
1272           old_addr0 = ip0->dst_address.as_u32;
1273           ip0->dst_address = s0->in2out.addr;
1274           new_addr0 = ip0->dst_address.as_u32;
1275           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1276
1277           sum0 = ip0->checksum;
1278           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1279                                  ip4_header_t,
1280                                  dst_address /* changed member */);
1281           ip0->checksum = ip_csum_fold (sum0);
1282
1283           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1284             {
1285               old_port0 = tcp0->dst_port;
1286               tcp0->dst_port = s0->in2out.port;
1287               new_port0 = tcp0->dst_port;
1288
1289               sum0 = tcp0->checksum;
1290               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1291                                      ip4_header_t,
1292                                      dst_address /* changed member */);
1293
1294               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1295                                      ip4_header_t /* cheat */,
1296                                      length /* changed member */);
1297               tcp0->checksum = ip_csum_fold(sum0);
1298             }
1299           else
1300             {
1301               old_port0 = udp0->dst_port;
1302               udp0->dst_port = s0->in2out.port;
1303               udp0->checksum = 0;
1304             }
1305
1306           /* Accounting */
1307           s0->last_heard = now;
1308           s0->total_pkts++;
1309           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1310           /* Per-user LRU list maintenance */
1311           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1312                              s0->per_user_index);
1313           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1314                               s0->per_user_list_head_index,
1315                               s0->per_user_index);
1316         trace0:
1317
1318           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1319                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1320             {
1321               snat_out2in_trace_t *t =
1322                  vlib_add_trace (vm, node, b0, sizeof (*t));
1323               t->sw_if_index = sw_if_index0;
1324               t->next_index = next0;
1325               t->session_index = ~0;
1326               if (s0)
1327                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1328             }
1329
1330           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1331
1332
1333           ip1 = vlib_buffer_get_current (b1);
1334           udp1 = ip4_next_header (ip1);
1335           tcp1 = (tcp_header_t *) udp1;
1336           icmp1 = (icmp46_header_t *) udp1;
1337
1338           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1339           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1340                                    sw_if_index1);
1341
1342           if (PREDICT_FALSE(ip1->ttl == 1))
1343             {
1344               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1345               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1346                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1347                                            0);
1348               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1349               goto trace1;
1350             }
1351
1352           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1353
1354           if (PREDICT_FALSE (proto1 == ~0))
1355             {
1356               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1357                                              thread_index, now, vm, node);
1358               if (!sm->forwarding_enabled)
1359                 if (!s1)
1360                   next1 = SNAT_OUT2IN_NEXT_DROP;
1361               goto trace1;
1362             }
1363
1364           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1365             {
1366               next1 = icmp_out2in_slow_path
1367                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1368                  next1, now, thread_index, &s1);
1369               goto trace1;
1370             }
1371
1372           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1373             {
1374               next1 = SNAT_OUT2IN_NEXT_REASS;
1375               goto trace1;
1376             }
1377
1378           key1.addr = ip1->dst_address;
1379           key1.port = udp1->dst_port;
1380           key1.protocol = proto1;
1381           key1.fib_index = rx_fib_index1;
1382
1383           kv1.key = key1.as_u64;
1384
1385           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1386                                       &kv1, &value1))
1387             {
1388               /* Try to match static mapping by external address and port,
1389                  destination address and port in packet */
1390               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1391                 {
1392                   /*
1393                    * Send DHCP packets to the ipv4 stack, or we won't
1394                    * be able to use dhcp client on the outside interface
1395                    */
1396                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1397                       && (udp1->dst_port ==
1398                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1399                     {
1400                       vnet_feature_next
1401                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1402                       goto trace1;
1403                     }
1404
1405                   if (!sm->forwarding_enabled)
1406                     {
1407                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1408                       next1 = SNAT_OUT2IN_NEXT_DROP;
1409                       goto trace1;
1410                     }
1411                   else
1412                     {
1413                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1414                         {
1415                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1416                           goto trace1;
1417                         }
1418                       create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1419                       goto trace1;
1420                     }
1421                 }
1422
1423               /* Create session initiated by host from external network */
1424               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1425                                                      thread_index);
1426               if (!s1)
1427                 {
1428                   next1 = SNAT_OUT2IN_NEXT_DROP;
1429                   goto trace1;
1430                 }
1431             }
1432           else
1433             {
1434               if (PREDICT_FALSE (value1.value == ~0ULL))
1435                 {
1436                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1437                                       now, vm, node);
1438                   if (!s1)
1439                     next1 = SNAT_OUT2IN_NEXT_DROP;
1440                   goto trace1;
1441                 }
1442               else
1443                 {
1444                   s1 = pool_elt_at_index (
1445                     sm->per_thread_data[thread_index].sessions,
1446                     value1.value);
1447                 }
1448             }
1449
1450           old_addr1 = ip1->dst_address.as_u32;
1451           ip1->dst_address = s1->in2out.addr;
1452           new_addr1 = ip1->dst_address.as_u32;
1453           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1454
1455           sum1 = ip1->checksum;
1456           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1457                                  ip4_header_t,
1458                                  dst_address /* changed member */);
1459           ip1->checksum = ip_csum_fold (sum1);
1460
1461           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1462             {
1463               old_port1 = tcp1->dst_port;
1464               tcp1->dst_port = s1->in2out.port;
1465               new_port1 = tcp1->dst_port;
1466
1467               sum1 = tcp1->checksum;
1468               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1469                                      ip4_header_t,
1470                                      dst_address /* changed member */);
1471
1472               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1473                                      ip4_header_t /* cheat */,
1474                                      length /* changed member */);
1475               tcp1->checksum = ip_csum_fold(sum1);
1476             }
1477           else
1478             {
1479               old_port1 = udp1->dst_port;
1480               udp1->dst_port = s1->in2out.port;
1481               udp1->checksum = 0;
1482             }
1483
1484           /* Accounting */
1485           s1->last_heard = now;
1486           s1->total_pkts++;
1487           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1488           /* Per-user LRU list maintenance */
1489           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1490                              s1->per_user_index);
1491           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1492                               s1->per_user_list_head_index,
1493                               s1->per_user_index);
1494         trace1:
1495
1496           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1497                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1498             {
1499               snat_out2in_trace_t *t =
1500                  vlib_add_trace (vm, node, b1, sizeof (*t));
1501               t->sw_if_index = sw_if_index1;
1502               t->next_index = next1;
1503               t->session_index = ~0;
1504               if (s1)
1505                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1506             }
1507
1508           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1509
1510           /* verify speculative enqueues, maybe switch current next frame */
1511           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1512                                            to_next, n_left_to_next,
1513                                            bi0, bi1, next0, next1);
1514         }
1515
1516       while (n_left_from > 0 && n_left_to_next > 0)
1517         {
1518           u32 bi0;
1519           vlib_buffer_t * b0;
1520           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1521           u32 sw_if_index0;
1522           ip4_header_t * ip0;
1523           ip_csum_t sum0;
1524           u32 new_addr0, old_addr0;
1525           u16 new_port0, old_port0;
1526           udp_header_t * udp0;
1527           tcp_header_t * tcp0;
1528           icmp46_header_t * icmp0;
1529           snat_session_key_t key0, sm0;
1530           u32 rx_fib_index0;
1531           u32 proto0;
1532           snat_session_t * s0 = 0;
1533           clib_bihash_kv_8_8_t kv0, value0;
1534
1535           /* speculatively enqueue b0 to the current next frame */
1536           bi0 = from[0];
1537           to_next[0] = bi0;
1538           from += 1;
1539           to_next += 1;
1540           n_left_from -= 1;
1541           n_left_to_next -= 1;
1542
1543           b0 = vlib_get_buffer (vm, bi0);
1544
1545           vnet_buffer (b0)->snat.flags = 0;
1546
1547           ip0 = vlib_buffer_get_current (b0);
1548           udp0 = ip4_next_header (ip0);
1549           tcp0 = (tcp_header_t *) udp0;
1550           icmp0 = (icmp46_header_t *) udp0;
1551
1552           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1553           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1554                                    sw_if_index0);
1555
1556           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1557
1558           if (PREDICT_FALSE (proto0 == ~0))
1559             {
1560               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1561                                              thread_index, now, vm, node);
1562               if (!sm->forwarding_enabled)
1563                 if (!s0)
1564                   next0 = SNAT_OUT2IN_NEXT_DROP;
1565               goto trace00;
1566             }
1567
1568           if (PREDICT_FALSE(ip0->ttl == 1))
1569             {
1570               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1571               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1572                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1573                                            0);
1574               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1575               goto trace00;
1576             }
1577
1578           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1579             {
1580               next0 = icmp_out2in_slow_path
1581                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1582                  next0, now, thread_index, &s0);
1583               goto trace00;
1584             }
1585
1586           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1587             {
1588               next0 = SNAT_OUT2IN_NEXT_REASS;
1589               goto trace00;
1590             }
1591
1592           key0.addr = ip0->dst_address;
1593           key0.port = udp0->dst_port;
1594           key0.protocol = proto0;
1595           key0.fib_index = rx_fib_index0;
1596
1597           kv0.key = key0.as_u64;
1598
1599           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1600                                       &kv0, &value0))
1601             {
1602               /* Try to match static mapping by external address and port,
1603                  destination address and port in packet */
1604               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1605                 {
1606                   /*
1607                    * Send DHCP packets to the ipv4 stack, or we won't
1608                    * be able to use dhcp client on the outside interface
1609                    */
1610                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1611                       && (udp0->dst_port ==
1612                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1613                     {
1614                       vnet_feature_next
1615                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1616                       goto trace00;
1617                     }
1618
1619                   if (!sm->forwarding_enabled)
1620                     {
1621                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1622                       next0 = SNAT_OUT2IN_NEXT_DROP;
1623                       goto trace00;
1624                     }
1625                   else
1626                     {
1627                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1628                         {
1629                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1630                           goto trace00;
1631                         }
1632                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1633                       goto trace00;
1634                     }
1635                 }
1636
1637               /* Create session initiated by host from external network */
1638               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1639                                                      thread_index);
1640               if (!s0)
1641                 {
1642                   next0 = SNAT_OUT2IN_NEXT_DROP;
1643                   goto trace00;
1644                 }
1645             }
1646           else
1647             {
1648               if (PREDICT_FALSE (value0.value == ~0ULL))
1649                 {
1650                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1651                                       now, vm, node);
1652                   if (!s0)
1653                     next0 = SNAT_OUT2IN_NEXT_DROP;
1654                   goto trace00;
1655                 }
1656               else
1657                 {
1658                   s0 = pool_elt_at_index (
1659                     sm->per_thread_data[thread_index].sessions,
1660                     value0.value);
1661                 }
1662             }
1663
1664           old_addr0 = ip0->dst_address.as_u32;
1665           ip0->dst_address = s0->in2out.addr;
1666           new_addr0 = ip0->dst_address.as_u32;
1667           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1668
1669           sum0 = ip0->checksum;
1670           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1671                                  ip4_header_t,
1672                                  dst_address /* changed member */);
1673           ip0->checksum = ip_csum_fold (sum0);
1674
1675           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1676             {
1677               old_port0 = tcp0->dst_port;
1678               tcp0->dst_port = s0->in2out.port;
1679               new_port0 = tcp0->dst_port;
1680
1681               sum0 = tcp0->checksum;
1682               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1683                                      ip4_header_t,
1684                                      dst_address /* changed member */);
1685
1686               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1687                                      ip4_header_t /* cheat */,
1688                                      length /* changed member */);
1689               tcp0->checksum = ip_csum_fold(sum0);
1690             }
1691           else
1692             {
1693               old_port0 = udp0->dst_port;
1694               udp0->dst_port = s0->in2out.port;
1695               udp0->checksum = 0;
1696             }
1697
1698           /* Accounting */
1699           s0->last_heard = now;
1700           s0->total_pkts++;
1701           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1702           /* Per-user LRU list maintenance */
1703           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1704                              s0->per_user_index);
1705           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1706                               s0->per_user_list_head_index,
1707                               s0->per_user_index);
1708         trace00:
1709
1710           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1711                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1712             {
1713               snat_out2in_trace_t *t =
1714                  vlib_add_trace (vm, node, b0, sizeof (*t));
1715               t->sw_if_index = sw_if_index0;
1716               t->next_index = next0;
1717               t->session_index = ~0;
1718               if (s0)
1719                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1720             }
1721
1722           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1723
1724           /* verify speculative enqueue, maybe switch current next frame */
1725           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1726                                            to_next, n_left_to_next,
1727                                            bi0, next0);
1728         }
1729
1730       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1731     }
1732
1733   vlib_node_increment_counter (vm, snat_out2in_node.index,
1734                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1735                                pkts_processed);
1736   return frame->n_vectors;
1737 }
1738
1739 VLIB_REGISTER_NODE (snat_out2in_node) = {
1740   .function = snat_out2in_node_fn,
1741   .name = "nat44-out2in",
1742   .vector_size = sizeof (u32),
1743   .format_trace = format_snat_out2in_trace,
1744   .type = VLIB_NODE_TYPE_INTERNAL,
1745
1746   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1747   .error_strings = snat_out2in_error_strings,
1748
1749   .runtime_data_bytes = sizeof (snat_runtime_t),
1750
1751   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1752
1753   /* edit / add dispositions here */
1754   .next_nodes = {
1755     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1756     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1757     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1758     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1759     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1760   },
1761 };
1762 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1763
1764 static uword
1765 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1766                             vlib_node_runtime_t * node,
1767                             vlib_frame_t * frame)
1768 {
1769   u32 n_left_from, *from, *to_next;
1770   snat_out2in_next_t next_index;
1771   u32 pkts_processed = 0;
1772   snat_main_t *sm = &snat_main;
1773   f64 now = vlib_time_now (vm);
1774   u32 thread_index = vlib_get_thread_index ();
1775   snat_main_per_thread_data_t *per_thread_data =
1776     &sm->per_thread_data[thread_index];
1777   u32 *fragments_to_drop = 0;
1778   u32 *fragments_to_loopback = 0;
1779
1780   from = vlib_frame_vector_args (frame);
1781   n_left_from = frame->n_vectors;
1782   next_index = node->cached_next_index;
1783
1784   while (n_left_from > 0)
1785     {
1786       u32 n_left_to_next;
1787
1788       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1789
1790       while (n_left_from > 0 && n_left_to_next > 0)
1791        {
1792           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1793           vlib_buffer_t *b0;
1794           u32 next0;
1795           u8 cached0 = 0;
1796           ip4_header_t *ip0;
1797           nat_reass_ip4_t *reass0;
1798           udp_header_t * udp0;
1799           tcp_header_t * tcp0;
1800           snat_session_key_t key0, sm0;
1801           clib_bihash_kv_8_8_t kv0, value0;
1802           snat_session_t * s0 = 0;
1803           u16 old_port0, new_port0;
1804           ip_csum_t sum0;
1805
1806           /* speculatively enqueue b0 to the current next frame */
1807           bi0 = from[0];
1808           to_next[0] = bi0;
1809           from += 1;
1810           to_next += 1;
1811           n_left_from -= 1;
1812           n_left_to_next -= 1;
1813
1814           b0 = vlib_get_buffer (vm, bi0);
1815           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1816
1817           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1818           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1819                                                                sw_if_index0);
1820
1821           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1822             {
1823               next0 = SNAT_OUT2IN_NEXT_DROP;
1824               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1825               goto trace0;
1826             }
1827
1828           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1829           udp0 = ip4_next_header (ip0);
1830           tcp0 = (tcp_header_t *) udp0;
1831           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1832
1833           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1834                                                  ip0->dst_address,
1835                                                  ip0->fragment_id,
1836                                                  ip0->protocol,
1837                                                  1,
1838                                                  &fragments_to_drop);
1839
1840           if (PREDICT_FALSE (!reass0))
1841             {
1842               next0 = SNAT_OUT2IN_NEXT_DROP;
1843               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1844               goto trace0;
1845             }
1846
1847           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1848             {
1849               key0.addr = ip0->dst_address;
1850               key0.port = udp0->dst_port;
1851               key0.protocol = proto0;
1852               key0.fib_index = rx_fib_index0;
1853               kv0.key = key0.as_u64;
1854
1855               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1856                 {
1857                   /* Try to match static mapping by external address and port,
1858                      destination address and port in packet */
1859                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1860                     {
1861                       /*
1862                        * Send DHCP packets to the ipv4 stack, or we won't
1863                        * be able to use dhcp client on the outside interface
1864                        */
1865                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1866                           && (udp0->dst_port
1867                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1868                         {
1869                           vnet_feature_next
1870                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1871                              &next0, b0);
1872                           goto trace0;
1873                         }
1874
1875                       if (!sm->forwarding_enabled)
1876                         {
1877                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1878                           next0 = SNAT_OUT2IN_NEXT_DROP;
1879                           goto trace0;
1880                         }
1881                       else
1882                         {
1883                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1884                             {
1885                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1886                               goto trace0;
1887                             }
1888                           create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1889                           goto trace0;
1890                         }
1891                     }
1892
1893                   /* Create session initiated by host from external network */
1894                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1895                                                          thread_index);
1896                   if (!s0)
1897                     {
1898                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1899                       next0 = SNAT_OUT2IN_NEXT_DROP;
1900                       goto trace0;
1901                     }
1902                   reass0->sess_index = s0 - per_thread_data->sessions;
1903                   reass0->thread_index = thread_index;
1904                 }
1905               else
1906                 {
1907                   s0 = pool_elt_at_index (per_thread_data->sessions,
1908                                           value0.value);
1909                   reass0->sess_index = value0.value;
1910                 }
1911               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1912             }
1913           else
1914             {
1915               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1916                 {
1917                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1918                     {
1919                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1920                       next0 = SNAT_OUT2IN_NEXT_DROP;
1921                       goto trace0;
1922                     }
1923                   cached0 = 1;
1924                   goto trace0;
1925                 }
1926               s0 = pool_elt_at_index (per_thread_data->sessions,
1927                                       reass0->sess_index);
1928             }
1929
1930           old_addr0 = ip0->dst_address.as_u32;
1931           ip0->dst_address = s0->in2out.addr;
1932           new_addr0 = ip0->dst_address.as_u32;
1933           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1934
1935           sum0 = ip0->checksum;
1936           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1937                                  ip4_header_t,
1938                                  dst_address /* changed member */);
1939           ip0->checksum = ip_csum_fold (sum0);
1940
1941           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1942             {
1943               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1944                 {
1945                   old_port0 = tcp0->dst_port;
1946                   tcp0->dst_port = s0->in2out.port;
1947                   new_port0 = tcp0->dst_port;
1948
1949                   sum0 = tcp0->checksum;
1950                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1951                                          ip4_header_t,
1952                                          dst_address /* changed member */);
1953
1954                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1955                                          ip4_header_t /* cheat */,
1956                                          length /* changed member */);
1957                   tcp0->checksum = ip_csum_fold(sum0);
1958                 }
1959               else
1960                 {
1961                   old_port0 = udp0->dst_port;
1962                   udp0->dst_port = s0->in2out.port;
1963                   udp0->checksum = 0;
1964                 }
1965             }
1966
1967           /* Accounting */
1968           s0->last_heard = now;
1969           s0->total_pkts++;
1970           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1971           /* Per-user LRU list maintenance */
1972           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1973                              s0->per_user_index);
1974           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1975                               s0->per_user_list_head_index,
1976                               s0->per_user_index);
1977
1978         trace0:
1979           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1980                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1981             {
1982               nat44_out2in_reass_trace_t *t =
1983                  vlib_add_trace (vm, node, b0, sizeof (*t));
1984               t->cached = cached0;
1985               t->sw_if_index = sw_if_index0;
1986               t->next_index = next0;
1987             }
1988
1989           if (cached0)
1990             {
1991               n_left_to_next++;
1992               to_next--;
1993             }
1994           else
1995             {
1996               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1997
1998               /* verify speculative enqueue, maybe switch current next frame */
1999               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2000                                                to_next, n_left_to_next,
2001                                                bi0, next0);
2002             }
2003
2004           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2005             {
2006               from = vlib_frame_vector_args (frame);
2007               u32 len = vec_len (fragments_to_loopback);
2008               if (len <= VLIB_FRAME_SIZE)
2009                 {
2010                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2011                   n_left_from = len;
2012                   vec_reset_length (fragments_to_loopback);
2013                 }
2014               else
2015                 {
2016                   clib_memcpy (from,
2017                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2018                                sizeof (u32) * VLIB_FRAME_SIZE);
2019                   n_left_from = VLIB_FRAME_SIZE;
2020                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2021                 }
2022             }
2023        }
2024
2025       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2026     }
2027
2028   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2029                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2030                                pkts_processed);
2031
2032   nat_send_all_to_node (vm, fragments_to_drop, node,
2033                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2034                         SNAT_OUT2IN_NEXT_DROP);
2035
2036   vec_free (fragments_to_drop);
2037   vec_free (fragments_to_loopback);
2038   return frame->n_vectors;
2039 }
2040
2041 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2042   .function = nat44_out2in_reass_node_fn,
2043   .name = "nat44-out2in-reass",
2044   .vector_size = sizeof (u32),
2045   .format_trace = format_nat44_out2in_reass_trace,
2046   .type = VLIB_NODE_TYPE_INTERNAL,
2047
2048   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2049   .error_strings = snat_out2in_error_strings,
2050
2051   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2052
2053   /* edit / add dispositions here */
2054   .next_nodes = {
2055     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2056     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2057     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2058     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2059     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2060   },
2061 };
2062 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2063                               nat44_out2in_reass_node_fn);
2064
2065 /**************************/
2066 /*** deterministic mode ***/
2067 /**************************/
2068 static uword
2069 snat_det_out2in_node_fn (vlib_main_t * vm,
2070                          vlib_node_runtime_t * node,
2071                          vlib_frame_t * frame)
2072 {
2073   u32 n_left_from, * from, * to_next;
2074   snat_out2in_next_t next_index;
2075   u32 pkts_processed = 0;
2076   snat_main_t * sm = &snat_main;
2077   u32 thread_index = vlib_get_thread_index ();
2078
2079   from = vlib_frame_vector_args (frame);
2080   n_left_from = frame->n_vectors;
2081   next_index = node->cached_next_index;
2082
2083   while (n_left_from > 0)
2084     {
2085       u32 n_left_to_next;
2086
2087       vlib_get_next_frame (vm, node, next_index,
2088                            to_next, n_left_to_next);
2089
2090       while (n_left_from >= 4 && n_left_to_next >= 2)
2091         {
2092           u32 bi0, bi1;
2093           vlib_buffer_t * b0, * b1;
2094           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2095           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2096           u32 sw_if_index0, sw_if_index1;
2097           ip4_header_t * ip0, * ip1;
2098           ip_csum_t sum0, sum1;
2099           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2100           u16 new_port0, old_port0, old_port1, new_port1;
2101           udp_header_t * udp0, * udp1;
2102           tcp_header_t * tcp0, * tcp1;
2103           u32 proto0, proto1;
2104           snat_det_out_key_t key0, key1;
2105           snat_det_map_t * dm0, * dm1;
2106           snat_det_session_t * ses0 = 0, * ses1 = 0;
2107           u32 rx_fib_index0, rx_fib_index1;
2108           icmp46_header_t * icmp0, * icmp1;
2109
2110           /* Prefetch next iteration. */
2111           {
2112             vlib_buffer_t * p2, * p3;
2113
2114             p2 = vlib_get_buffer (vm, from[2]);
2115             p3 = vlib_get_buffer (vm, from[3]);
2116
2117             vlib_prefetch_buffer_header (p2, LOAD);
2118             vlib_prefetch_buffer_header (p3, LOAD);
2119
2120             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2121             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2122           }
2123
2124           /* speculatively enqueue b0 and b1 to the current next frame */
2125           to_next[0] = bi0 = from[0];
2126           to_next[1] = bi1 = from[1];
2127           from += 2;
2128           to_next += 2;
2129           n_left_from -= 2;
2130           n_left_to_next -= 2;
2131
2132           b0 = vlib_get_buffer (vm, bi0);
2133           b1 = vlib_get_buffer (vm, bi1);
2134
2135           ip0 = vlib_buffer_get_current (b0);
2136           udp0 = ip4_next_header (ip0);
2137           tcp0 = (tcp_header_t *) udp0;
2138
2139           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2140
2141           if (PREDICT_FALSE(ip0->ttl == 1))
2142             {
2143               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2144               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2145                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2146                                            0);
2147               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2148               goto trace0;
2149             }
2150
2151           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2152
2153           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2154             {
2155               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2156               icmp0 = (icmp46_header_t *) udp0;
2157
2158               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2159                                   rx_fib_index0, node, next0, thread_index,
2160                                   &ses0, &dm0);
2161               goto trace0;
2162             }
2163
2164           key0.ext_host_addr = ip0->src_address;
2165           key0.ext_host_port = tcp0->src;
2166           key0.out_port = tcp0->dst;
2167
2168           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2169           if (PREDICT_FALSE(!dm0))
2170             {
2171               clib_warning("unknown dst address:  %U",
2172                            format_ip4_address, &ip0->dst_address);
2173               next0 = SNAT_OUT2IN_NEXT_DROP;
2174               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2175               goto trace0;
2176             }
2177
2178           snat_det_reverse(dm0, &ip0->dst_address,
2179                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2180
2181           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2182           if (PREDICT_FALSE(!ses0))
2183             {
2184               clib_warning("no match src %U:%d dst %U:%d for user %U",
2185                            format_ip4_address, &ip0->src_address,
2186                            clib_net_to_host_u16 (tcp0->src),
2187                            format_ip4_address, &ip0->dst_address,
2188                            clib_net_to_host_u16 (tcp0->dst),
2189                            format_ip4_address, &new_addr0);
2190               next0 = SNAT_OUT2IN_NEXT_DROP;
2191               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2192               goto trace0;
2193             }
2194           new_port0 = ses0->in_port;
2195
2196           old_addr0 = ip0->dst_address;
2197           ip0->dst_address = new_addr0;
2198           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2199
2200           sum0 = ip0->checksum;
2201           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2202                                  ip4_header_t,
2203                                  dst_address /* changed member */);
2204           ip0->checksum = ip_csum_fold (sum0);
2205
2206           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2207             {
2208               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2209                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2210               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2211                 snat_det_ses_close(dm0, ses0);
2212
2213               old_port0 = tcp0->dst;
2214               tcp0->dst = new_port0;
2215
2216               sum0 = tcp0->checksum;
2217               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2218                                      ip4_header_t,
2219                                      dst_address /* changed member */);
2220
2221               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2222                                      ip4_header_t /* cheat */,
2223                                      length /* changed member */);
2224               tcp0->checksum = ip_csum_fold(sum0);
2225             }
2226           else
2227             {
2228               old_port0 = udp0->dst_port;
2229               udp0->dst_port = new_port0;
2230               udp0->checksum = 0;
2231             }
2232
2233         trace0:
2234
2235           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2236                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2237             {
2238               snat_out2in_trace_t *t =
2239                  vlib_add_trace (vm, node, b0, sizeof (*t));
2240               t->sw_if_index = sw_if_index0;
2241               t->next_index = next0;
2242               t->session_index = ~0;
2243               if (ses0)
2244                 t->session_index = ses0 - dm0->sessions;
2245             }
2246
2247           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2248
2249           b1 = vlib_get_buffer (vm, bi1);
2250
2251           ip1 = vlib_buffer_get_current (b1);
2252           udp1 = ip4_next_header (ip1);
2253           tcp1 = (tcp_header_t *) udp1;
2254
2255           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2256
2257           if (PREDICT_FALSE(ip1->ttl == 1))
2258             {
2259               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2260               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2261                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2262                                            0);
2263               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2264               goto trace1;
2265             }
2266
2267           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2268
2269           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2270             {
2271               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2272               icmp1 = (icmp46_header_t *) udp1;
2273
2274               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2275                                   rx_fib_index1, node, next1, thread_index,
2276                                   &ses1, &dm1);
2277               goto trace1;
2278             }
2279
2280           key1.ext_host_addr = ip1->src_address;
2281           key1.ext_host_port = tcp1->src;
2282           key1.out_port = tcp1->dst;
2283
2284           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2285           if (PREDICT_FALSE(!dm1))
2286             {
2287               clib_warning("unknown dst address:  %U",
2288                            format_ip4_address, &ip1->dst_address);
2289               next1 = SNAT_OUT2IN_NEXT_DROP;
2290               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2291               goto trace1;
2292             }
2293
2294           snat_det_reverse(dm1, &ip1->dst_address,
2295                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2296
2297           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2298           if (PREDICT_FALSE(!ses1))
2299             {
2300               clib_warning("no match src %U:%d dst %U:%d for user %U",
2301                            format_ip4_address, &ip1->src_address,
2302                            clib_net_to_host_u16 (tcp1->src),
2303                            format_ip4_address, &ip1->dst_address,
2304                            clib_net_to_host_u16 (tcp1->dst),
2305                            format_ip4_address, &new_addr1);
2306               next1 = SNAT_OUT2IN_NEXT_DROP;
2307               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2308               goto trace1;
2309             }
2310           new_port1 = ses1->in_port;
2311
2312           old_addr1 = ip1->dst_address;
2313           ip1->dst_address = new_addr1;
2314           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2315
2316           sum1 = ip1->checksum;
2317           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2318                                  ip4_header_t,
2319                                  dst_address /* changed member */);
2320           ip1->checksum = ip_csum_fold (sum1);
2321
2322           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2323             {
2324               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2325                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2326               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2327                 snat_det_ses_close(dm1, ses1);
2328
2329               old_port1 = tcp1->dst;
2330               tcp1->dst = new_port1;
2331
2332               sum1 = tcp1->checksum;
2333               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2334                                      ip4_header_t,
2335                                      dst_address /* changed member */);
2336
2337               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2338                                      ip4_header_t /* cheat */,
2339                                      length /* changed member */);
2340               tcp1->checksum = ip_csum_fold(sum1);
2341             }
2342           else
2343             {
2344               old_port1 = udp1->dst_port;
2345               udp1->dst_port = new_port1;
2346               udp1->checksum = 0;
2347             }
2348
2349         trace1:
2350
2351           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2352                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2353             {
2354               snat_out2in_trace_t *t =
2355                  vlib_add_trace (vm, node, b1, sizeof (*t));
2356               t->sw_if_index = sw_if_index1;
2357               t->next_index = next1;
2358               t->session_index = ~0;
2359               if (ses1)
2360                 t->session_index = ses1 - dm1->sessions;
2361             }
2362
2363           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2364
2365           /* verify speculative enqueues, maybe switch current next frame */
2366           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2367                                            to_next, n_left_to_next,
2368                                            bi0, bi1, next0, next1);
2369          }
2370
2371       while (n_left_from > 0 && n_left_to_next > 0)
2372         {
2373           u32 bi0;
2374           vlib_buffer_t * b0;
2375           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2376           u32 sw_if_index0;
2377           ip4_header_t * ip0;
2378           ip_csum_t sum0;
2379           ip4_address_t new_addr0, old_addr0;
2380           u16 new_port0, old_port0;
2381           udp_header_t * udp0;
2382           tcp_header_t * tcp0;
2383           u32 proto0;
2384           snat_det_out_key_t key0;
2385           snat_det_map_t * dm0;
2386           snat_det_session_t * ses0 = 0;
2387           u32 rx_fib_index0;
2388           icmp46_header_t * icmp0;
2389
2390           /* speculatively enqueue b0 to the current next frame */
2391           bi0 = from[0];
2392           to_next[0] = bi0;
2393           from += 1;
2394           to_next += 1;
2395           n_left_from -= 1;
2396           n_left_to_next -= 1;
2397
2398           b0 = vlib_get_buffer (vm, bi0);
2399
2400           ip0 = vlib_buffer_get_current (b0);
2401           udp0 = ip4_next_header (ip0);
2402           tcp0 = (tcp_header_t *) udp0;
2403
2404           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2405
2406           if (PREDICT_FALSE(ip0->ttl == 1))
2407             {
2408               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2409               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2410                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2411                                            0);
2412               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2413               goto trace00;
2414             }
2415
2416           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2417
2418           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2419             {
2420               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2421               icmp0 = (icmp46_header_t *) udp0;
2422
2423               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2424                                   rx_fib_index0, node, next0, thread_index,
2425                                   &ses0, &dm0);
2426               goto trace00;
2427             }
2428
2429           key0.ext_host_addr = ip0->src_address;
2430           key0.ext_host_port = tcp0->src;
2431           key0.out_port = tcp0->dst;
2432
2433           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2434           if (PREDICT_FALSE(!dm0))
2435             {
2436               clib_warning("unknown dst address:  %U",
2437                            format_ip4_address, &ip0->dst_address);
2438               next0 = SNAT_OUT2IN_NEXT_DROP;
2439               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2440               goto trace00;
2441             }
2442
2443           snat_det_reverse(dm0, &ip0->dst_address,
2444                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2445
2446           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2447           if (PREDICT_FALSE(!ses0))
2448             {
2449               clib_warning("no match src %U:%d dst %U:%d for user %U",
2450                            format_ip4_address, &ip0->src_address,
2451                            clib_net_to_host_u16 (tcp0->src),
2452                            format_ip4_address, &ip0->dst_address,
2453                            clib_net_to_host_u16 (tcp0->dst),
2454                            format_ip4_address, &new_addr0);
2455               next0 = SNAT_OUT2IN_NEXT_DROP;
2456               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2457               goto trace00;
2458             }
2459           new_port0 = ses0->in_port;
2460
2461           old_addr0 = ip0->dst_address;
2462           ip0->dst_address = new_addr0;
2463           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2464
2465           sum0 = ip0->checksum;
2466           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2467                                  ip4_header_t,
2468                                  dst_address /* changed member */);
2469           ip0->checksum = ip_csum_fold (sum0);
2470
2471           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2472             {
2473               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2474                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2475               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2476                 snat_det_ses_close(dm0, ses0);
2477
2478               old_port0 = tcp0->dst;
2479               tcp0->dst = new_port0;
2480
2481               sum0 = tcp0->checksum;
2482               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2483                                      ip4_header_t,
2484                                      dst_address /* changed member */);
2485
2486               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2487                                      ip4_header_t /* cheat */,
2488                                      length /* changed member */);
2489               tcp0->checksum = ip_csum_fold(sum0);
2490             }
2491           else
2492             {
2493               old_port0 = udp0->dst_port;
2494               udp0->dst_port = new_port0;
2495               udp0->checksum = 0;
2496             }
2497
2498         trace00:
2499
2500           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2501                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2502             {
2503               snat_out2in_trace_t *t =
2504                  vlib_add_trace (vm, node, b0, sizeof (*t));
2505               t->sw_if_index = sw_if_index0;
2506               t->next_index = next0;
2507               t->session_index = ~0;
2508               if (ses0)
2509                 t->session_index = ses0 - dm0->sessions;
2510             }
2511
2512           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2513
2514           /* verify speculative enqueue, maybe switch current next frame */
2515           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2516                                            to_next, n_left_to_next,
2517                                            bi0, next0);
2518         }
2519
2520       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2521     }
2522
2523   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2524                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2525                                pkts_processed);
2526   return frame->n_vectors;
2527 }
2528
2529 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2530   .function = snat_det_out2in_node_fn,
2531   .name = "nat44-det-out2in",
2532   .vector_size = sizeof (u32),
2533   .format_trace = format_snat_out2in_trace,
2534   .type = VLIB_NODE_TYPE_INTERNAL,
2535
2536   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2537   .error_strings = snat_out2in_error_strings,
2538
2539   .runtime_data_bytes = sizeof (snat_runtime_t),
2540
2541   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2542
2543   /* edit / add dispositions here */
2544   .next_nodes = {
2545     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2546     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2547     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2548     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2549     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2550   },
2551 };
2552 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2553
2554 /**
2555  * Get address and port values to be used for ICMP packet translation
2556  * and create session if needed
2557  *
2558  * @param[in,out] sm             NAT main
2559  * @param[in,out] node           NAT node runtime
2560  * @param[in] thread_index       thread index
2561  * @param[in,out] b0             buffer containing packet to be translated
2562  * @param[out] p_proto           protocol used for matching
2563  * @param[out] p_value           address and port after NAT translation
2564  * @param[out] p_dont_translate  if packet should not be translated
2565  * @param d                      optional parameter
2566  * @param e                      optional parameter
2567  */
2568 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2569                           u32 thread_index, vlib_buffer_t *b0,
2570                           ip4_header_t *ip0, u8 *p_proto,
2571                           snat_session_key_t *p_value,
2572                           u8 *p_dont_translate, void *d, void *e)
2573 {
2574   icmp46_header_t *icmp0;
2575   u32 sw_if_index0;
2576   u8 protocol;
2577   snat_det_out_key_t key0;
2578   u8 dont_translate = 0;
2579   u32 next0 = ~0;
2580   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2581   ip4_header_t *inner_ip0;
2582   void *l4_header = 0;
2583   icmp46_header_t *inner_icmp0;
2584   snat_det_map_t * dm0 = 0;
2585   ip4_address_t new_addr0 = {{0}};
2586   snat_det_session_t * ses0 = 0;
2587   ip4_address_t out_addr;
2588
2589   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2590   echo0 = (icmp_echo_header_t *)(icmp0+1);
2591   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2592
2593   if (!icmp_is_error_message (icmp0))
2594     {
2595       protocol = SNAT_PROTOCOL_ICMP;
2596       key0.ext_host_addr = ip0->src_address;
2597       key0.ext_host_port = 0;
2598       key0.out_port = echo0->identifier;
2599       out_addr = ip0->dst_address;
2600     }
2601   else
2602     {
2603       inner_ip0 = (ip4_header_t *)(echo0+1);
2604       l4_header = ip4_next_header (inner_ip0);
2605       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2606       key0.ext_host_addr = inner_ip0->dst_address;
2607       out_addr = inner_ip0->src_address;
2608       switch (protocol)
2609         {
2610         case SNAT_PROTOCOL_ICMP:
2611           inner_icmp0 = (icmp46_header_t*)l4_header;
2612           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2613           key0.ext_host_port = 0;
2614           key0.out_port = inner_echo0->identifier;
2615           break;
2616         case SNAT_PROTOCOL_UDP:
2617         case SNAT_PROTOCOL_TCP:
2618           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2619           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2620           break;
2621         default:
2622           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2623           next0 = SNAT_OUT2IN_NEXT_DROP;
2624           goto out;
2625         }
2626     }
2627
2628   dm0 = snat_det_map_by_out(sm, &out_addr);
2629   if (PREDICT_FALSE(!dm0))
2630     {
2631       /* Don't NAT packet aimed at the intfc address */
2632       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2633                                           ip0->dst_address.as_u32)))
2634         {
2635           dont_translate = 1;
2636           goto out;
2637         }
2638       clib_warning("unknown dst address:  %U",
2639                    format_ip4_address, &ip0->dst_address);
2640       goto out;
2641     }
2642
2643   snat_det_reverse(dm0, &ip0->dst_address,
2644                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2645
2646   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2647   if (PREDICT_FALSE(!ses0))
2648     {
2649       /* Don't NAT packet aimed at the intfc address */
2650       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2651                                           ip0->dst_address.as_u32)))
2652         {
2653           dont_translate = 1;
2654           goto out;
2655         }
2656       clib_warning("no match src %U:%d dst %U:%d for user %U",
2657                    format_ip4_address, &key0.ext_host_addr,
2658                    clib_net_to_host_u16 (key0.ext_host_port),
2659                    format_ip4_address, &out_addr,
2660                    clib_net_to_host_u16 (key0.out_port),
2661                    format_ip4_address, &new_addr0);
2662       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2663       next0 = SNAT_OUT2IN_NEXT_DROP;
2664       goto out;
2665     }
2666
2667   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2668                     !icmp_is_error_message (icmp0)))
2669     {
2670       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2671       next0 = SNAT_OUT2IN_NEXT_DROP;
2672       goto out;
2673     }
2674
2675   goto out;
2676
2677 out:
2678   *p_proto = protocol;
2679   if (ses0)
2680     {
2681       p_value->addr = new_addr0;
2682       p_value->fib_index = sm->inside_fib_index;
2683       p_value->port = ses0->in_port;
2684     }
2685   *p_dont_translate = dont_translate;
2686   if (d)
2687     *(snat_det_session_t**)d = ses0;
2688   if (e)
2689     *(snat_det_map_t**)e = dm0;
2690   return next0;
2691 }
2692
2693 /**********************/
2694 /*** worker handoff ***/
2695 /**********************/
2696 static uword
2697 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2698                                vlib_node_runtime_t * node,
2699                                vlib_frame_t * frame)
2700 {
2701   snat_main_t *sm = &snat_main;
2702   vlib_thread_main_t *tm = vlib_get_thread_main ();
2703   u32 n_left_from, *from, *to_next = 0;
2704   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2705   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2706     = 0;
2707   vlib_frame_queue_elt_t *hf = 0;
2708   vlib_frame_t *f = 0;
2709   int i;
2710   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2711   u32 next_worker_index = 0;
2712   u32 current_worker_index = ~0;
2713   u32 thread_index = vlib_get_thread_index ();
2714
2715   ASSERT (vec_len (sm->workers));
2716
2717   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2718     {
2719       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2720
2721       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2722                                sm->first_worker_index + sm->num_workers - 1,
2723                                (vlib_frame_queue_t *) (~0));
2724     }
2725
2726   from = vlib_frame_vector_args (frame);
2727   n_left_from = frame->n_vectors;
2728
2729   while (n_left_from > 0)
2730     {
2731       u32 bi0;
2732       vlib_buffer_t *b0;
2733       u32 sw_if_index0;
2734       u32 rx_fib_index0;
2735       ip4_header_t * ip0;
2736       u8 do_handoff;
2737
2738       bi0 = from[0];
2739       from += 1;
2740       n_left_from -= 1;
2741
2742       b0 = vlib_get_buffer (vm, bi0);
2743
2744       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2745       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2746
2747       ip0 = vlib_buffer_get_current (b0);
2748
2749       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2750
2751       if (PREDICT_FALSE (next_worker_index != thread_index))
2752         {
2753           do_handoff = 1;
2754
2755           if (next_worker_index != current_worker_index)
2756             {
2757               if (hf)
2758                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2759
2760               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2761                                                       next_worker_index,
2762                                                       handoff_queue_elt_by_worker_index);
2763
2764               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2765               to_next_worker = &hf->buffer_index[hf->n_vectors];
2766               current_worker_index = next_worker_index;
2767             }
2768
2769           /* enqueue to correct worker thread */
2770           to_next_worker[0] = bi0;
2771           to_next_worker++;
2772           n_left_to_next_worker--;
2773
2774           if (n_left_to_next_worker == 0)
2775             {
2776               hf->n_vectors = VLIB_FRAME_SIZE;
2777               vlib_put_frame_queue_elt (hf);
2778               current_worker_index = ~0;
2779               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2780               hf = 0;
2781             }
2782         }
2783       else
2784         {
2785           do_handoff = 0;
2786           /* if this is 1st frame */
2787           if (!f)
2788             {
2789               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2790               to_next = vlib_frame_vector_args (f);
2791             }
2792
2793           to_next[0] = bi0;
2794           to_next += 1;
2795           f->n_vectors++;
2796         }
2797
2798       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2799                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2800         {
2801           snat_out2in_worker_handoff_trace_t *t =
2802             vlib_add_trace (vm, node, b0, sizeof (*t));
2803           t->next_worker_index = next_worker_index;
2804           t->do_handoff = do_handoff;
2805         }
2806     }
2807
2808   if (f)
2809     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2810
2811   if (hf)
2812     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2813
2814   /* Ship frames to the worker nodes */
2815   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2816     {
2817       if (handoff_queue_elt_by_worker_index[i])
2818         {
2819           hf = handoff_queue_elt_by_worker_index[i];
2820           /*
2821            * It works better to let the handoff node
2822            * rate-adapt, always ship the handoff queue element.
2823            */
2824           if (1 || hf->n_vectors == hf->last_n_vectors)
2825             {
2826               vlib_put_frame_queue_elt (hf);
2827               handoff_queue_elt_by_worker_index[i] = 0;
2828             }
2829           else
2830             hf->last_n_vectors = hf->n_vectors;
2831         }
2832       congested_handoff_queue_by_worker_index[i] =
2833         (vlib_frame_queue_t *) (~0);
2834     }
2835   hf = 0;
2836   current_worker_index = ~0;
2837   return frame->n_vectors;
2838 }
2839
2840 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2841   .function = snat_out2in_worker_handoff_fn,
2842   .name = "nat44-out2in-worker-handoff",
2843   .vector_size = sizeof (u32),
2844   .format_trace = format_snat_out2in_worker_handoff_trace,
2845   .type = VLIB_NODE_TYPE_INTERNAL,
2846
2847   .n_next_nodes = 1,
2848
2849   .next_nodes = {
2850     [0] = "error-drop",
2851   },
2852 };
2853
2854 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2855
2856 static uword
2857 snat_out2in_fast_node_fn (vlib_main_t * vm,
2858                           vlib_node_runtime_t * node,
2859                           vlib_frame_t * frame)
2860 {
2861   u32 n_left_from, * from, * to_next;
2862   snat_out2in_next_t next_index;
2863   u32 pkts_processed = 0;
2864   snat_main_t * sm = &snat_main;
2865
2866   from = vlib_frame_vector_args (frame);
2867   n_left_from = frame->n_vectors;
2868   next_index = node->cached_next_index;
2869
2870   while (n_left_from > 0)
2871     {
2872       u32 n_left_to_next;
2873
2874       vlib_get_next_frame (vm, node, next_index,
2875                            to_next, n_left_to_next);
2876
2877       while (n_left_from > 0 && n_left_to_next > 0)
2878         {
2879           u32 bi0;
2880           vlib_buffer_t * b0;
2881           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2882           u32 sw_if_index0;
2883           ip4_header_t * ip0;
2884           ip_csum_t sum0;
2885           u32 new_addr0, old_addr0;
2886           u16 new_port0, old_port0;
2887           udp_header_t * udp0;
2888           tcp_header_t * tcp0;
2889           icmp46_header_t * icmp0;
2890           snat_session_key_t key0, sm0;
2891           u32 proto0;
2892           u32 rx_fib_index0;
2893
2894           /* speculatively enqueue b0 to the current next frame */
2895           bi0 = from[0];
2896           to_next[0] = bi0;
2897           from += 1;
2898           to_next += 1;
2899           n_left_from -= 1;
2900           n_left_to_next -= 1;
2901
2902           b0 = vlib_get_buffer (vm, bi0);
2903
2904           ip0 = vlib_buffer_get_current (b0);
2905           udp0 = ip4_next_header (ip0);
2906           tcp0 = (tcp_header_t *) udp0;
2907           icmp0 = (icmp46_header_t *) udp0;
2908
2909           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2910           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2911
2912           vnet_feature_next (sw_if_index0, &next0, b0);
2913
2914           if (PREDICT_FALSE(ip0->ttl == 1))
2915             {
2916               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2917               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2918                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2919                                            0);
2920               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2921               goto trace00;
2922             }
2923
2924           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2925
2926           if (PREDICT_FALSE (proto0 == ~0))
2927               goto trace00;
2928
2929           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2930             {
2931               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2932                                   rx_fib_index0, node, next0, ~0, 0, 0);
2933               goto trace00;
2934             }
2935
2936           key0.addr = ip0->dst_address;
2937           key0.port = udp0->dst_port;
2938           key0.fib_index = rx_fib_index0;
2939
2940           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2941             {
2942               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2943               goto trace00;
2944             }
2945
2946           new_addr0 = sm0.addr.as_u32;
2947           new_port0 = sm0.port;
2948           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2949           old_addr0 = ip0->dst_address.as_u32;
2950           ip0->dst_address.as_u32 = new_addr0;
2951
2952           sum0 = ip0->checksum;
2953           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2954                                  ip4_header_t,
2955                                  dst_address /* changed member */);
2956           ip0->checksum = ip_csum_fold (sum0);
2957
2958           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2959             {
2960                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2961                 {
2962                   old_port0 = tcp0->dst_port;
2963                   tcp0->dst_port = new_port0;
2964
2965                   sum0 = tcp0->checksum;
2966                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2967                                          ip4_header_t,
2968                                          dst_address /* changed member */);
2969
2970                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2971                                          ip4_header_t /* cheat */,
2972                                          length /* changed member */);
2973                   tcp0->checksum = ip_csum_fold(sum0);
2974                 }
2975               else
2976                 {
2977                   old_port0 = udp0->dst_port;
2978                   udp0->dst_port = new_port0;
2979                   udp0->checksum = 0;
2980                 }
2981             }
2982           else
2983             {
2984               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2985                 {
2986                   sum0 = tcp0->checksum;
2987                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2988                                          ip4_header_t,
2989                                          dst_address /* changed member */);
2990
2991                   tcp0->checksum = ip_csum_fold(sum0);
2992                 }
2993             }
2994
2995         trace00:
2996
2997           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2998                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2999             {
3000               snat_out2in_trace_t *t =
3001                  vlib_add_trace (vm, node, b0, sizeof (*t));
3002               t->sw_if_index = sw_if_index0;
3003               t->next_index = next0;
3004             }
3005
3006           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3007
3008           /* verify speculative enqueue, maybe switch current next frame */
3009           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3010                                            to_next, n_left_to_next,
3011                                            bi0, next0);
3012         }
3013
3014       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3015     }
3016
3017   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3018                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3019                                pkts_processed);
3020   return frame->n_vectors;
3021 }
3022
3023 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3024   .function = snat_out2in_fast_node_fn,
3025   .name = "nat44-out2in-fast",
3026   .vector_size = sizeof (u32),
3027   .format_trace = format_snat_out2in_fast_trace,
3028   .type = VLIB_NODE_TYPE_INTERNAL,
3029
3030   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3031   .error_strings = snat_out2in_error_strings,
3032
3033   .runtime_data_bytes = sizeof (snat_runtime_t),
3034
3035   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3036
3037   /* edit / add dispositions here */
3038   .next_nodes = {
3039     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3040     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3041     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3042     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3043     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3044   },
3045 };
3046 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);