src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_NEXT_IN2OUT,
 137   SNAT_OUT2IN_N_NEXT,
 138 } snat_out2in_next_t;
 139
 140 /**
 141  * @brief Create session for static mapping.
 142  *
 143  * Create NAT session initiated by host from external network with static
 144  * mapping.
 145  *
 146  * @param sm     NAT main.
 147  * @param b0     Vlib buffer.
 148  * @param in2out In2out NAT44 session key.
 149  * @param out2in Out2in NAT44 session key.
 150  * @param node   Vlib node.
 151  *
 152  * @returns SNAT session if successfully created otherwise 0.
 153  */
 154 static inline snat_session_t *
 155 create_session_for_static_mapping (snat_main_t *sm,
 156                                    vlib_buffer_t *b0,
 157                                    snat_session_key_t in2out,
 158                                    snat_session_key_t out2in,
 159                                    vlib_node_runtime_t * node,
 160                                    u32 thread_index)
 161 {
 162   snat_user_t *u;
 163   snat_session_t *s;
 164   clib_bihash_kv_8_8_t kv0;
 165   ip4_header_t *ip0;
 166   udp_header_t *udp0;
 167
 168   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 169     {
 170       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 171       return 0;
 172     }
 173
 174   ip0 = vlib_buffer_get_current (b0);
 175   udp0 = ip4_next_header (ip0);
 176
 177   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 178   if (!u)
 179     {
 180       clib_warning ("create NAT user failed");
 181       return 0;
 182     }
 183
 184   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 185   if (!s)
 186     {
 187       clib_warning ("create NAT session failed");
 188       return 0;
 189     }
 190
 191   s->outside_address_index = ~0;
 192   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 193   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 194   s->ext_host_port = udp0->src_port;
 195   user_session_increment (sm, u, 1 /* static */);
 196   s->in2out = in2out;
 197   s->out2in = out2in;
 198   s->in2out.protocol = out2in.protocol;
 199
 200   /* Add to translation hashes */
 201   kv0.key = s->in2out.as_u64;
 202   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 203   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 204                                1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208
 209   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 210                                1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   /* log NAT event */
 214   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 215                                       s->out2in.addr.as_u32,
 216                                       s->in2out.protocol,
 217                                       s->in2out.port,
 218                                       s->out2in.port,
 219                                       s->in2out.fib_index);
 220    return s;
 221 }
 222
 223 static_always_inline
 224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 225                                  snat_session_key_t *p_key0)
 226 {
 227   icmp46_header_t *icmp0;
 228   snat_session_key_t key0;
 229   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 230   ip4_header_t *inner_ip0;
 231   void *l4_header = 0;
 232   icmp46_header_t *inner_icmp0;
 233
 234   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   if (!icmp_is_error_message (icmp0))
 238     {
 239       key0.protocol = SNAT_PROTOCOL_ICMP;
 240       key0.addr = ip0->dst_address;
 241       key0.port = echo0->identifier;
 242     }
 243   else
 244     {
 245       inner_ip0 = (ip4_header_t *)(echo0+1);
 246       l4_header = ip4_next_header (inner_ip0);
 247       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 248       key0.addr = inner_ip0->src_address;
 249       switch (key0.protocol)
 250         {
 251         case SNAT_PROTOCOL_ICMP:
 252           inner_icmp0 = (icmp46_header_t*)l4_header;
 253           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 254           key0.port = inner_echo0->identifier;
 255           break;
 256         case SNAT_PROTOCOL_UDP:
 257         case SNAT_PROTOCOL_TCP:
 258           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 259           break;
 260         default:
 261           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 262         }
 263     }
 264   *p_key0 = key0;
 265   return -1; /* success */
 266 }
 267
 268 static_always_inline int
 269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 270 {
 271   icmp46_header_t *icmp0;
 272   nat_ed_ses_key_t key0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *)(icmp0+1);
 280
 281   if (!icmp_is_error_message (icmp0))
 282     {
 283       key0.proto = IP_PROTOCOL_ICMP;
 284       key0.l_addr = ip0->dst_address;
 285       key0.r_addr = ip0->src_address;
 286       key0.l_port = key0.r_port = echo0->identifier;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *)(echo0+1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       key0.proto = inner_ip0->protocol;
 293       key0.l_addr = inner_ip0->src_address;
 294       key0.r_addr = inner_ip0->dst_address;
 295       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 296         {
 297         case SNAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t*)l4_header;
 299           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 300           key0.l_port = key0.r_port = inner_echo0->identifier;
 301           break;
 302         case SNAT_PROTOCOL_UDP:
 303         case SNAT_PROTOCOL_TCP:
 304           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 305           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 306           break;
 307         default:
 308           return -1;
 309         }
 310     }
 311   *p_key0 = key0;
 312   return 0;
 313 }
 314
 315 static int
 316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 317               u32 thread_index)
 318 {
 319   snat_session_key_t key;
 320   clib_bihash_kv_8_8_t kv, value;
 321
 322   key.addr = ip->src_address;
 323   key.port = src_port;
 324   key.protocol = proto;
 325   key.fib_index = sm->inside_fib_index;
 326   kv.key = key.as_u64;
 327
 328   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 329                                &value))
 330     return 1;
 331
 332   return 0;
 333 }
 334
 335 static void
 336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 337                       u32 thread_index)
 338 {
 339   nat_ed_ses_key_t key;
 340   clib_bihash_kv_16_8_t kv, value;
 341   udp_header_t *udp;
 342   snat_user_t *u;
 343   snat_session_t *s = 0;
 344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 345
 346   if (ip->protocol == IP_PROTOCOL_ICMP)
 347     {
 348       if (icmp_get_ed_key (ip, &key))
 349         return;
 350     }
 351   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 352     {
 353       udp = ip4_next_header(ip);
 354       key.r_addr = ip->src_address;
 355       key.l_addr = ip->dst_address;
 356       key.proto = ip->protocol;
 357       key.l_port = udp->dst_port;
 358       key.r_port = udp->src_port;
 359     }
 360   else
 361     {
 362       key.r_addr = ip->src_address;
 363       key.l_addr = ip->dst_address;
 364       key.proto = ip->protocol;
 365       key.l_port = key.r_port = 0;
 366     }
 367   key.fib_index = 0;
 368   kv.key[0] = key.as_u64[0];
 369   kv.key[1] = key.as_u64[1];
 370
 371   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 372     {
 373       s = pool_elt_at_index (tsm->sessions, value.value);
 374     }
 375   else
 376     {
 377       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 378         return;
 379
 380       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
 381       if (!u)
 382         {
 383           clib_warning ("create NAT user failed");
 384           return;
 385         }
 386
 387       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 388       if (!s)
 389         {
 390           clib_warning ("create NAT session failed");
 391           return;
 392         }
 393
 394       s->ext_host_addr = key.r_addr;
 395       s->ext_host_port = key.r_port;
 396       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 397       s->outside_address_index = ~0;
 398       s->out2in.addr = key.l_addr;
 399       s->out2in.port = key.l_port;
 400       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 401       s->out2in.fib_index = 0;
 402       s->in2out = s->out2in;
 403       user_session_increment (sm, u, 0);
 404
 405       kv.value = s - tsm->sessions;
 406       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 407         clib_warning ("in2out_ed key add failed");
 408     }
 409
 410   if (ip->protocol == IP_PROTOCOL_TCP)
 411     {
 412       tcp_header_t *tcp = ip4_next_header(ip);
 413       nat44_set_tcp_session_state (sm, s, tcp, thread_index);
 414     }
 415   /* Per-user LRU list maintenance */
 416   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 417   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 418                       s->per_user_index);
 419 }
 420
 421 /**
 422  * Get address and port values to be used for ICMP packet translation
 423  * and create session if needed
 424  *
 425  * @param[in,out] sm             NAT main
 426  * @param[in,out] node           NAT node runtime
 427  * @param[in] thread_index       thread index
 428  * @param[in,out] b0             buffer containing packet to be translated
 429  * @param[out] p_proto           protocol used for matching
 430  * @param[out] p_value           address and port after NAT translation
 431  * @param[out] p_dont_translate  if packet should not be translated
 432  * @param d                      optional parameter
 433  * @param e                      optional parameter
 434  */
 435 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 436                            u32 thread_index, vlib_buffer_t *b0,
 437                            ip4_header_t *ip0, u8 *p_proto,
 438                            snat_session_key_t *p_value,
 439                            u8 *p_dont_translate, void *d, void *e)
 440 {
 441   icmp46_header_t *icmp0;
 442   u32 sw_if_index0;
 443   u32 rx_fib_index0;
 444   snat_session_key_t key0;
 445   snat_session_key_t sm0;
 446   snat_session_t *s0 = 0;
 447   u8 dont_translate = 0;
 448   clib_bihash_kv_8_8_t kv0, value0;
 449   u8 is_addr_only;
 450   u32 next0 = ~0;
 451   int err;
 452
 453   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 454   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 455   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 456
 457   key0.protocol = 0;
 458
 459   err = icmp_get_key (ip0, &key0);
 460   if (err != -1)
 461     {
 462       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 463       next0 = SNAT_OUT2IN_NEXT_DROP;
 464       goto out;
 465     }
 466   key0.fib_index = rx_fib_index0;
 467
 468   kv0.key = key0.as_u64;
 469
 470   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 471                               &value0))
 472     {
 473       /* Try to match static mapping by external address and port,
 474          destination address and port in packet */
 475       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 476         {
 477           if (!sm->forwarding_enabled)
 478             {
 479               /* Don't NAT packet aimed at the intfc address */
 480               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 481                                                   ip0->dst_address.as_u32)))
 482                 {
 483                   dont_translate = 1;
 484                   goto out;
 485                 }
 486               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 487               next0 = SNAT_OUT2IN_NEXT_DROP;
 488               goto out;
 489             }
 490           else
 491             {
 492               dont_translate = 1;
 493               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 494                 {
 495                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 496                   goto out;
 497                 }
 498               create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
 499               goto out;
 500             }
 501         }
 502
 503       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 504                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 505         {
 506           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 507           next0 = SNAT_OUT2IN_NEXT_DROP;
 508           goto out;
 509         }
 510
 511       /* Create session initiated by host from external network */
 512       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 513                                              node, thread_index);
 514
 515       if (!s0)
 516         {
 517           next0 = SNAT_OUT2IN_NEXT_DROP;
 518           goto out;
 519         }
 520     }
 521   else
 522     {
 523       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 524                         icmp0->type != ICMP4_echo_request &&
 525                         !icmp_is_error_message (icmp0)))
 526         {
 527           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 528           next0 = SNAT_OUT2IN_NEXT_DROP;
 529           goto out;
 530         }
 531
 532       if (PREDICT_FALSE (value0.value == ~0ULL))
 533         {
 534           nat_ed_ses_key_t key;
 535           clib_bihash_kv_16_8_t s_kv, s_value;
 536
 537           key.as_u64[0] = 0;
 538           key.as_u64[1] = 0;
 539           if (icmp_get_ed_key (ip0, &key))
 540             {
 541               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 542               next0 = SNAT_OUT2IN_NEXT_DROP;
 543               goto out;
 544             }
 545           key.fib_index = rx_fib_index0;
 546           s_kv.key[0] = key.as_u64[0];
 547           s_kv.key[1] = key.as_u64[1];
 548           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 549             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 550                                     s_value.value);
 551           else
 552            {
 553               next0 = SNAT_OUT2IN_NEXT_DROP;
 554               goto out;
 555            }
 556         }
 557       else
 558         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 559                                 value0.value);
 560     }
 561
 562 out:
 563   *p_proto = key0.protocol;
 564   if (s0)
 565     *p_value = s0->in2out;
 566   *p_dont_translate = dont_translate;
 567   if (d)
 568     *(snat_session_t**)d = s0;
 569   return next0;
 570 }
 571
 572 /**
 573  * Get address and port values to be used for ICMP packet translation
 574  *
 575  * @param[in] sm                 NAT main
 576  * @param[in,out] node           NAT node runtime
 577  * @param[in] thread_index       thread index
 578  * @param[in,out] b0             buffer containing packet to be translated
 579  * @param[out] p_proto           protocol used for matching
 580  * @param[out] p_value           address and port after NAT translation
 581  * @param[out] p_dont_translate  if packet should not be translated
 582  * @param d                      optional parameter
 583  * @param e                      optional parameter
 584  */
 585 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 586                            u32 thread_index, vlib_buffer_t *b0,
 587                            ip4_header_t *ip0, u8 *p_proto,
 588                            snat_session_key_t *p_value,
 589                            u8 *p_dont_translate, void *d, void *e)
 590 {
 591   icmp46_header_t *icmp0;
 592   u32 sw_if_index0;
 593   u32 rx_fib_index0;
 594   snat_session_key_t key0;
 595   snat_session_key_t sm0;
 596   u8 dont_translate = 0;
 597   u8 is_addr_only;
 598   u32 next0 = ~0;
 599   int err;
 600
 601   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 602   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 603   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 604
 605   err = icmp_get_key (ip0, &key0);
 606   if (err != -1)
 607     {
 608       b0->error = node->errors[err];
 609       next0 = SNAT_OUT2IN_NEXT_DROP;
 610       goto out2;
 611     }
 612   key0.fib_index = rx_fib_index0;
 613
 614   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 615     {
 616       /* Don't NAT packet aimed at the intfc address */
 617       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 618         {
 619           dont_translate = 1;
 620           goto out;
 621         }
 622       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 623       next0 = SNAT_OUT2IN_NEXT_DROP;
 624       goto out;
 625     }
 626
 627   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 628                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 629                     !icmp_is_error_message (icmp0)))
 630     {
 631       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 632       next0 = SNAT_OUT2IN_NEXT_DROP;
 633       goto out;
 634     }
 635
 636 out:
 637   *p_value = sm0;
 638 out2:
 639   *p_proto = key0.protocol;
 640   *p_dont_translate = dont_translate;
 641   return next0;
 642 }
 643
 644 static inline u32 icmp_out2in (snat_main_t *sm,
 645                                vlib_buffer_t * b0,
 646                                ip4_header_t * ip0,
 647                                icmp46_header_t * icmp0,
 648                                u32 sw_if_index0,
 649                                u32 rx_fib_index0,
 650                                vlib_node_runtime_t * node,
 651                                u32 next0,
 652                                u32 thread_index,
 653                                void *d,
 654                                void *e)
 655 {
 656   snat_session_key_t sm0;
 657   u8 protocol;
 658   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 659   ip4_header_t *inner_ip0 = 0;
 660   void *l4_header = 0;
 661   icmp46_header_t *inner_icmp0;
 662   u8 dont_translate;
 663   u32 new_addr0, old_addr0;
 664   u16 old_id0, new_id0;
 665   ip_csum_t sum0;
 666   u16 checksum0;
 667   u32 next0_tmp;
 668
 669   echo0 = (icmp_echo_header_t *)(icmp0+1);
 670
 671   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 672                                        &protocol, &sm0, &dont_translate, d, e);
 673   if (next0_tmp != ~0)
 674     next0 = next0_tmp;
 675   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 676     goto out;
 677
 678   sum0 = ip_incremental_checksum (0, icmp0,
 679                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 680   checksum0 = ~ip_csum_fold (sum0);
 681   if (checksum0 != 0 && checksum0 != 0xffff)
 682     {
 683       next0 = SNAT_OUT2IN_NEXT_DROP;
 684       goto out;
 685     }
 686
 687   old_addr0 = ip0->dst_address.as_u32;
 688   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 689   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 690
 691   sum0 = ip0->checksum;
 692   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 693                          dst_address /* changed member */);
 694   ip0->checksum = ip_csum_fold (sum0);
 695
 696   if (icmp0->checksum == 0)
 697     icmp0->checksum = 0xffff;
 698
 699   if (!icmp_is_error_message (icmp0))
 700     {
 701       new_id0 = sm0.port;
 702       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 703         {
 704           old_id0 = echo0->identifier;
 705           new_id0 = sm0.port;
 706           echo0->identifier = new_id0;
 707
 708           sum0 = icmp0->checksum;
 709           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 710                                  identifier /* changed member */);
 711           icmp0->checksum = ip_csum_fold (sum0);
 712         }
 713     }
 714   else
 715     {
 716       inner_ip0 = (ip4_header_t *)(echo0+1);
 717       l4_header = ip4_next_header (inner_ip0);
 718
 719       if (!ip4_header_checksum_is_valid (inner_ip0))
 720         {
 721           next0 = SNAT_OUT2IN_NEXT_DROP;
 722           goto out;
 723         }
 724
 725       old_addr0 = inner_ip0->src_address.as_u32;
 726       inner_ip0->src_address = sm0.addr;
 727       new_addr0 = inner_ip0->src_address.as_u32;
 728
 729       sum0 = icmp0->checksum;
 730       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 731                              src_address /* changed member */);
 732       icmp0->checksum = ip_csum_fold (sum0);
 733
 734       switch (protocol)
 735         {
 736         case SNAT_PROTOCOL_ICMP:
 737           inner_icmp0 = (icmp46_header_t*)l4_header;
 738           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 739
 740           old_id0 = inner_echo0->identifier;
 741           new_id0 = sm0.port;
 742           inner_echo0->identifier = new_id0;
 743
 744           sum0 = icmp0->checksum;
 745           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 746                                  identifier);
 747           icmp0->checksum = ip_csum_fold (sum0);
 748           break;
 749         case SNAT_PROTOCOL_UDP:
 750         case SNAT_PROTOCOL_TCP:
 751           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 752           new_id0 = sm0.port;
 753           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 754
 755           sum0 = icmp0->checksum;
 756           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 757                                  src_port);
 758           icmp0->checksum = ip_csum_fold (sum0);
 759           break;
 760         default:
 761           ASSERT(0);
 762         }
 763     }
 764
 765 out:
 766   return next0;
 767 }
 768
 769
 770 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 771                                          vlib_buffer_t * b0,
 772                                          ip4_header_t * ip0,
 773                                          icmp46_header_t * icmp0,
 774                                          u32 sw_if_index0,
 775                                          u32 rx_fib_index0,
 776                                          vlib_node_runtime_t * node,
 777                                          u32 next0, f64 now,
 778                                          u32 thread_index,
 779                                          snat_session_t ** p_s0)
 780 {
 781   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 782                       next0, thread_index, p_s0, 0);
 783   snat_session_t * s0 = *p_s0;
 784   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 785     {
 786       /* Accounting */
 787       s0->last_heard = now;
 788       s0->total_pkts++;
 789       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 790       /* Per-user LRU list maintenance */
 791       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 792                          s0->per_user_index);
 793       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 794                           s0->per_user_list_head_index,
 795                           s0->per_user_index);
 796     }
 797   return next0;
 798 }
 799
 800 static snat_session_t *
 801 snat_out2in_unknown_proto (snat_main_t *sm,
 802                            vlib_buffer_t * b,
 803                            ip4_header_t * ip,
 804                            u32 rx_fib_index,
 805                            u32 thread_index,
 806                            f64 now,
 807                            vlib_main_t * vm,
 808                            vlib_node_runtime_t * node)
 809 {
 810   clib_bihash_kv_8_8_t kv, value;
 811   clib_bihash_kv_16_8_t s_kv, s_value;
 812   snat_static_mapping_t *m;
 813   snat_session_key_t m_key;
 814   u32 old_addr, new_addr;
 815   ip_csum_t sum;
 816   nat_ed_ses_key_t key;
 817   snat_session_t * s;
 818   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 819   snat_user_t *u;
 820
 821   old_addr = ip->dst_address.as_u32;
 822
 823   key.l_addr = ip->dst_address;
 824   key.r_addr = ip->src_address;
 825   key.fib_index = rx_fib_index;
 826   key.proto = ip->protocol;
 827   key.r_port = 0;
 828   key.l_port = 0;
 829   s_kv.key[0] = key.as_u64[0];
 830   s_kv.key[1] = key.as_u64[1];
 831
 832   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 833     {
 834       s = pool_elt_at_index (tsm->sessions, s_value.value);
 835       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 836     }
 837   else
 838     {
 839       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 840         {
 841           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 842           return 0;
 843         }
 844
 845       m_key.addr = ip->dst_address;
 846       m_key.port = 0;
 847       m_key.protocol = 0;
 848       m_key.fib_index = rx_fib_index;
 849       kv.key = m_key.as_u64;
 850       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 851         {
 852           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 853           return 0;
 854         }
 855
 856       m = pool_elt_at_index (sm->static_mappings, value.value);
 857
 858       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 859
 860       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 861                                   thread_index);
 862       if (!u)
 863         {
 864           clib_warning ("create NAT user failed");
 865           return 0;
 866         }
 867
 868       /* Create a new session */
 869       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 870       if (!s)
 871         {
 872           clib_warning ("create NAT session failed");
 873           return 0;
 874         }
 875
 876       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 877       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 878       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 879       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 880       s->outside_address_index = ~0;
 881       s->out2in.addr.as_u32 = old_addr;
 882       s->out2in.fib_index = rx_fib_index;
 883       s->in2out.addr.as_u32 = new_addr;
 884       s->in2out.fib_index = m->fib_index;
 885       s->in2out.port = s->out2in.port = ip->protocol;
 886       user_session_increment (sm, u, 1 /* static */);
 887
 888       /* Add to lookup tables */
 889       s_kv.value = s - tsm->sessions;
 890       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 891         clib_warning ("out2in key add failed");
 892
 893       key.l_addr = ip->dst_address;
 894       key.fib_index = m->fib_index;
 895       s_kv.key[0] = key.as_u64[0];
 896       s_kv.key[1] = key.as_u64[1];
 897       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 898         clib_warning ("in2out key add failed");
 899    }
 900
 901   /* Update IP checksum */
 902   sum = ip->checksum;
 903   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 904   ip->checksum = ip_csum_fold (sum);
 905
 906   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 907
 908   /* Accounting */
 909   s->last_heard = now;
 910   s->total_pkts++;
 911   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 912   /* Per-user LRU list maintenance */
 913   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 914   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 915                       s->per_user_index);
 916
 917   return s;
 918 }
 919
 920 static snat_session_t *
 921 snat_out2in_lb (snat_main_t *sm,
 922                 vlib_buffer_t * b,
 923                 ip4_header_t * ip,
 924                 u32 rx_fib_index,
 925                 u32 thread_index,
 926                 f64 now,
 927                 vlib_main_t * vm,
 928                 vlib_node_runtime_t * node)
 929 {
 930   nat_ed_ses_key_t key;
 931   clib_bihash_kv_16_8_t s_kv, s_value;
 932   udp_header_t *udp = ip4_next_header (ip);
 933   tcp_header_t *tcp = (tcp_header_t *) udp;
 934   snat_session_t *s = 0;
 935   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 936   snat_session_key_t e_key, l_key;
 937   u32 old_addr, new_addr;
 938   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 939   u16 new_port, old_port;
 940   ip_csum_t sum;
 941   snat_user_t *u;
 942   u32 address_index;
 943   snat_session_key_t eh_key;
 944   twice_nat_type_t twice_nat;
 945   u8 lb;
 946
 947   old_addr = ip->dst_address.as_u32;
 948
 949   key.l_addr = ip->dst_address;
 950   key.r_addr = ip->src_address;
 951   key.fib_index = rx_fib_index;
 952   key.proto = ip->protocol;
 953   key.r_port = udp->src_port;
 954   key.l_port = udp->dst_port;
 955   s_kv.key[0] = key.as_u64[0];
 956   s_kv.key[1] = key.as_u64[1];
 957
 958   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 959     {
 960       s = pool_elt_at_index (tsm->sessions, s_value.value);
 961     }
 962   else
 963     {
 964       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 965         {
 966           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 967           return 0;
 968         }
 969
 970       e_key.addr = ip->dst_address;
 971       e_key.port = udp->dst_port;
 972       e_key.protocol = proto;
 973       e_key.fib_index = rx_fib_index;
 974       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 975         return 0;
 976
 977       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 978                                   thread_index);
 979       if (!u)
 980       {
 981         clib_warning ("create NAT user failed");
 982         return 0;
 983       }
 984
 985       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 986       if (!s)
 987         {
 988           clib_warning ("create NAT session failed");
 989           return 0;
 990         }
 991
 992       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 993       s->ext_host_port = udp->src_port;
 994       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 995       if (lb)
 996         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 997       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 998       s->outside_address_index = ~0;
 999       s->out2in = e_key;
1000       s->in2out = l_key;
1001       user_session_increment (sm, u, 1 /* static */);
1002
1003       /* Add to lookup tables */
1004       s_kv.value = s - tsm->sessions;
1005       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1006         clib_warning ("out2in-ed key add failed");
1007
1008       if (twice_nat == TWICE_NAT ||
1009           (twice_nat == TWICE_NAT_SELF &&
1010            ip->src_address.as_u32 == l_key.addr.as_u32))
1011         {
1012           eh_key.protocol = proto;
1013           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1014                                                    thread_index, &eh_key,
1015                                                    &address_index,
1016                                                    sm->port_per_thread,
1017                                                    sm->per_thread_data[thread_index].snat_thread_index))
1018             {
1019               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1020               return 0;
1021             }
1022           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1023           key.r_port = s->ext_host_nat_port = eh_key.port;
1024           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1025         }
1026       key.l_addr = l_key.addr;
1027       key.fib_index = l_key.fib_index;
1028       key.l_port = l_key.port;
1029       s_kv.key[0] = key.as_u64[0];
1030       s_kv.key[1] = key.as_u64[1];
1031       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1032         clib_warning ("in2out-ed key add failed");
1033     }
1034
1035   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1036
1037   /* Update IP checksum */
1038   sum = ip->checksum;
1039   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1040   if (is_twice_nat_session (s))
1041     sum = ip_csum_update (sum, ip->src_address.as_u32,
1042                           s->ext_host_nat_addr.as_u32, ip4_header_t,
1043                           src_address);
1044   ip->checksum = ip_csum_fold (sum);
1045
1046   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1047     {
1048       old_port = tcp->dst_port;
1049       tcp->dst_port = s->in2out.port;
1050       new_port = tcp->dst_port;
1051
1052       sum = tcp->checksum;
1053       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1054       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1055       if (is_twice_nat_session (s))
1056         {
1057           sum = ip_csum_update (sum, ip->src_address.as_u32,
1058                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1059                                 dst_address);
1060           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1061                                 ip4_header_t, length);
1062           tcp->src_port = s->ext_host_nat_port;
1063           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1064         }
1065       tcp->checksum = ip_csum_fold(sum);
1066       nat44_set_tcp_session_state (sm, s, tcp, thread_index);
1067     }
1068   else
1069     {
1070       udp->dst_port = s->in2out.port;
1071       if (is_twice_nat_session (s))
1072         {
1073           udp->src_port = s->ext_host_nat_port;
1074           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1075         }
1076       udp->checksum = 0;
1077     }
1078
1079   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1080
1081   /* Accounting */
1082   s->last_heard = now;
1083   s->total_pkts++;
1084   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1085   /* Per-user LRU list maintenance */
1086   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1087   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1088                       s->per_user_index);
1089
1090   return s;
1091 }
1092
1093 static uword
1094 snat_out2in_node_fn (vlib_main_t * vm,
1095                   vlib_node_runtime_t * node,
1096                   vlib_frame_t * frame)
1097 {
1098   u32 n_left_from, * from, * to_next;
1099   snat_out2in_next_t next_index;
1100   u32 pkts_processed = 0;
1101   snat_main_t * sm = &snat_main;
1102   f64 now = vlib_time_now (vm);
1103   u32 thread_index = vlib_get_thread_index ();
1104
1105   from = vlib_frame_vector_args (frame);
1106   n_left_from = frame->n_vectors;
1107   next_index = node->cached_next_index;
1108
1109   while (n_left_from > 0)
1110     {
1111       u32 n_left_to_next;
1112
1113       vlib_get_next_frame (vm, node, next_index,
1114                            to_next, n_left_to_next);
1115
1116       while (n_left_from >= 4 && n_left_to_next >= 2)
1117         {
1118           u32 bi0, bi1;
1119           vlib_buffer_t * b0, * b1;
1120           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1121           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1122           u32 sw_if_index0, sw_if_index1;
1123           ip4_header_t * ip0, *ip1;
1124           ip_csum_t sum0, sum1;
1125           u32 new_addr0, old_addr0;
1126           u16 new_port0, old_port0;
1127           u32 new_addr1, old_addr1;
1128           u16 new_port1, old_port1;
1129           udp_header_t * udp0, * udp1;
1130           tcp_header_t * tcp0, * tcp1;
1131           icmp46_header_t * icmp0, * icmp1;
1132           snat_session_key_t key0, key1, sm0, sm1;
1133           u32 rx_fib_index0, rx_fib_index1;
1134           u32 proto0, proto1;
1135           snat_session_t * s0 = 0, * s1 = 0;
1136           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1137
1138           /* Prefetch next iteration. */
1139           {
1140             vlib_buffer_t * p2, * p3;
1141
1142             p2 = vlib_get_buffer (vm, from[2]);
1143             p3 = vlib_get_buffer (vm, from[3]);
1144
1145             vlib_prefetch_buffer_header (p2, LOAD);
1146             vlib_prefetch_buffer_header (p3, LOAD);
1147
1148             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1149             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1150           }
1151
1152           /* speculatively enqueue b0 and b1 to the current next frame */
1153           to_next[0] = bi0 = from[0];
1154           to_next[1] = bi1 = from[1];
1155           from += 2;
1156           to_next += 2;
1157           n_left_from -= 2;
1158           n_left_to_next -= 2;
1159
1160           b0 = vlib_get_buffer (vm, bi0);
1161           b1 = vlib_get_buffer (vm, bi1);
1162
1163           vnet_buffer (b0)->snat.flags = 0;
1164           vnet_buffer (b1)->snat.flags = 0;
1165
1166           ip0 = vlib_buffer_get_current (b0);
1167           udp0 = ip4_next_header (ip0);
1168           tcp0 = (tcp_header_t *) udp0;
1169           icmp0 = (icmp46_header_t *) udp0;
1170
1171           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1172           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1173                                    sw_if_index0);
1174
1175           if (PREDICT_FALSE(ip0->ttl == 1))
1176             {
1177               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1178               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1179                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1180                                            0);
1181               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1182               goto trace0;
1183             }
1184
1185           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1186
1187           if (PREDICT_FALSE (proto0 == ~0))
1188             {
1189               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1190                                              thread_index, now, vm, node);
1191               if (!sm->forwarding_enabled)
1192                 if (!s0)
1193                   next0 = SNAT_OUT2IN_NEXT_DROP;
1194               goto trace0;
1195             }
1196
1197           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1198             {
1199               next0 = icmp_out2in_slow_path
1200                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1201                  next0, now, thread_index, &s0);
1202               goto trace0;
1203             }
1204
1205           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1206             {
1207               next0 = SNAT_OUT2IN_NEXT_REASS;
1208               goto trace0;
1209             }
1210
1211           key0.addr = ip0->dst_address;
1212           key0.port = udp0->dst_port;
1213           key0.protocol = proto0;
1214           key0.fib_index = rx_fib_index0;
1215
1216           kv0.key = key0.as_u64;
1217
1218           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1219                                       &kv0, &value0))
1220             {
1221               /* Try to match static mapping by external address and port,
1222                  destination address and port in packet */
1223               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1224                 {
1225                   /*
1226                    * Send DHCP packets to the ipv4 stack, or we won't
1227                    * be able to use dhcp client on the outside interface
1228                    */
1229                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1230                       && (udp0->dst_port ==
1231                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1232                     {
1233                       vnet_feature_next
1234                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1235                       goto trace0;
1236                     }
1237
1238                   if (!sm->forwarding_enabled)
1239                     {
1240                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1241                       next0 = SNAT_OUT2IN_NEXT_DROP;
1242                       goto trace0;
1243                     }
1244                   else
1245                     {
1246                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1247                         {
1248                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1249                           goto trace0;
1250                         }
1251                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1252                       goto trace0;
1253                     }
1254                 }
1255
1256               /* Create session initiated by host from external network */
1257               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1258                                                      thread_index);
1259               if (!s0)
1260                 {
1261                   next0 = SNAT_OUT2IN_NEXT_DROP;
1262                   goto trace0;
1263                 }
1264             }
1265           else
1266             {
1267               if (PREDICT_FALSE (value0.value == ~0ULL))
1268                 {
1269                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1270                                       now, vm, node);
1271                   if (!s0)
1272                     next0 = SNAT_OUT2IN_NEXT_DROP;
1273                   goto trace0;
1274                 }
1275               else
1276                 {
1277                   s0 = pool_elt_at_index (
1278                     sm->per_thread_data[thread_index].sessions,
1279                     value0.value);
1280                 }
1281             }
1282
1283           old_addr0 = ip0->dst_address.as_u32;
1284           ip0->dst_address = s0->in2out.addr;
1285           new_addr0 = ip0->dst_address.as_u32;
1286           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1287
1288           sum0 = ip0->checksum;
1289           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1290                                  ip4_header_t,
1291                                  dst_address /* changed member */);
1292           ip0->checksum = ip_csum_fold (sum0);
1293
1294           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1295             {
1296               old_port0 = tcp0->dst_port;
1297               tcp0->dst_port = s0->in2out.port;
1298               new_port0 = tcp0->dst_port;
1299
1300               sum0 = tcp0->checksum;
1301               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1302                                      ip4_header_t,
1303                                      dst_address /* changed member */);
1304
1305               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1306                                      ip4_header_t /* cheat */,
1307                                      length /* changed member */);
1308               tcp0->checksum = ip_csum_fold(sum0);
1309               nat44_set_tcp_session_state (sm, s0, tcp0, thread_index);
1310             }
1311           else
1312             {
1313               old_port0 = udp0->dst_port;
1314               udp0->dst_port = s0->in2out.port;
1315               udp0->checksum = 0;
1316             }
1317
1318           /* Accounting */
1319           s0->last_heard = now;
1320           s0->total_pkts++;
1321           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1322           /* Per-user LRU list maintenance */
1323           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1324                              s0->per_user_index);
1325           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1326                               s0->per_user_list_head_index,
1327                               s0->per_user_index);
1328         trace0:
1329
1330           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1331                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1332             {
1333               snat_out2in_trace_t *t =
1334                  vlib_add_trace (vm, node, b0, sizeof (*t));
1335               t->sw_if_index = sw_if_index0;
1336               t->next_index = next0;
1337               t->session_index = ~0;
1338               if (s0)
1339                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1340             }
1341
1342           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1343
1344
1345           ip1 = vlib_buffer_get_current (b1);
1346           udp1 = ip4_next_header (ip1);
1347           tcp1 = (tcp_header_t *) udp1;
1348           icmp1 = (icmp46_header_t *) udp1;
1349
1350           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1351           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1352                                    sw_if_index1);
1353
1354           if (PREDICT_FALSE(ip1->ttl == 1))
1355             {
1356               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1357               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1358                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1359                                            0);
1360               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1361               goto trace1;
1362             }
1363
1364           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1365
1366           if (PREDICT_FALSE (proto1 == ~0))
1367             {
1368               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1369                                              thread_index, now, vm, node);
1370               if (!sm->forwarding_enabled)
1371                 if (!s1)
1372                   next1 = SNAT_OUT2IN_NEXT_DROP;
1373               goto trace1;
1374             }
1375
1376           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1377             {
1378               next1 = icmp_out2in_slow_path
1379                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1380                  next1, now, thread_index, &s1);
1381               goto trace1;
1382             }
1383
1384           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1385             {
1386               next1 = SNAT_OUT2IN_NEXT_REASS;
1387               goto trace1;
1388             }
1389
1390           key1.addr = ip1->dst_address;
1391           key1.port = udp1->dst_port;
1392           key1.protocol = proto1;
1393           key1.fib_index = rx_fib_index1;
1394
1395           kv1.key = key1.as_u64;
1396
1397           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1398                                       &kv1, &value1))
1399             {
1400               /* Try to match static mapping by external address and port,
1401                  destination address and port in packet */
1402               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1403                 {
1404                   /*
1405                    * Send DHCP packets to the ipv4 stack, or we won't
1406                    * be able to use dhcp client on the outside interface
1407                    */
1408                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1409                       && (udp1->dst_port ==
1410                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1411                     {
1412                       vnet_feature_next
1413                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1414                       goto trace1;
1415                     }
1416
1417                   if (!sm->forwarding_enabled)
1418                     {
1419                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1420                       next1 = SNAT_OUT2IN_NEXT_DROP;
1421                       goto trace1;
1422                     }
1423                   else
1424                     {
1425                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1426                         {
1427                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1428                           goto trace1;
1429                         }
1430                       create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1431                       goto trace1;
1432                     }
1433                 }
1434
1435               /* Create session initiated by host from external network */
1436               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1437                                                      thread_index);
1438               if (!s1)
1439                 {
1440                   next1 = SNAT_OUT2IN_NEXT_DROP;
1441                   goto trace1;
1442                 }
1443             }
1444           else
1445             {
1446               if (PREDICT_FALSE (value1.value == ~0ULL))
1447                 {
1448                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1449                                       now, vm, node);
1450                   if (!s1)
1451                     next1 = SNAT_OUT2IN_NEXT_DROP;
1452                   goto trace1;
1453                 }
1454               else
1455                 {
1456                   s1 = pool_elt_at_index (
1457                     sm->per_thread_data[thread_index].sessions,
1458                     value1.value);
1459                 }
1460             }
1461
1462           old_addr1 = ip1->dst_address.as_u32;
1463           ip1->dst_address = s1->in2out.addr;
1464           new_addr1 = ip1->dst_address.as_u32;
1465           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1466
1467           sum1 = ip1->checksum;
1468           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1469                                  ip4_header_t,
1470                                  dst_address /* changed member */);
1471           ip1->checksum = ip_csum_fold (sum1);
1472
1473           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1474             {
1475               old_port1 = tcp1->dst_port;
1476               tcp1->dst_port = s1->in2out.port;
1477               new_port1 = tcp1->dst_port;
1478
1479               sum1 = tcp1->checksum;
1480               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1481                                      ip4_header_t,
1482                                      dst_address /* changed member */);
1483
1484               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1485                                      ip4_header_t /* cheat */,
1486                                      length /* changed member */);
1487               tcp1->checksum = ip_csum_fold(sum1);
1488               nat44_set_tcp_session_state (sm, s1, tcp1, thread_index);
1489             }
1490           else
1491             {
1492               old_port1 = udp1->dst_port;
1493               udp1->dst_port = s1->in2out.port;
1494               udp1->checksum = 0;
1495             }
1496
1497           /* Accounting */
1498           s1->last_heard = now;
1499           s1->total_pkts++;
1500           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1501           /* Per-user LRU list maintenance */
1502           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1503                              s1->per_user_index);
1504           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1505                               s1->per_user_list_head_index,
1506                               s1->per_user_index);
1507         trace1:
1508
1509           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1510                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1511             {
1512               snat_out2in_trace_t *t =
1513                  vlib_add_trace (vm, node, b1, sizeof (*t));
1514               t->sw_if_index = sw_if_index1;
1515               t->next_index = next1;
1516               t->session_index = ~0;
1517               if (s1)
1518                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1519             }
1520
1521           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1522
1523           /* verify speculative enqueues, maybe switch current next frame */
1524           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1525                                            to_next, n_left_to_next,
1526                                            bi0, bi1, next0, next1);
1527         }
1528
1529       while (n_left_from > 0 && n_left_to_next > 0)
1530         {
1531           u32 bi0;
1532           vlib_buffer_t * b0;
1533           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1534           u32 sw_if_index0;
1535           ip4_header_t * ip0;
1536           ip_csum_t sum0;
1537           u32 new_addr0, old_addr0;
1538           u16 new_port0, old_port0;
1539           udp_header_t * udp0;
1540           tcp_header_t * tcp0;
1541           icmp46_header_t * icmp0;
1542           snat_session_key_t key0, sm0;
1543           u32 rx_fib_index0;
1544           u32 proto0;
1545           snat_session_t * s0 = 0;
1546           clib_bihash_kv_8_8_t kv0, value0;
1547
1548           /* speculatively enqueue b0 to the current next frame */
1549           bi0 = from[0];
1550           to_next[0] = bi0;
1551           from += 1;
1552           to_next += 1;
1553           n_left_from -= 1;
1554           n_left_to_next -= 1;
1555
1556           b0 = vlib_get_buffer (vm, bi0);
1557
1558           vnet_buffer (b0)->snat.flags = 0;
1559
1560           ip0 = vlib_buffer_get_current (b0);
1561           udp0 = ip4_next_header (ip0);
1562           tcp0 = (tcp_header_t *) udp0;
1563           icmp0 = (icmp46_header_t *) udp0;
1564
1565           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1566           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1567                                    sw_if_index0);
1568
1569           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1570
1571           if (PREDICT_FALSE (proto0 == ~0))
1572             {
1573               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1574                                              thread_index, now, vm, node);
1575               if (!sm->forwarding_enabled)
1576                 if (!s0)
1577                   next0 = SNAT_OUT2IN_NEXT_DROP;
1578               goto trace00;
1579             }
1580
1581           if (PREDICT_FALSE(ip0->ttl == 1))
1582             {
1583               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1584               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1585                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1586                                            0);
1587               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1588               goto trace00;
1589             }
1590
1591           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1592             {
1593               next0 = icmp_out2in_slow_path
1594                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1595                  next0, now, thread_index, &s0);
1596               goto trace00;
1597             }
1598
1599           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1600             {
1601               next0 = SNAT_OUT2IN_NEXT_REASS;
1602               goto trace00;
1603             }
1604
1605           key0.addr = ip0->dst_address;
1606           key0.port = udp0->dst_port;
1607           key0.protocol = proto0;
1608           key0.fib_index = rx_fib_index0;
1609
1610           kv0.key = key0.as_u64;
1611
1612           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1613                                       &kv0, &value0))
1614             {
1615               /* Try to match static mapping by external address and port,
1616                  destination address and port in packet */
1617               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1618                 {
1619                   /*
1620                    * Send DHCP packets to the ipv4 stack, or we won't
1621                    * be able to use dhcp client on the outside interface
1622                    */
1623                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1624                       && (udp0->dst_port ==
1625                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1626                     {
1627                       vnet_feature_next
1628                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1629                       goto trace00;
1630                     }
1631
1632                   if (!sm->forwarding_enabled)
1633                     {
1634                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1635                       next0 = SNAT_OUT2IN_NEXT_DROP;
1636                       goto trace00;
1637                     }
1638                   else
1639                     {
1640                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1641                         {
1642                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1643                           goto trace00;
1644                         }
1645                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1646                       goto trace00;
1647                     }
1648                 }
1649
1650               /* Create session initiated by host from external network */
1651               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1652                                                      thread_index);
1653               if (!s0)
1654                 {
1655                   next0 = SNAT_OUT2IN_NEXT_DROP;
1656                   goto trace00;
1657                 }
1658             }
1659           else
1660             {
1661               if (PREDICT_FALSE (value0.value == ~0ULL))
1662                 {
1663                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1664                                       now, vm, node);
1665                   if (!s0)
1666                     next0 = SNAT_OUT2IN_NEXT_DROP;
1667                   goto trace00;
1668                 }
1669               else
1670                 {
1671                   s0 = pool_elt_at_index (
1672                     sm->per_thread_data[thread_index].sessions,
1673                     value0.value);
1674                 }
1675             }
1676
1677           old_addr0 = ip0->dst_address.as_u32;
1678           ip0->dst_address = s0->in2out.addr;
1679           new_addr0 = ip0->dst_address.as_u32;
1680           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1681
1682           sum0 = ip0->checksum;
1683           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1684                                  ip4_header_t,
1685                                  dst_address /* changed member */);
1686           ip0->checksum = ip_csum_fold (sum0);
1687
1688           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1689             {
1690               old_port0 = tcp0->dst_port;
1691               tcp0->dst_port = s0->in2out.port;
1692               new_port0 = tcp0->dst_port;
1693
1694               sum0 = tcp0->checksum;
1695               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1696                                      ip4_header_t,
1697                                      dst_address /* changed member */);
1698
1699               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1700                                      ip4_header_t /* cheat */,
1701                                      length /* changed member */);
1702               tcp0->checksum = ip_csum_fold(sum0);
1703               nat44_set_tcp_session_state (sm, s0, tcp0, thread_index);
1704             }
1705           else
1706             {
1707               old_port0 = udp0->dst_port;
1708               udp0->dst_port = s0->in2out.port;
1709               udp0->checksum = 0;
1710             }
1711
1712           /* Accounting */
1713           s0->last_heard = now;
1714           s0->total_pkts++;
1715           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1716           /* Per-user LRU list maintenance */
1717           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1718                              s0->per_user_index);
1719           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1720                               s0->per_user_list_head_index,
1721                               s0->per_user_index);
1722         trace00:
1723
1724           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1725                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1726             {
1727               snat_out2in_trace_t *t =
1728                  vlib_add_trace (vm, node, b0, sizeof (*t));
1729               t->sw_if_index = sw_if_index0;
1730               t->next_index = next0;
1731               t->session_index = ~0;
1732               if (s0)
1733                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1734             }
1735
1736           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1737
1738           /* verify speculative enqueue, maybe switch current next frame */
1739           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1740                                            to_next, n_left_to_next,
1741                                            bi0, next0);
1742         }
1743
1744       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1745     }
1746
1747   vlib_node_increment_counter (vm, snat_out2in_node.index,
1748                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1749                                pkts_processed);
1750   return frame->n_vectors;
1751 }
1752
1753 VLIB_REGISTER_NODE (snat_out2in_node) = {
1754   .function = snat_out2in_node_fn,
1755   .name = "nat44-out2in",
1756   .vector_size = sizeof (u32),
1757   .format_trace = format_snat_out2in_trace,
1758   .type = VLIB_NODE_TYPE_INTERNAL,
1759
1760   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1761   .error_strings = snat_out2in_error_strings,
1762
1763   .runtime_data_bytes = sizeof (snat_runtime_t),
1764
1765   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1766
1767   /* edit / add dispositions here */
1768   .next_nodes = {
1769     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1770     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1771     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1772     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1773     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1774   },
1775 };
1776 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1777
1778 static uword
1779 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1780                             vlib_node_runtime_t * node,
1781                             vlib_frame_t * frame)
1782 {
1783   u32 n_left_from, *from, *to_next;
1784   snat_out2in_next_t next_index;
1785   u32 pkts_processed = 0;
1786   snat_main_t *sm = &snat_main;
1787   f64 now = vlib_time_now (vm);
1788   u32 thread_index = vlib_get_thread_index ();
1789   snat_main_per_thread_data_t *per_thread_data =
1790     &sm->per_thread_data[thread_index];
1791   u32 *fragments_to_drop = 0;
1792   u32 *fragments_to_loopback = 0;
1793
1794   from = vlib_frame_vector_args (frame);
1795   n_left_from = frame->n_vectors;
1796   next_index = node->cached_next_index;
1797
1798   while (n_left_from > 0)
1799     {
1800       u32 n_left_to_next;
1801
1802       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1803
1804       while (n_left_from > 0 && n_left_to_next > 0)
1805        {
1806           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1807           vlib_buffer_t *b0;
1808           u32 next0;
1809           u8 cached0 = 0;
1810           ip4_header_t *ip0;
1811           nat_reass_ip4_t *reass0;
1812           udp_header_t * udp0;
1813           tcp_header_t * tcp0;
1814           snat_session_key_t key0, sm0;
1815           clib_bihash_kv_8_8_t kv0, value0;
1816           snat_session_t * s0 = 0;
1817           u16 old_port0, new_port0;
1818           ip_csum_t sum0;
1819
1820           /* speculatively enqueue b0 to the current next frame */
1821           bi0 = from[0];
1822           to_next[0] = bi0;
1823           from += 1;
1824           to_next += 1;
1825           n_left_from -= 1;
1826           n_left_to_next -= 1;
1827
1828           b0 = vlib_get_buffer (vm, bi0);
1829           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1830
1831           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1832           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1833                                                                sw_if_index0);
1834
1835           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1836             {
1837               next0 = SNAT_OUT2IN_NEXT_DROP;
1838               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1839               goto trace0;
1840             }
1841
1842           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1843           udp0 = ip4_next_header (ip0);
1844           tcp0 = (tcp_header_t *) udp0;
1845           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1846
1847           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1848                                                  ip0->dst_address,
1849                                                  ip0->fragment_id,
1850                                                  ip0->protocol,
1851                                                  1,
1852                                                  &fragments_to_drop);
1853
1854           if (PREDICT_FALSE (!reass0))
1855             {
1856               next0 = SNAT_OUT2IN_NEXT_DROP;
1857               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1858               goto trace0;
1859             }
1860
1861           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1862             {
1863               key0.addr = ip0->dst_address;
1864               key0.port = udp0->dst_port;
1865               key0.protocol = proto0;
1866               key0.fib_index = rx_fib_index0;
1867               kv0.key = key0.as_u64;
1868
1869               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1870                 {
1871                   /* Try to match static mapping by external address and port,
1872                      destination address and port in packet */
1873                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1874                     {
1875                       /*
1876                        * Send DHCP packets to the ipv4 stack, or we won't
1877                        * be able to use dhcp client on the outside interface
1878                        */
1879                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1880                           && (udp0->dst_port
1881                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1882                         {
1883                           vnet_feature_next
1884                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1885                              &next0, b0);
1886                           goto trace0;
1887                         }
1888
1889                       if (!sm->forwarding_enabled)
1890                         {
1891                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1892                           next0 = SNAT_OUT2IN_NEXT_DROP;
1893                           goto trace0;
1894                         }
1895                       else
1896                         {
1897                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1898                             {
1899                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1900                               goto trace0;
1901                             }
1902                           create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1903                           goto trace0;
1904                         }
1905                     }
1906
1907                   /* Create session initiated by host from external network */
1908                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1909                                                          thread_index);
1910                   if (!s0)
1911                     {
1912                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1913                       next0 = SNAT_OUT2IN_NEXT_DROP;
1914                       goto trace0;
1915                     }
1916                   reass0->sess_index = s0 - per_thread_data->sessions;
1917                   reass0->thread_index = thread_index;
1918                 }
1919               else
1920                 {
1921                   s0 = pool_elt_at_index (per_thread_data->sessions,
1922                                           value0.value);
1923                   reass0->sess_index = value0.value;
1924                 }
1925               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1926             }
1927           else
1928             {
1929               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1930                 {
1931                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1932                     {
1933                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1934                       next0 = SNAT_OUT2IN_NEXT_DROP;
1935                       goto trace0;
1936                     }
1937                   cached0 = 1;
1938                   goto trace0;
1939                 }
1940               s0 = pool_elt_at_index (per_thread_data->sessions,
1941                                       reass0->sess_index);
1942             }
1943
1944           old_addr0 = ip0->dst_address.as_u32;
1945           ip0->dst_address = s0->in2out.addr;
1946           new_addr0 = ip0->dst_address.as_u32;
1947           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1948
1949           sum0 = ip0->checksum;
1950           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1951                                  ip4_header_t,
1952                                  dst_address /* changed member */);
1953           ip0->checksum = ip_csum_fold (sum0);
1954
1955           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1956             {
1957               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1958                 {
1959                   old_port0 = tcp0->dst_port;
1960                   tcp0->dst_port = s0->in2out.port;
1961                   new_port0 = tcp0->dst_port;
1962
1963                   sum0 = tcp0->checksum;
1964                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1965                                          ip4_header_t,
1966                                          dst_address /* changed member */);
1967
1968                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1969                                          ip4_header_t /* cheat */,
1970                                          length /* changed member */);
1971                   tcp0->checksum = ip_csum_fold(sum0);
1972                   nat44_set_tcp_session_state (sm, s0, tcp0, thread_index);
1973                 }
1974               else
1975                 {
1976                   old_port0 = udp0->dst_port;
1977                   udp0->dst_port = s0->in2out.port;
1978                   udp0->checksum = 0;
1979                 }
1980             }
1981
1982           /* Accounting */
1983           s0->last_heard = now;
1984           s0->total_pkts++;
1985           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1986           /* Per-user LRU list maintenance */
1987           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1988                              s0->per_user_index);
1989           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1990                               s0->per_user_list_head_index,
1991                               s0->per_user_index);
1992
1993         trace0:
1994           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1995                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1996             {
1997               nat44_out2in_reass_trace_t *t =
1998                  vlib_add_trace (vm, node, b0, sizeof (*t));
1999               t->cached = cached0;
2000               t->sw_if_index = sw_if_index0;
2001               t->next_index = next0;
2002             }
2003
2004           if (cached0)
2005             {
2006               n_left_to_next++;
2007               to_next--;
2008             }
2009           else
2010             {
2011               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2012
2013               /* verify speculative enqueue, maybe switch current next frame */
2014               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2015                                                to_next, n_left_to_next,
2016                                                bi0, next0);
2017             }
2018
2019           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2020             {
2021               from = vlib_frame_vector_args (frame);
2022               u32 len = vec_len (fragments_to_loopback);
2023               if (len <= VLIB_FRAME_SIZE)
2024                 {
2025                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2026                   n_left_from = len;
2027                   vec_reset_length (fragments_to_loopback);
2028                 }
2029               else
2030                 {
2031                   clib_memcpy (from,
2032                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2033                                sizeof (u32) * VLIB_FRAME_SIZE);
2034                   n_left_from = VLIB_FRAME_SIZE;
2035                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2036                 }
2037             }
2038        }
2039
2040       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2041     }
2042
2043   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2044                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2045                                pkts_processed);
2046
2047   nat_send_all_to_node (vm, fragments_to_drop, node,
2048                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2049                         SNAT_OUT2IN_NEXT_DROP);
2050
2051   vec_free (fragments_to_drop);
2052   vec_free (fragments_to_loopback);
2053   return frame->n_vectors;
2054 }
2055
2056 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2057   .function = nat44_out2in_reass_node_fn,
2058   .name = "nat44-out2in-reass",
2059   .vector_size = sizeof (u32),
2060   .format_trace = format_nat44_out2in_reass_trace,
2061   .type = VLIB_NODE_TYPE_INTERNAL,
2062
2063   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2064   .error_strings = snat_out2in_error_strings,
2065
2066   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2067
2068   /* edit / add dispositions here */
2069   .next_nodes = {
2070     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2071     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2072     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2073     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2074     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2075   },
2076 };
2077 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2078                               nat44_out2in_reass_node_fn);
2079
2080 /**************************/
2081 /*** deterministic mode ***/
2082 /**************************/
2083 static uword
2084 snat_det_out2in_node_fn (vlib_main_t * vm,
2085                          vlib_node_runtime_t * node,
2086                          vlib_frame_t * frame)
2087 {
2088   u32 n_left_from, * from, * to_next;
2089   snat_out2in_next_t next_index;
2090   u32 pkts_processed = 0;
2091   snat_main_t * sm = &snat_main;
2092   u32 thread_index = vlib_get_thread_index ();
2093
2094   from = vlib_frame_vector_args (frame);
2095   n_left_from = frame->n_vectors;
2096   next_index = node->cached_next_index;
2097
2098   while (n_left_from > 0)
2099     {
2100       u32 n_left_to_next;
2101
2102       vlib_get_next_frame (vm, node, next_index,
2103                            to_next, n_left_to_next);
2104
2105       while (n_left_from >= 4 && n_left_to_next >= 2)
2106         {
2107           u32 bi0, bi1;
2108           vlib_buffer_t * b0, * b1;
2109           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2110           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2111           u32 sw_if_index0, sw_if_index1;
2112           ip4_header_t * ip0, * ip1;
2113           ip_csum_t sum0, sum1;
2114           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2115           u16 new_port0, old_port0, old_port1, new_port1;
2116           udp_header_t * udp0, * udp1;
2117           tcp_header_t * tcp0, * tcp1;
2118           u32 proto0, proto1;
2119           snat_det_out_key_t key0, key1;
2120           snat_det_map_t * dm0, * dm1;
2121           snat_det_session_t * ses0 = 0, * ses1 = 0;
2122           u32 rx_fib_index0, rx_fib_index1;
2123           icmp46_header_t * icmp0, * icmp1;
2124
2125           /* Prefetch next iteration. */
2126           {
2127             vlib_buffer_t * p2, * p3;
2128
2129             p2 = vlib_get_buffer (vm, from[2]);
2130             p3 = vlib_get_buffer (vm, from[3]);
2131
2132             vlib_prefetch_buffer_header (p2, LOAD);
2133             vlib_prefetch_buffer_header (p3, LOAD);
2134
2135             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2136             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2137           }
2138
2139           /* speculatively enqueue b0 and b1 to the current next frame */
2140           to_next[0] = bi0 = from[0];
2141           to_next[1] = bi1 = from[1];
2142           from += 2;
2143           to_next += 2;
2144           n_left_from -= 2;
2145           n_left_to_next -= 2;
2146
2147           b0 = vlib_get_buffer (vm, bi0);
2148           b1 = vlib_get_buffer (vm, bi1);
2149
2150           ip0 = vlib_buffer_get_current (b0);
2151           udp0 = ip4_next_header (ip0);
2152           tcp0 = (tcp_header_t *) udp0;
2153
2154           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2155
2156           if (PREDICT_FALSE(ip0->ttl == 1))
2157             {
2158               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2159               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2160                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2161                                            0);
2162               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2163               goto trace0;
2164             }
2165
2166           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2167
2168           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2169             {
2170               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2171               icmp0 = (icmp46_header_t *) udp0;
2172
2173               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2174                                   rx_fib_index0, node, next0, thread_index,
2175                                   &ses0, &dm0);
2176               goto trace0;
2177             }
2178
2179           key0.ext_host_addr = ip0->src_address;
2180           key0.ext_host_port = tcp0->src;
2181           key0.out_port = tcp0->dst;
2182
2183           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2184           if (PREDICT_FALSE(!dm0))
2185             {
2186               clib_warning("unknown dst address:  %U",
2187                            format_ip4_address, &ip0->dst_address);
2188               next0 = SNAT_OUT2IN_NEXT_DROP;
2189               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2190               goto trace0;
2191             }
2192
2193           snat_det_reverse(dm0, &ip0->dst_address,
2194                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2195
2196           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2197           if (PREDICT_FALSE(!ses0))
2198             {
2199               clib_warning("no match src %U:%d dst %U:%d for user %U",
2200                            format_ip4_address, &ip0->src_address,
2201                            clib_net_to_host_u16 (tcp0->src),
2202                            format_ip4_address, &ip0->dst_address,
2203                            clib_net_to_host_u16 (tcp0->dst),
2204                            format_ip4_address, &new_addr0);
2205               next0 = SNAT_OUT2IN_NEXT_DROP;
2206               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2207               goto trace0;
2208             }
2209           new_port0 = ses0->in_port;
2210
2211           old_addr0 = ip0->dst_address;
2212           ip0->dst_address = new_addr0;
2213           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2214
2215           sum0 = ip0->checksum;
2216           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2217                                  ip4_header_t,
2218                                  dst_address /* changed member */);
2219           ip0->checksum = ip_csum_fold (sum0);
2220
2221           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2222             {
2223               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2224                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2225               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2226                 snat_det_ses_close(dm0, ses0);
2227
2228               old_port0 = tcp0->dst;
2229               tcp0->dst = new_port0;
2230
2231               sum0 = tcp0->checksum;
2232               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2233                                      ip4_header_t,
2234                                      dst_address /* changed member */);
2235
2236               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2237                                      ip4_header_t /* cheat */,
2238                                      length /* changed member */);
2239               tcp0->checksum = ip_csum_fold(sum0);
2240             }
2241           else
2242             {
2243               old_port0 = udp0->dst_port;
2244               udp0->dst_port = new_port0;
2245               udp0->checksum = 0;
2246             }
2247
2248         trace0:
2249
2250           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2251                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2252             {
2253               snat_out2in_trace_t *t =
2254                  vlib_add_trace (vm, node, b0, sizeof (*t));
2255               t->sw_if_index = sw_if_index0;
2256               t->next_index = next0;
2257               t->session_index = ~0;
2258               if (ses0)
2259                 t->session_index = ses0 - dm0->sessions;
2260             }
2261
2262           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2263
2264           b1 = vlib_get_buffer (vm, bi1);
2265
2266           ip1 = vlib_buffer_get_current (b1);
2267           udp1 = ip4_next_header (ip1);
2268           tcp1 = (tcp_header_t *) udp1;
2269
2270           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2271
2272           if (PREDICT_FALSE(ip1->ttl == 1))
2273             {
2274               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2275               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2276                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2277                                            0);
2278               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2279               goto trace1;
2280             }
2281
2282           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2283
2284           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2285             {
2286               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2287               icmp1 = (icmp46_header_t *) udp1;
2288
2289               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2290                                   rx_fib_index1, node, next1, thread_index,
2291                                   &ses1, &dm1);
2292               goto trace1;
2293             }
2294
2295           key1.ext_host_addr = ip1->src_address;
2296           key1.ext_host_port = tcp1->src;
2297           key1.out_port = tcp1->dst;
2298
2299           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2300           if (PREDICT_FALSE(!dm1))
2301             {
2302               clib_warning("unknown dst address:  %U",
2303                            format_ip4_address, &ip1->dst_address);
2304               next1 = SNAT_OUT2IN_NEXT_DROP;
2305               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2306               goto trace1;
2307             }
2308
2309           snat_det_reverse(dm1, &ip1->dst_address,
2310                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2311
2312           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2313           if (PREDICT_FALSE(!ses1))
2314             {
2315               clib_warning("no match src %U:%d dst %U:%d for user %U",
2316                            format_ip4_address, &ip1->src_address,
2317                            clib_net_to_host_u16 (tcp1->src),
2318                            format_ip4_address, &ip1->dst_address,
2319                            clib_net_to_host_u16 (tcp1->dst),
2320                            format_ip4_address, &new_addr1);
2321               next1 = SNAT_OUT2IN_NEXT_DROP;
2322               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2323               goto trace1;
2324             }
2325           new_port1 = ses1->in_port;
2326
2327           old_addr1 = ip1->dst_address;
2328           ip1->dst_address = new_addr1;
2329           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2330
2331           sum1 = ip1->checksum;
2332           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2333                                  ip4_header_t,
2334                                  dst_address /* changed member */);
2335           ip1->checksum = ip_csum_fold (sum1);
2336
2337           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2338             {
2339               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2340                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2341               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2342                 snat_det_ses_close(dm1, ses1);
2343
2344               old_port1 = tcp1->dst;
2345               tcp1->dst = new_port1;
2346
2347               sum1 = tcp1->checksum;
2348               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2349                                      ip4_header_t,
2350                                      dst_address /* changed member */);
2351
2352               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2353                                      ip4_header_t /* cheat */,
2354                                      length /* changed member */);
2355               tcp1->checksum = ip_csum_fold(sum1);
2356             }
2357           else
2358             {
2359               old_port1 = udp1->dst_port;
2360               udp1->dst_port = new_port1;
2361               udp1->checksum = 0;
2362             }
2363
2364         trace1:
2365
2366           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2367                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2368             {
2369               snat_out2in_trace_t *t =
2370                  vlib_add_trace (vm, node, b1, sizeof (*t));
2371               t->sw_if_index = sw_if_index1;
2372               t->next_index = next1;
2373               t->session_index = ~0;
2374               if (ses1)
2375                 t->session_index = ses1 - dm1->sessions;
2376             }
2377
2378           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2379
2380           /* verify speculative enqueues, maybe switch current next frame */
2381           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2382                                            to_next, n_left_to_next,
2383                                            bi0, bi1, next0, next1);
2384          }
2385
2386       while (n_left_from > 0 && n_left_to_next > 0)
2387         {
2388           u32 bi0;
2389           vlib_buffer_t * b0;
2390           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2391           u32 sw_if_index0;
2392           ip4_header_t * ip0;
2393           ip_csum_t sum0;
2394           ip4_address_t new_addr0, old_addr0;
2395           u16 new_port0, old_port0;
2396           udp_header_t * udp0;
2397           tcp_header_t * tcp0;
2398           u32 proto0;
2399           snat_det_out_key_t key0;
2400           snat_det_map_t * dm0;
2401           snat_det_session_t * ses0 = 0;
2402           u32 rx_fib_index0;
2403           icmp46_header_t * icmp0;
2404
2405           /* speculatively enqueue b0 to the current next frame */
2406           bi0 = from[0];
2407           to_next[0] = bi0;
2408           from += 1;
2409           to_next += 1;
2410           n_left_from -= 1;
2411           n_left_to_next -= 1;
2412
2413           b0 = vlib_get_buffer (vm, bi0);
2414
2415           ip0 = vlib_buffer_get_current (b0);
2416           udp0 = ip4_next_header (ip0);
2417           tcp0 = (tcp_header_t *) udp0;
2418
2419           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2420
2421           if (PREDICT_FALSE(ip0->ttl == 1))
2422             {
2423               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2424               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2425                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2426                                            0);
2427               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2428               goto trace00;
2429             }
2430
2431           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2432
2433           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2434             {
2435               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2436               icmp0 = (icmp46_header_t *) udp0;
2437
2438               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2439                                   rx_fib_index0, node, next0, thread_index,
2440                                   &ses0, &dm0);
2441               goto trace00;
2442             }
2443
2444           key0.ext_host_addr = ip0->src_address;
2445           key0.ext_host_port = tcp0->src;
2446           key0.out_port = tcp0->dst;
2447
2448           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2449           if (PREDICT_FALSE(!dm0))
2450             {
2451               clib_warning("unknown dst address:  %U",
2452                            format_ip4_address, &ip0->dst_address);
2453               next0 = SNAT_OUT2IN_NEXT_DROP;
2454               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2455               goto trace00;
2456             }
2457
2458           snat_det_reverse(dm0, &ip0->dst_address,
2459                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2460
2461           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2462           if (PREDICT_FALSE(!ses0))
2463             {
2464               clib_warning("no match src %U:%d dst %U:%d for user %U",
2465                            format_ip4_address, &ip0->src_address,
2466                            clib_net_to_host_u16 (tcp0->src),
2467                            format_ip4_address, &ip0->dst_address,
2468                            clib_net_to_host_u16 (tcp0->dst),
2469                            format_ip4_address, &new_addr0);
2470               next0 = SNAT_OUT2IN_NEXT_DROP;
2471               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2472               goto trace00;
2473             }
2474           new_port0 = ses0->in_port;
2475
2476           old_addr0 = ip0->dst_address;
2477           ip0->dst_address = new_addr0;
2478           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2479
2480           sum0 = ip0->checksum;
2481           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2482                                  ip4_header_t,
2483                                  dst_address /* changed member */);
2484           ip0->checksum = ip_csum_fold (sum0);
2485
2486           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2487             {
2488               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2489                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2490               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2491                 snat_det_ses_close(dm0, ses0);
2492
2493               old_port0 = tcp0->dst;
2494               tcp0->dst = new_port0;
2495
2496               sum0 = tcp0->checksum;
2497               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2498                                      ip4_header_t,
2499                                      dst_address /* changed member */);
2500
2501               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2502                                      ip4_header_t /* cheat */,
2503                                      length /* changed member */);
2504               tcp0->checksum = ip_csum_fold(sum0);
2505             }
2506           else
2507             {
2508               old_port0 = udp0->dst_port;
2509               udp0->dst_port = new_port0;
2510               udp0->checksum = 0;
2511             }
2512
2513         trace00:
2514
2515           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2516                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2517             {
2518               snat_out2in_trace_t *t =
2519                  vlib_add_trace (vm, node, b0, sizeof (*t));
2520               t->sw_if_index = sw_if_index0;
2521               t->next_index = next0;
2522               t->session_index = ~0;
2523               if (ses0)
2524                 t->session_index = ses0 - dm0->sessions;
2525             }
2526
2527           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2528
2529           /* verify speculative enqueue, maybe switch current next frame */
2530           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2531                                            to_next, n_left_to_next,
2532                                            bi0, next0);
2533         }
2534
2535       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2536     }
2537
2538   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2539                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2540                                pkts_processed);
2541   return frame->n_vectors;
2542 }
2543
2544 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2545   .function = snat_det_out2in_node_fn,
2546   .name = "nat44-det-out2in",
2547   .vector_size = sizeof (u32),
2548   .format_trace = format_snat_out2in_trace,
2549   .type = VLIB_NODE_TYPE_INTERNAL,
2550
2551   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2552   .error_strings = snat_out2in_error_strings,
2553
2554   .runtime_data_bytes = sizeof (snat_runtime_t),
2555
2556   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2557
2558   /* edit / add dispositions here */
2559   .next_nodes = {
2560     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2561     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2562     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2563     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2564     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2565   },
2566 };
2567 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2568
2569 /**
2570  * Get address and port values to be used for ICMP packet translation
2571  * and create session if needed
2572  *
2573  * @param[in,out] sm             NAT main
2574  * @param[in,out] node           NAT node runtime
2575  * @param[in] thread_index       thread index
2576  * @param[in,out] b0             buffer containing packet to be translated
2577  * @param[out] p_proto           protocol used for matching
2578  * @param[out] p_value           address and port after NAT translation
2579  * @param[out] p_dont_translate  if packet should not be translated
2580  * @param d                      optional parameter
2581  * @param e                      optional parameter
2582  */
2583 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2584                           u32 thread_index, vlib_buffer_t *b0,
2585                           ip4_header_t *ip0, u8 *p_proto,
2586                           snat_session_key_t *p_value,
2587                           u8 *p_dont_translate, void *d, void *e)
2588 {
2589   icmp46_header_t *icmp0;
2590   u32 sw_if_index0;
2591   u8 protocol;
2592   snat_det_out_key_t key0;
2593   u8 dont_translate = 0;
2594   u32 next0 = ~0;
2595   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2596   ip4_header_t *inner_ip0;
2597   void *l4_header = 0;
2598   icmp46_header_t *inner_icmp0;
2599   snat_det_map_t * dm0 = 0;
2600   ip4_address_t new_addr0 = {{0}};
2601   snat_det_session_t * ses0 = 0;
2602   ip4_address_t out_addr;
2603
2604   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2605   echo0 = (icmp_echo_header_t *)(icmp0+1);
2606   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2607
2608   if (!icmp_is_error_message (icmp0))
2609     {
2610       protocol = SNAT_PROTOCOL_ICMP;
2611       key0.ext_host_addr = ip0->src_address;
2612       key0.ext_host_port = 0;
2613       key0.out_port = echo0->identifier;
2614       out_addr = ip0->dst_address;
2615     }
2616   else
2617     {
2618       inner_ip0 = (ip4_header_t *)(echo0+1);
2619       l4_header = ip4_next_header (inner_ip0);
2620       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2621       key0.ext_host_addr = inner_ip0->dst_address;
2622       out_addr = inner_ip0->src_address;
2623       switch (protocol)
2624         {
2625         case SNAT_PROTOCOL_ICMP:
2626           inner_icmp0 = (icmp46_header_t*)l4_header;
2627           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2628           key0.ext_host_port = 0;
2629           key0.out_port = inner_echo0->identifier;
2630           break;
2631         case SNAT_PROTOCOL_UDP:
2632         case SNAT_PROTOCOL_TCP:
2633           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2634           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2635           break;
2636         default:
2637           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2638           next0 = SNAT_OUT2IN_NEXT_DROP;
2639           goto out;
2640         }
2641     }
2642
2643   dm0 = snat_det_map_by_out(sm, &out_addr);
2644   if (PREDICT_FALSE(!dm0))
2645     {
2646       /* Don't NAT packet aimed at the intfc address */
2647       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2648                                           ip0->dst_address.as_u32)))
2649         {
2650           dont_translate = 1;
2651           goto out;
2652         }
2653       clib_warning("unknown dst address:  %U",
2654                    format_ip4_address, &ip0->dst_address);
2655       goto out;
2656     }
2657
2658   snat_det_reverse(dm0, &ip0->dst_address,
2659                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2660
2661   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2662   if (PREDICT_FALSE(!ses0))
2663     {
2664       /* Don't NAT packet aimed at the intfc address */
2665       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2666                                           ip0->dst_address.as_u32)))
2667         {
2668           dont_translate = 1;
2669           goto out;
2670         }
2671       clib_warning("no match src %U:%d dst %U:%d for user %U",
2672                    format_ip4_address, &key0.ext_host_addr,
2673                    clib_net_to_host_u16 (key0.ext_host_port),
2674                    format_ip4_address, &out_addr,
2675                    clib_net_to_host_u16 (key0.out_port),
2676                    format_ip4_address, &new_addr0);
2677       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2678       next0 = SNAT_OUT2IN_NEXT_DROP;
2679       goto out;
2680     }
2681
2682   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2683                     !icmp_is_error_message (icmp0)))
2684     {
2685       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2686       next0 = SNAT_OUT2IN_NEXT_DROP;
2687       goto out;
2688     }
2689
2690   goto out;
2691
2692 out:
2693   *p_proto = protocol;
2694   if (ses0)
2695     {
2696       p_value->addr = new_addr0;
2697       p_value->fib_index = sm->inside_fib_index;
2698       p_value->port = ses0->in_port;
2699     }
2700   *p_dont_translate = dont_translate;
2701   if (d)
2702     *(snat_det_session_t**)d = ses0;
2703   if (e)
2704     *(snat_det_map_t**)e = dm0;
2705   return next0;
2706 }
2707
2708 /**********************/
2709 /*** worker handoff ***/
2710 /**********************/
2711 static uword
2712 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2713                                vlib_node_runtime_t * node,
2714                                vlib_frame_t * frame)
2715 {
2716   snat_main_t *sm = &snat_main;
2717   vlib_thread_main_t *tm = vlib_get_thread_main ();
2718   u32 n_left_from, *from, *to_next = 0;
2719   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2720   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2721     = 0;
2722   vlib_frame_queue_elt_t *hf = 0;
2723   vlib_frame_t *f = 0;
2724   int i;
2725   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2726   u32 next_worker_index = 0;
2727   u32 current_worker_index = ~0;
2728   u32 thread_index = vlib_get_thread_index ();
2729
2730   ASSERT (vec_len (sm->workers));
2731
2732   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2733     {
2734       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2735
2736       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2737                                sm->first_worker_index + sm->num_workers - 1,
2738                                (vlib_frame_queue_t *) (~0));
2739     }
2740
2741   from = vlib_frame_vector_args (frame);
2742   n_left_from = frame->n_vectors;
2743
2744   while (n_left_from > 0)
2745     {
2746       u32 bi0;
2747       vlib_buffer_t *b0;
2748       u32 sw_if_index0;
2749       u32 rx_fib_index0;
2750       ip4_header_t * ip0;
2751       u8 do_handoff;
2752
2753       bi0 = from[0];
2754       from += 1;
2755       n_left_from -= 1;
2756
2757       b0 = vlib_get_buffer (vm, bi0);
2758
2759       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2760       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2761
2762       ip0 = vlib_buffer_get_current (b0);
2763
2764       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2765
2766       if (PREDICT_FALSE (next_worker_index != thread_index))
2767         {
2768           do_handoff = 1;
2769
2770           if (next_worker_index != current_worker_index)
2771             {
2772               if (hf)
2773                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2774
2775               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2776                                                       next_worker_index,
2777                                                       handoff_queue_elt_by_worker_index);
2778
2779               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2780               to_next_worker = &hf->buffer_index[hf->n_vectors];
2781               current_worker_index = next_worker_index;
2782             }
2783
2784           /* enqueue to correct worker thread */
2785           to_next_worker[0] = bi0;
2786           to_next_worker++;
2787           n_left_to_next_worker--;
2788
2789           if (n_left_to_next_worker == 0)
2790             {
2791               hf->n_vectors = VLIB_FRAME_SIZE;
2792               vlib_put_frame_queue_elt (hf);
2793               current_worker_index = ~0;
2794               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2795               hf = 0;
2796             }
2797         }
2798       else
2799         {
2800           do_handoff = 0;
2801           /* if this is 1st frame */
2802           if (!f)
2803             {
2804               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2805               to_next = vlib_frame_vector_args (f);
2806             }
2807
2808           to_next[0] = bi0;
2809           to_next += 1;
2810           f->n_vectors++;
2811         }
2812
2813       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2814                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2815         {
2816           snat_out2in_worker_handoff_trace_t *t =
2817             vlib_add_trace (vm, node, b0, sizeof (*t));
2818           t->next_worker_index = next_worker_index;
2819           t->do_handoff = do_handoff;
2820         }
2821     }
2822
2823   if (f)
2824     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2825
2826   if (hf)
2827     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2828
2829   /* Ship frames to the worker nodes */
2830   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2831     {
2832       if (handoff_queue_elt_by_worker_index[i])
2833         {
2834           hf = handoff_queue_elt_by_worker_index[i];
2835           /*
2836            * It works better to let the handoff node
2837            * rate-adapt, always ship the handoff queue element.
2838            */
2839           if (1 || hf->n_vectors == hf->last_n_vectors)
2840             {
2841               vlib_put_frame_queue_elt (hf);
2842               handoff_queue_elt_by_worker_index[i] = 0;
2843             }
2844           else
2845             hf->last_n_vectors = hf->n_vectors;
2846         }
2847       congested_handoff_queue_by_worker_index[i] =
2848         (vlib_frame_queue_t *) (~0);
2849     }
2850   hf = 0;
2851   current_worker_index = ~0;
2852   return frame->n_vectors;
2853 }
2854
2855 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2856   .function = snat_out2in_worker_handoff_fn,
2857   .name = "nat44-out2in-worker-handoff",
2858   .vector_size = sizeof (u32),
2859   .format_trace = format_snat_out2in_worker_handoff_trace,
2860   .type = VLIB_NODE_TYPE_INTERNAL,
2861
2862   .n_next_nodes = 1,
2863
2864   .next_nodes = {
2865     [0] = "error-drop",
2866   },
2867 };
2868
2869 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2870
2871 static uword
2872 snat_out2in_fast_node_fn (vlib_main_t * vm,
2873                           vlib_node_runtime_t * node,
2874                           vlib_frame_t * frame)
2875 {
2876   u32 n_left_from, * from, * to_next;
2877   snat_out2in_next_t next_index;
2878   u32 pkts_processed = 0;
2879   snat_main_t * sm = &snat_main;
2880
2881   from = vlib_frame_vector_args (frame);
2882   n_left_from = frame->n_vectors;
2883   next_index = node->cached_next_index;
2884
2885   while (n_left_from > 0)
2886     {
2887       u32 n_left_to_next;
2888
2889       vlib_get_next_frame (vm, node, next_index,
2890                            to_next, n_left_to_next);
2891
2892       while (n_left_from > 0 && n_left_to_next > 0)
2893         {
2894           u32 bi0;
2895           vlib_buffer_t * b0;
2896           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2897           u32 sw_if_index0;
2898           ip4_header_t * ip0;
2899           ip_csum_t sum0;
2900           u32 new_addr0, old_addr0;
2901           u16 new_port0, old_port0;
2902           udp_header_t * udp0;
2903           tcp_header_t * tcp0;
2904           icmp46_header_t * icmp0;
2905           snat_session_key_t key0, sm0;
2906           u32 proto0;
2907           u32 rx_fib_index0;
2908
2909           /* speculatively enqueue b0 to the current next frame */
2910           bi0 = from[0];
2911           to_next[0] = bi0;
2912           from += 1;
2913           to_next += 1;
2914           n_left_from -= 1;
2915           n_left_to_next -= 1;
2916
2917           b0 = vlib_get_buffer (vm, bi0);
2918
2919           ip0 = vlib_buffer_get_current (b0);
2920           udp0 = ip4_next_header (ip0);
2921           tcp0 = (tcp_header_t *) udp0;
2922           icmp0 = (icmp46_header_t *) udp0;
2923
2924           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2925           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2926
2927           vnet_feature_next (sw_if_index0, &next0, b0);
2928
2929           if (PREDICT_FALSE(ip0->ttl == 1))
2930             {
2931               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2932               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2933                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2934                                            0);
2935               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2936               goto trace00;
2937             }
2938
2939           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2940
2941           if (PREDICT_FALSE (proto0 == ~0))
2942               goto trace00;
2943
2944           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2945             {
2946               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2947                                   rx_fib_index0, node, next0, ~0, 0, 0);
2948               goto trace00;
2949             }
2950
2951           key0.addr = ip0->dst_address;
2952           key0.port = udp0->dst_port;
2953           key0.fib_index = rx_fib_index0;
2954
2955           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2956             {
2957               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2958               goto trace00;
2959             }
2960
2961           new_addr0 = sm0.addr.as_u32;
2962           new_port0 = sm0.port;
2963           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2964           old_addr0 = ip0->dst_address.as_u32;
2965           ip0->dst_address.as_u32 = new_addr0;
2966
2967           sum0 = ip0->checksum;
2968           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2969                                  ip4_header_t,
2970                                  dst_address /* changed member */);
2971           ip0->checksum = ip_csum_fold (sum0);
2972
2973           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2974             {
2975                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2976                 {
2977                   old_port0 = tcp0->dst_port;
2978                   tcp0->dst_port = new_port0;
2979
2980                   sum0 = tcp0->checksum;
2981                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2982                                          ip4_header_t,
2983                                          dst_address /* changed member */);
2984
2985                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2986                                          ip4_header_t /* cheat */,
2987                                          length /* changed member */);
2988                   tcp0->checksum = ip_csum_fold(sum0);
2989                 }
2990               else
2991                 {
2992                   old_port0 = udp0->dst_port;
2993                   udp0->dst_port = new_port0;
2994                   udp0->checksum = 0;
2995                 }
2996             }
2997           else
2998             {
2999               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3000                 {
3001                   sum0 = tcp0->checksum;
3002                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3003                                          ip4_header_t,
3004                                          dst_address /* changed member */);
3005
3006                   tcp0->checksum = ip_csum_fold(sum0);
3007                 }
3008             }
3009
3010         trace00:
3011
3012           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3013                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3014             {
3015               snat_out2in_trace_t *t =
3016                  vlib_add_trace (vm, node, b0, sizeof (*t));
3017               t->sw_if_index = sw_if_index0;
3018               t->next_index = next0;
3019             }
3020
3021           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3022
3023           /* verify speculative enqueue, maybe switch current next frame */
3024           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3025                                            to_next, n_left_to_next,
3026                                            bi0, next0);
3027         }
3028
3029       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3030     }
3031
3032   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3033                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3034                                pkts_processed);
3035   return frame->n_vectors;
3036 }
3037
3038 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3039   .function = snat_out2in_fast_node_fn,
3040   .name = "nat44-out2in-fast",
3041   .vector_size = sizeof (u32),
3042   .format_trace = format_snat_out2in_fast_trace,
3043   .type = VLIB_NODE_TYPE_INTERNAL,
3044
3045   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3046   .error_strings = snat_out2in_error_strings,
3047
3048   .runtime_data_bytes = sizeof (snat_runtime_t),
3049
3050   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3051
3052   /* edit / add dispositions here */
3053   .next_nodes = {
3054     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3055     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3056     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3057     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3058     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3059   },
3060 };
3061 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);