src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29 #include <nat/nat_inlines.h>
  30
  31 #include <vppinfra/hash.h>
  32 #include <vppinfra/error.h>
  33 #include <vppinfra/elog.h>
  34
  35 typedef struct {
  36   u32 sw_if_index;
  37   u32 next_index;
  38   u32 session_index;
  39 } snat_out2in_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_out2in_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  52
  53   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  54               t->sw_if_index, t->next_index, t->session_index);
  55   return s;
  56 }
  57
  58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  63
  64   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  65               t->sw_if_index, t->next_index);
  66   return s;
  67 }
  68
  69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  70 {
  71   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  72   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  73   snat_out2in_worker_handoff_trace_t * t =
  74     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  75   char * m;
  76
  77   m = t->do_handoff ? "next worker" : "same worker";
  78   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  79
  80   return s;
  81 }
  82
  83 typedef struct {
  84   u32 sw_if_index;
  85   u32 next_index;
  86   u8 cached;
  87 } nat44_out2in_reass_trace_t;
  88
  89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  90 {
  91   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  92   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  93   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  94
  95   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  96               t->sw_if_index, t->next_index,
  97               t->cached ? "cached" : "translated");
  98
  99   return s;
 100 }
 101
 102 vlib_node_registration_t snat_out2in_node;
 103 vlib_node_registration_t snat_out2in_fast_node;
 104 vlib_node_registration_t snat_out2in_worker_handoff_node;
 105 vlib_node_registration_t snat_det_out2in_node;
 106 vlib_node_registration_t nat44_out2in_reass_node;
 107
 108 #define foreach_snat_out2in_error                       \
 109 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 110 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 111 _(OUT_OF_PORTS, "Out of ports")                         \
 112 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 113 _(NO_TRANSLATION, "No translation")                     \
 114 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 115 _(DROP_FRAGMENT, "Drop fragment")                       \
 116 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 117 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 118 _(FQ_CONGESTED, "Handoff frame queue congested")
 119
 120 typedef enum {
 121 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 122   foreach_snat_out2in_error
 123 #undef _
 124   SNAT_OUT2IN_N_ERROR,
 125 } snat_out2in_error_t;
 126
 127 static char * snat_out2in_error_strings[] = {
 128 #define _(sym,string) string,
 129   foreach_snat_out2in_error
 130 #undef _
 131 };
 132
 133 typedef enum {
 134   SNAT_OUT2IN_NEXT_DROP,
 135   SNAT_OUT2IN_NEXT_LOOKUP,
 136   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 137   SNAT_OUT2IN_NEXT_REASS,
 138   SNAT_OUT2IN_NEXT_IN2OUT,
 139   SNAT_OUT2IN_N_NEXT,
 140 } snat_out2in_next_t;
 141
 142 /**
 143  * @brief Create session for static mapping.
 144  *
 145  * Create NAT session initiated by host from external network with static
 146  * mapping.
 147  *
 148  * @param sm     NAT main.
 149  * @param b0     Vlib buffer.
 150  * @param in2out In2out NAT44 session key.
 151  * @param out2in Out2in NAT44 session key.
 152  * @param node   Vlib node.
 153  *
 154  * @returns SNAT session if successfully created otherwise 0.
 155  */
 156 static inline snat_session_t *
 157 create_session_for_static_mapping (snat_main_t *sm,
 158                                    vlib_buffer_t *b0,
 159                                    snat_session_key_t in2out,
 160                                    snat_session_key_t out2in,
 161                                    vlib_node_runtime_t * node,
 162                                    u32 thread_index)
 163 {
 164   snat_user_t *u;
 165   snat_session_t *s;
 166   clib_bihash_kv_8_8_t kv0;
 167   ip4_header_t *ip0;
 168   udp_header_t *udp0;
 169
 170   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 171     {
 172       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 173       nat_log_notice ("maximum sessions exceeded");
 174       return 0;
 175     }
 176
 177   ip0 = vlib_buffer_get_current (b0);
 178   udp0 = ip4_next_header (ip0);
 179
 180   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 181   if (!u)
 182     {
 183       nat_log_warn ("create NAT user failed");
 184       return 0;
 185     }
 186
 187   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 188   if (!s)
 189     {
 190       nat_log_warn ("create NAT session failed");
 191       return 0;
 192     }
 193
 194   s->outside_address_index = ~0;
 195   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 196   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 197   s->ext_host_port = udp0->src_port;
 198   user_session_increment (sm, u, 1 /* static */);
 199   s->in2out = in2out;
 200   s->out2in = out2in;
 201   s->in2out.protocol = out2in.protocol;
 202
 203   /* Add to translation hashes */
 204   kv0.key = s->in2out.as_u64;
 205   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 206   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 207                                1 /* is_add */))
 208       nat_log_notice ("in2out key add failed");
 209
 210   kv0.key = s->out2in.as_u64;
 211
 212   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 213                                1 /* is_add */))
 214       nat_log_notice ("out2in key add failed");
 215
 216   /* log NAT event */
 217   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 218                                       s->out2in.addr.as_u32,
 219                                       s->in2out.protocol,
 220                                       s->in2out.port,
 221                                       s->out2in.port,
 222                                       s->in2out.fib_index);
 223    return s;
 224 }
 225
 226 static_always_inline
 227 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 228                                  snat_session_key_t *p_key0)
 229 {
 230   icmp46_header_t *icmp0;
 231   snat_session_key_t key0;
 232   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 233   ip4_header_t *inner_ip0;
 234   void *l4_header = 0;
 235   icmp46_header_t *inner_icmp0;
 236
 237   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 238   echo0 = (icmp_echo_header_t *)(icmp0+1);
 239
 240   if (!icmp_is_error_message (icmp0))
 241     {
 242       key0.protocol = SNAT_PROTOCOL_ICMP;
 243       key0.addr = ip0->dst_address;
 244       key0.port = echo0->identifier;
 245     }
 246   else
 247     {
 248       inner_ip0 = (ip4_header_t *)(echo0+1);
 249       l4_header = ip4_next_header (inner_ip0);
 250       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 251       key0.addr = inner_ip0->src_address;
 252       switch (key0.protocol)
 253         {
 254         case SNAT_PROTOCOL_ICMP:
 255           inner_icmp0 = (icmp46_header_t*)l4_header;
 256           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 257           key0.port = inner_echo0->identifier;
 258           break;
 259         case SNAT_PROTOCOL_UDP:
 260         case SNAT_PROTOCOL_TCP:
 261           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 262           break;
 263         default:
 264           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 265         }
 266     }
 267   *p_key0 = key0;
 268   return -1; /* success */
 269 }
 270
 271 static_always_inline int
 272 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 273 {
 274   icmp46_header_t *icmp0;
 275   nat_ed_ses_key_t key0;
 276   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 277   ip4_header_t *inner_ip0;
 278   void *l4_header = 0;
 279   icmp46_header_t *inner_icmp0;
 280
 281   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 282   echo0 = (icmp_echo_header_t *)(icmp0+1);
 283
 284   if (!icmp_is_error_message (icmp0))
 285     {
 286       key0.proto = IP_PROTOCOL_ICMP;
 287       key0.l_addr = ip0->dst_address;
 288       key0.r_addr = ip0->src_address;
 289       key0.l_port = key0.r_port = echo0->identifier;
 290     }
 291   else
 292     {
 293       inner_ip0 = (ip4_header_t *)(echo0+1);
 294       l4_header = ip4_next_header (inner_ip0);
 295       key0.proto = inner_ip0->protocol;
 296       key0.l_addr = inner_ip0->src_address;
 297       key0.r_addr = inner_ip0->dst_address;
 298       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 299         {
 300         case SNAT_PROTOCOL_ICMP:
 301           inner_icmp0 = (icmp46_header_t*)l4_header;
 302           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 303           key0.l_port = key0.r_port = inner_echo0->identifier;
 304           break;
 305         case SNAT_PROTOCOL_UDP:
 306         case SNAT_PROTOCOL_TCP:
 307           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
 308           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 309           break;
 310         default:
 311           return -1;
 312         }
 313     }
 314   *p_key0 = key0;
 315   return 0;
 316 }
 317
 318 static int
 319 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
 320               u32 thread_index)
 321 {
 322   snat_session_key_t key;
 323   clib_bihash_kv_8_8_t kv, value;
 324
 325   key.addr = ip->src_address;
 326   key.port = src_port;
 327   key.protocol = proto;
 328   key.fib_index = sm->inside_fib_index;
 329   kv.key = key.as_u64;
 330
 331   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
 332                                &value))
 333     return 1;
 334
 335   return 0;
 336 }
 337
 338 static void
 339 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 340                       u32 thread_index)
 341 {
 342   nat_ed_ses_key_t key;
 343   clib_bihash_kv_16_8_t kv, value;
 344   udp_header_t *udp;
 345   snat_user_t *u;
 346   snat_session_t *s = 0;
 347   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 348   f64 now = vlib_time_now (sm->vlib_main);
 349
 350   if (ip->protocol == IP_PROTOCOL_ICMP)
 351     {
 352       if (icmp_get_ed_key (ip, &key))
 353         return;
 354     }
 355   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 356     {
 357       udp = ip4_next_header(ip);
 358       key.r_addr = ip->src_address;
 359       key.l_addr = ip->dst_address;
 360       key.proto = ip->protocol;
 361       key.l_port = udp->dst_port;
 362       key.r_port = udp->src_port;
 363     }
 364   else
 365     {
 366       key.r_addr = ip->src_address;
 367       key.l_addr = ip->dst_address;
 368       key.proto = ip->protocol;
 369       key.l_port = key.r_port = 0;
 370     }
 371   key.fib_index = 0;
 372   kv.key[0] = key.as_u64[0];
 373   kv.key[1] = key.as_u64[1];
 374
 375   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 376     {
 377       s = pool_elt_at_index (tsm->sessions, value.value);
 378     }
 379   else
 380     {
 381       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 382         return;
 383
 384       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
 385       if (!u)
 386         {
 387           nat_log_warn ("create NAT user failed");
 388           return;
 389         }
 390
 391       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 392       if (!s)
 393         {
 394           nat_log_warn ("create NAT session failed");
 395           return;
 396         }
 397
 398       s->ext_host_addr = key.r_addr;
 399       s->ext_host_port = key.r_port;
 400       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 401       s->outside_address_index = ~0;
 402       s->out2in.addr = key.l_addr;
 403       s->out2in.port = key.l_port;
 404       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 405       s->out2in.fib_index = 0;
 406       s->in2out = s->out2in;
 407       user_session_increment (sm, u, 0);
 408
 409       kv.value = s - tsm->sessions;
 410       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
 411         nat_log_notice ("in2out_ed key add failed");
 412     }
 413
 414   if (ip->protocol == IP_PROTOCOL_TCP)
 415     {
 416       tcp_header_t *tcp = ip4_next_header(ip);
 417       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
 418         return;
 419     }
 420   /* Per-user LRU list maintenance */
 421   nat44_session_update_lru (sm, s, thread_index);
 422   /* Accounting */
 423   nat44_session_update_counters (s, now, 0);
 424 }
 425
 426 /**
 427  * Get address and port values to be used for ICMP packet translation
 428  * and create session if needed
 429  *
 430  * @param[in,out] sm             NAT main
 431  * @param[in,out] node           NAT node runtime
 432  * @param[in] thread_index       thread index
 433  * @param[in,out] b0             buffer containing packet to be translated
 434  * @param[out] p_proto           protocol used for matching
 435  * @param[out] p_value           address and port after NAT translation
 436  * @param[out] p_dont_translate  if packet should not be translated
 437  * @param d                      optional parameter
 438  * @param e                      optional parameter
 439  */
 440 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 441                            u32 thread_index, vlib_buffer_t *b0,
 442                            ip4_header_t *ip0, u8 *p_proto,
 443                            snat_session_key_t *p_value,
 444                            u8 *p_dont_translate, void *d, void *e)
 445 {
 446   icmp46_header_t *icmp0;
 447   u32 sw_if_index0;
 448   u32 rx_fib_index0;
 449   snat_session_key_t key0;
 450   snat_session_key_t sm0;
 451   snat_session_t *s0 = 0;
 452   u8 dont_translate = 0;
 453   clib_bihash_kv_8_8_t kv0, value0;
 454   u8 is_addr_only;
 455   u32 next0 = ~0;
 456   int err;
 457
 458   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 459   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 460   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 461
 462   key0.protocol = 0;
 463
 464   err = icmp_get_key (ip0, &key0);
 465   if (err != -1)
 466     {
 467       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 468       next0 = SNAT_OUT2IN_NEXT_DROP;
 469       goto out;
 470     }
 471   key0.fib_index = rx_fib_index0;
 472
 473   kv0.key = key0.as_u64;
 474
 475   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 476                               &value0))
 477     {
 478       /* Try to match static mapping by external address and port,
 479          destination address and port in packet */
 480       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 481         {
 482           if (!sm->forwarding_enabled)
 483             {
 484               /* Don't NAT packet aimed at the intfc address */
 485               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 486                                                   ip0->dst_address.as_u32)))
 487                 {
 488                   dont_translate = 1;
 489                   goto out;
 490                 }
 491               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 492               next0 = SNAT_OUT2IN_NEXT_DROP;
 493               goto out;
 494             }
 495           else
 496             {
 497               dont_translate = 1;
 498               if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
 499                 {
 500                   next0 = SNAT_OUT2IN_NEXT_IN2OUT;
 501                   goto out;
 502                 }
 503               create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
 504               goto out;
 505             }
 506         }
 507
 508       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 509                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 510         {
 511           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 512           next0 = SNAT_OUT2IN_NEXT_DROP;
 513           goto out;
 514         }
 515
 516       /* Create session initiated by host from external network */
 517       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 518                                              node, thread_index);
 519
 520       if (!s0)
 521         {
 522           next0 = SNAT_OUT2IN_NEXT_DROP;
 523           goto out;
 524         }
 525     }
 526   else
 527     {
 528       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 529                         icmp0->type != ICMP4_echo_request &&
 530                         !icmp_is_error_message (icmp0)))
 531         {
 532           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 533           next0 = SNAT_OUT2IN_NEXT_DROP;
 534           goto out;
 535         }
 536
 537       if (PREDICT_FALSE (value0.value == ~0ULL))
 538         {
 539           nat_ed_ses_key_t key;
 540           clib_bihash_kv_16_8_t s_kv, s_value;
 541
 542           key.as_u64[0] = 0;
 543           key.as_u64[1] = 0;
 544           if (icmp_get_ed_key (ip0, &key))
 545             {
 546               b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 547               next0 = SNAT_OUT2IN_NEXT_DROP;
 548               goto out;
 549             }
 550           key.fib_index = rx_fib_index0;
 551           s_kv.key[0] = key.as_u64[0];
 552           s_kv.key[1] = key.as_u64[1];
 553           if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 554             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 555                                     s_value.value);
 556           else
 557            {
 558               next0 = SNAT_OUT2IN_NEXT_DROP;
 559               goto out;
 560            }
 561         }
 562       else
 563         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 564                                 value0.value);
 565     }
 566
 567 out:
 568   *p_proto = key0.protocol;
 569   if (s0)
 570     *p_value = s0->in2out;
 571   *p_dont_translate = dont_translate;
 572   if (d)
 573     *(snat_session_t**)d = s0;
 574   return next0;
 575 }
 576
 577 /**
 578  * Get address and port values to be used for ICMP packet translation
 579  *
 580  * @param[in] sm                 NAT main
 581  * @param[in,out] node           NAT node runtime
 582  * @param[in] thread_index       thread index
 583  * @param[in,out] b0             buffer containing packet to be translated
 584  * @param[out] p_proto           protocol used for matching
 585  * @param[out] p_value           address and port after NAT translation
 586  * @param[out] p_dont_translate  if packet should not be translated
 587  * @param d                      optional parameter
 588  * @param e                      optional parameter
 589  */
 590 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 591                            u32 thread_index, vlib_buffer_t *b0,
 592                            ip4_header_t *ip0, u8 *p_proto,
 593                            snat_session_key_t *p_value,
 594                            u8 *p_dont_translate, void *d, void *e)
 595 {
 596   icmp46_header_t *icmp0;
 597   u32 sw_if_index0;
 598   u32 rx_fib_index0;
 599   snat_session_key_t key0;
 600   snat_session_key_t sm0;
 601   u8 dont_translate = 0;
 602   u8 is_addr_only;
 603   u32 next0 = ~0;
 604   int err;
 605
 606   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 607   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 608   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 609
 610   err = icmp_get_key (ip0, &key0);
 611   if (err != -1)
 612     {
 613       b0->error = node->errors[err];
 614       next0 = SNAT_OUT2IN_NEXT_DROP;
 615       goto out2;
 616     }
 617   key0.fib_index = rx_fib_index0;
 618
 619   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 620     {
 621       /* Don't NAT packet aimed at the intfc address */
 622       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 623         {
 624           dont_translate = 1;
 625           goto out;
 626         }
 627       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 628       next0 = SNAT_OUT2IN_NEXT_DROP;
 629       goto out;
 630     }
 631
 632   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 633                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 634                     !icmp_is_error_message (icmp0)))
 635     {
 636       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 637       next0 = SNAT_OUT2IN_NEXT_DROP;
 638       goto out;
 639     }
 640
 641 out:
 642   *p_value = sm0;
 643 out2:
 644   *p_proto = key0.protocol;
 645   *p_dont_translate = dont_translate;
 646   return next0;
 647 }
 648
 649 static inline u32 icmp_out2in (snat_main_t *sm,
 650                                vlib_buffer_t * b0,
 651                                ip4_header_t * ip0,
 652                                icmp46_header_t * icmp0,
 653                                u32 sw_if_index0,
 654                                u32 rx_fib_index0,
 655                                vlib_node_runtime_t * node,
 656                                u32 next0,
 657                                u32 thread_index,
 658                                void *d,
 659                                void *e)
 660 {
 661   snat_session_key_t sm0;
 662   u8 protocol;
 663   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 664   ip4_header_t *inner_ip0 = 0;
 665   void *l4_header = 0;
 666   icmp46_header_t *inner_icmp0;
 667   u8 dont_translate;
 668   u32 new_addr0, old_addr0;
 669   u16 old_id0, new_id0;
 670   ip_csum_t sum0;
 671   u16 checksum0;
 672   u32 next0_tmp;
 673
 674   echo0 = (icmp_echo_header_t *)(icmp0+1);
 675
 676   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 677                                        &protocol, &sm0, &dont_translate, d, e);
 678   if (next0_tmp != ~0)
 679     next0 = next0_tmp;
 680   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 681     goto out;
 682
 683   sum0 = ip_incremental_checksum (0, icmp0,
 684                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 685   checksum0 = ~ip_csum_fold (sum0);
 686   if (checksum0 != 0 && checksum0 != 0xffff)
 687     {
 688       next0 = SNAT_OUT2IN_NEXT_DROP;
 689       goto out;
 690     }
 691
 692   old_addr0 = ip0->dst_address.as_u32;
 693   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 694   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 695
 696   sum0 = ip0->checksum;
 697   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 698                          dst_address /* changed member */);
 699   ip0->checksum = ip_csum_fold (sum0);
 700
 701   if (icmp0->checksum == 0)
 702     icmp0->checksum = 0xffff;
 703
 704   if (!icmp_is_error_message (icmp0))
 705     {
 706       new_id0 = sm0.port;
 707       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 708         {
 709           old_id0 = echo0->identifier;
 710           new_id0 = sm0.port;
 711           echo0->identifier = new_id0;
 712
 713           sum0 = icmp0->checksum;
 714           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 715                                  identifier /* changed member */);
 716           icmp0->checksum = ip_csum_fold (sum0);
 717         }
 718     }
 719   else
 720     {
 721       inner_ip0 = (ip4_header_t *)(echo0+1);
 722       l4_header = ip4_next_header (inner_ip0);
 723
 724       if (!ip4_header_checksum_is_valid (inner_ip0))
 725         {
 726           next0 = SNAT_OUT2IN_NEXT_DROP;
 727           goto out;
 728         }
 729
 730       old_addr0 = inner_ip0->src_address.as_u32;
 731       inner_ip0->src_address = sm0.addr;
 732       new_addr0 = inner_ip0->src_address.as_u32;
 733
 734       sum0 = icmp0->checksum;
 735       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 736                              src_address /* changed member */);
 737       icmp0->checksum = ip_csum_fold (sum0);
 738
 739       switch (protocol)
 740         {
 741         case SNAT_PROTOCOL_ICMP:
 742           inner_icmp0 = (icmp46_header_t*)l4_header;
 743           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 744
 745           old_id0 = inner_echo0->identifier;
 746           new_id0 = sm0.port;
 747           inner_echo0->identifier = new_id0;
 748
 749           sum0 = icmp0->checksum;
 750           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 751                                  identifier);
 752           icmp0->checksum = ip_csum_fold (sum0);
 753           break;
 754         case SNAT_PROTOCOL_UDP:
 755         case SNAT_PROTOCOL_TCP:
 756           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 757           new_id0 = sm0.port;
 758           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 759
 760           sum0 = icmp0->checksum;
 761           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 762                                  src_port);
 763           icmp0->checksum = ip_csum_fold (sum0);
 764           break;
 765         default:
 766           ASSERT(0);
 767         }
 768     }
 769
 770 out:
 771   return next0;
 772 }
 773
 774
 775 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 776                                          vlib_buffer_t * b0,
 777                                          ip4_header_t * ip0,
 778                                          icmp46_header_t * icmp0,
 779                                          u32 sw_if_index0,
 780                                          u32 rx_fib_index0,
 781                                          vlib_node_runtime_t * node,
 782                                          u32 next0, f64 now,
 783                                          u32 thread_index,
 784                                          snat_session_t ** p_s0)
 785 {
 786   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 787                       next0, thread_index, p_s0, 0);
 788   snat_session_t * s0 = *p_s0;
 789   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 790     {
 791       /* Accounting */
 792       nat44_session_update_counters (s0, now,
 793                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
 794       /* Per-user LRU list maintenance */
 795       nat44_session_update_lru (sm, s0, thread_index);
 796     }
 797   return next0;
 798 }
 799
 800 static snat_session_t *
 801 snat_out2in_unknown_proto (snat_main_t *sm,
 802                            vlib_buffer_t * b,
 803                            ip4_header_t * ip,
 804                            u32 rx_fib_index,
 805                            u32 thread_index,
 806                            f64 now,
 807                            vlib_main_t * vm,
 808                            vlib_node_runtime_t * node)
 809 {
 810   clib_bihash_kv_8_8_t kv, value;
 811   clib_bihash_kv_16_8_t s_kv, s_value;
 812   snat_static_mapping_t *m;
 813   snat_session_key_t m_key;
 814   u32 old_addr, new_addr;
 815   ip_csum_t sum;
 816   nat_ed_ses_key_t key;
 817   snat_session_t * s;
 818   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 819   snat_user_t *u;
 820
 821   old_addr = ip->dst_address.as_u32;
 822
 823   key.l_addr = ip->dst_address;
 824   key.r_addr = ip->src_address;
 825   key.fib_index = rx_fib_index;
 826   key.proto = ip->protocol;
 827   key.r_port = 0;
 828   key.l_port = 0;
 829   s_kv.key[0] = key.as_u64[0];
 830   s_kv.key[1] = key.as_u64[1];
 831
 832   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 833     {
 834       s = pool_elt_at_index (tsm->sessions, s_value.value);
 835       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 836     }
 837   else
 838     {
 839       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 840         {
 841           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 842           nat_log_notice ("maximum sessions exceeded");
 843           return 0;
 844         }
 845
 846       m_key.addr = ip->dst_address;
 847       m_key.port = 0;
 848       m_key.protocol = 0;
 849       m_key.fib_index = rx_fib_index;
 850       kv.key = m_key.as_u64;
 851       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 852         {
 853           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 854           return 0;
 855         }
 856
 857       m = pool_elt_at_index (sm->static_mappings, value.value);
 858
 859       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 860
 861       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 862                                   thread_index);
 863       if (!u)
 864         {
 865           nat_log_warn ("create NAT user failed");
 866           return 0;
 867         }
 868
 869       /* Create a new session */
 870       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 871       if (!s)
 872         {
 873           nat_log_warn ("create NAT session failed");
 874           return 0;
 875         }
 876
 877       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 878       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 879       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 880       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 881       s->outside_address_index = ~0;
 882       s->out2in.addr.as_u32 = old_addr;
 883       s->out2in.fib_index = rx_fib_index;
 884       s->in2out.addr.as_u32 = new_addr;
 885       s->in2out.fib_index = m->fib_index;
 886       s->in2out.port = s->out2in.port = ip->protocol;
 887       user_session_increment (sm, u, 1 /* static */);
 888
 889       /* Add to lookup tables */
 890       s_kv.value = s - tsm->sessions;
 891       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 892         nat_log_notice ("out2in key add failed");
 893
 894       key.l_addr = ip->dst_address;
 895       key.fib_index = m->fib_index;
 896       s_kv.key[0] = key.as_u64[0];
 897       s_kv.key[1] = key.as_u64[1];
 898       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 899         nat_log_notice ("in2out key add failed");
 900    }
 901
 902   /* Update IP checksum */
 903   sum = ip->checksum;
 904   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 905   ip->checksum = ip_csum_fold (sum);
 906
 907   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 908
 909   /* Accounting */
 910   nat44_session_update_counters (s, now,
 911                                  vlib_buffer_length_in_chain (vm, b));
 912   /* Per-user LRU list maintenance */
 913   nat44_session_update_lru (sm, s, thread_index);
 914
 915   return s;
 916 }
 917
 918 static snat_session_t *
 919 snat_out2in_lb (snat_main_t *sm,
 920                 vlib_buffer_t * b,
 921                 ip4_header_t * ip,
 922                 u32 rx_fib_index,
 923                 u32 thread_index,
 924                 f64 now,
 925                 vlib_main_t * vm,
 926                 vlib_node_runtime_t * node)
 927 {
 928   nat_ed_ses_key_t key;
 929   clib_bihash_kv_16_8_t s_kv, s_value;
 930   udp_header_t *udp = ip4_next_header (ip);
 931   tcp_header_t *tcp = (tcp_header_t *) udp;
 932   snat_session_t *s = 0;
 933   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 934   snat_session_key_t e_key, l_key;
 935   u32 old_addr, new_addr;
 936   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 937   u16 new_port, old_port;
 938   ip_csum_t sum;
 939   snat_user_t *u;
 940   u32 address_index;
 941   snat_session_key_t eh_key;
 942   twice_nat_type_t twice_nat;
 943   u8 lb;
 944
 945   old_addr = ip->dst_address.as_u32;
 946
 947   key.l_addr = ip->dst_address;
 948   key.r_addr = ip->src_address;
 949   key.fib_index = rx_fib_index;
 950   key.proto = ip->protocol;
 951   key.r_port = udp->src_port;
 952   key.l_port = udp->dst_port;
 953   s_kv.key[0] = key.as_u64[0];
 954   s_kv.key[1] = key.as_u64[1];
 955
 956   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 957     {
 958       s = pool_elt_at_index (tsm->sessions, s_value.value);
 959     }
 960   else
 961     {
 962       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 963         {
 964           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 965           nat_log_notice ("maximum sessions exceeded");
 966           return 0;
 967         }
 968
 969       e_key.addr = ip->dst_address;
 970       e_key.port = udp->dst_port;
 971       e_key.protocol = proto;
 972       e_key.fib_index = rx_fib_index;
 973       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
 974         return 0;
 975
 976       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 977                                   thread_index);
 978       if (!u)
 979       {
 980         nat_log_warn ("create NAT user failed");
 981         return 0;
 982       }
 983
 984       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 985       if (!s)
 986         {
 987           nat_log_warn ("create NAT session failed");
 988           return 0;
 989         }
 990
 991       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 992       s->ext_host_port = udp->src_port;
 993       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 994       if (lb)
 995         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 996       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 997       s->outside_address_index = ~0;
 998       s->out2in = e_key;
 999       s->in2out = l_key;
1000       user_session_increment (sm, u, 1 /* static */);
1001
1002       /* Add to lookup tables */
1003       s_kv.value = s - tsm->sessions;
1004       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1005         nat_log_notice ("out2in-ed key add failed");
1006
1007       if (twice_nat == TWICE_NAT ||
1008           (twice_nat == TWICE_NAT_SELF &&
1009            ip->src_address.as_u32 == l_key.addr.as_u32))
1010         {
1011           eh_key.protocol = proto;
1012           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1013                                                    thread_index, &eh_key,
1014                                                    &address_index,
1015                                                    sm->port_per_thread,
1016                                                    sm->per_thread_data[thread_index].snat_thread_index))
1017             {
1018               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1019               return 0;
1020             }
1021           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1022           key.r_port = s->ext_host_nat_port = eh_key.port;
1023           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1024         }
1025       key.l_addr = l_key.addr;
1026       key.fib_index = l_key.fib_index;
1027       key.l_port = l_key.port;
1028       s_kv.key[0] = key.as_u64[0];
1029       s_kv.key[1] = key.as_u64[1];
1030       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1031         nat_log_notice ("in2out-ed key add failed");
1032     }
1033
1034   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1035
1036   /* Update IP checksum */
1037   sum = ip->checksum;
1038   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1039   if (is_twice_nat_session (s))
1040     sum = ip_csum_update (sum, ip->src_address.as_u32,
1041                           s->ext_host_nat_addr.as_u32, ip4_header_t,
1042                           src_address);
1043   ip->checksum = ip_csum_fold (sum);
1044
1045   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1046
1047   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1048     {
1049       old_port = tcp->dst_port;
1050       tcp->dst_port = s->in2out.port;
1051       new_port = tcp->dst_port;
1052
1053       sum = tcp->checksum;
1054       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1055       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1056       if (is_twice_nat_session (s))
1057         {
1058           sum = ip_csum_update (sum, ip->src_address.as_u32,
1059                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
1060                                 dst_address);
1061           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1062                                 ip4_header_t, length);
1063           tcp->src_port = s->ext_host_nat_port;
1064           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1065         }
1066       tcp->checksum = ip_csum_fold(sum);
1067       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1068         return s;
1069     }
1070   else
1071     {
1072       udp->dst_port = s->in2out.port;
1073       if (is_twice_nat_session (s))
1074         {
1075           udp->src_port = s->ext_host_nat_port;
1076           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1077         }
1078       udp->checksum = 0;
1079     }
1080
1081   /* Accounting */
1082   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
1083   /* Per-user LRU list maintenance */
1084   nat44_session_update_lru (sm, s, thread_index);
1085
1086   return s;
1087 }
1088
1089 static uword
1090 snat_out2in_node_fn (vlib_main_t * vm,
1091                   vlib_node_runtime_t * node,
1092                   vlib_frame_t * frame)
1093 {
1094   u32 n_left_from, * from, * to_next;
1095   snat_out2in_next_t next_index;
1096   u32 pkts_processed = 0;
1097   snat_main_t * sm = &snat_main;
1098   f64 now = vlib_time_now (vm);
1099   u32 thread_index = vlib_get_thread_index ();
1100
1101   from = vlib_frame_vector_args (frame);
1102   n_left_from = frame->n_vectors;
1103   next_index = node->cached_next_index;
1104
1105   while (n_left_from > 0)
1106     {
1107       u32 n_left_to_next;
1108
1109       vlib_get_next_frame (vm, node, next_index,
1110                            to_next, n_left_to_next);
1111
1112       while (n_left_from >= 4 && n_left_to_next >= 2)
1113         {
1114           u32 bi0, bi1;
1115           vlib_buffer_t * b0, * b1;
1116           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1117           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1118           u32 sw_if_index0, sw_if_index1;
1119           ip4_header_t * ip0, *ip1;
1120           ip_csum_t sum0, sum1;
1121           u32 new_addr0, old_addr0;
1122           u16 new_port0, old_port0;
1123           u32 new_addr1, old_addr1;
1124           u16 new_port1, old_port1;
1125           udp_header_t * udp0, * udp1;
1126           tcp_header_t * tcp0, * tcp1;
1127           icmp46_header_t * icmp0, * icmp1;
1128           snat_session_key_t key0, key1, sm0, sm1;
1129           u32 rx_fib_index0, rx_fib_index1;
1130           u32 proto0, proto1;
1131           snat_session_t * s0 = 0, * s1 = 0;
1132           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1133
1134           /* Prefetch next iteration. */
1135           {
1136             vlib_buffer_t * p2, * p3;
1137
1138             p2 = vlib_get_buffer (vm, from[2]);
1139             p3 = vlib_get_buffer (vm, from[3]);
1140
1141             vlib_prefetch_buffer_header (p2, LOAD);
1142             vlib_prefetch_buffer_header (p3, LOAD);
1143
1144             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1145             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1146           }
1147
1148           /* speculatively enqueue b0 and b1 to the current next frame */
1149           to_next[0] = bi0 = from[0];
1150           to_next[1] = bi1 = from[1];
1151           from += 2;
1152           to_next += 2;
1153           n_left_from -= 2;
1154           n_left_to_next -= 2;
1155
1156           b0 = vlib_get_buffer (vm, bi0);
1157           b1 = vlib_get_buffer (vm, bi1);
1158
1159           vnet_buffer (b0)->snat.flags = 0;
1160           vnet_buffer (b1)->snat.flags = 0;
1161
1162           ip0 = vlib_buffer_get_current (b0);
1163           udp0 = ip4_next_header (ip0);
1164           tcp0 = (tcp_header_t *) udp0;
1165           icmp0 = (icmp46_header_t *) udp0;
1166
1167           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1168           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1169                                    sw_if_index0);
1170
1171           if (PREDICT_FALSE(ip0->ttl == 1))
1172             {
1173               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1174               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1175                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1176                                            0);
1177               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1178               goto trace0;
1179             }
1180
1181           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1182
1183           if (PREDICT_FALSE (proto0 == ~0))
1184             {
1185               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1186                                              thread_index, now, vm, node);
1187               if (!sm->forwarding_enabled)
1188                 if (!s0)
1189                   next0 = SNAT_OUT2IN_NEXT_DROP;
1190               goto trace0;
1191             }
1192
1193           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1194             {
1195               next0 = icmp_out2in_slow_path
1196                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1197                  next0, now, thread_index, &s0);
1198               goto trace0;
1199             }
1200
1201           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1202             {
1203               next0 = SNAT_OUT2IN_NEXT_REASS;
1204               goto trace0;
1205             }
1206
1207           key0.addr = ip0->dst_address;
1208           key0.port = udp0->dst_port;
1209           key0.protocol = proto0;
1210           key0.fib_index = rx_fib_index0;
1211
1212           kv0.key = key0.as_u64;
1213
1214           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1215                                       &kv0, &value0))
1216             {
1217               /* Try to match static mapping by external address and port,
1218                  destination address and port in packet */
1219               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1220                 {
1221                   /*
1222                    * Send DHCP packets to the ipv4 stack, or we won't
1223                    * be able to use dhcp client on the outside interface
1224                    */
1225                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1226                       && (udp0->dst_port ==
1227                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1228                     {
1229                       vnet_feature_next
1230                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1231                       goto trace0;
1232                     }
1233
1234                   if (!sm->forwarding_enabled)
1235                     {
1236                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1237                       next0 = SNAT_OUT2IN_NEXT_DROP;
1238                       goto trace0;
1239                     }
1240                   else
1241                     {
1242                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1243                         {
1244                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1245                           goto trace0;
1246                         }
1247                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1248                       goto trace0;
1249                     }
1250                 }
1251
1252               /* Create session initiated by host from external network */
1253               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1254                                                      thread_index);
1255               if (!s0)
1256                 {
1257                   next0 = SNAT_OUT2IN_NEXT_DROP;
1258                   goto trace0;
1259                 }
1260             }
1261           else
1262             {
1263               if (PREDICT_FALSE (value0.value == ~0ULL))
1264                 {
1265                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1266                                       now, vm, node);
1267                   if (!s0)
1268                     next0 = SNAT_OUT2IN_NEXT_DROP;
1269                   goto trace0;
1270                 }
1271               else
1272                 {
1273                   s0 = pool_elt_at_index (
1274                     sm->per_thread_data[thread_index].sessions,
1275                     value0.value);
1276                 }
1277             }
1278
1279           old_addr0 = ip0->dst_address.as_u32;
1280           ip0->dst_address = s0->in2out.addr;
1281           new_addr0 = ip0->dst_address.as_u32;
1282           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1283
1284           sum0 = ip0->checksum;
1285           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1286                                  ip4_header_t,
1287                                  dst_address /* changed member */);
1288           ip0->checksum = ip_csum_fold (sum0);
1289
1290           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1291             {
1292               old_port0 = tcp0->dst_port;
1293               tcp0->dst_port = s0->in2out.port;
1294               new_port0 = tcp0->dst_port;
1295
1296               sum0 = tcp0->checksum;
1297               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1298                                      ip4_header_t,
1299                                      dst_address /* changed member */);
1300
1301               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1302                                      ip4_header_t /* cheat */,
1303                                      length /* changed member */);
1304               tcp0->checksum = ip_csum_fold(sum0);
1305             }
1306           else
1307             {
1308               old_port0 = udp0->dst_port;
1309               udp0->dst_port = s0->in2out.port;
1310               udp0->checksum = 0;
1311             }
1312
1313           /* Accounting */
1314           nat44_session_update_counters (s0, now,
1315                                          vlib_buffer_length_in_chain (vm, b0));
1316           /* Per-user LRU list maintenance */
1317           nat44_session_update_lru (sm, s0, thread_index);
1318         trace0:
1319
1320           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1321                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1322             {
1323               snat_out2in_trace_t *t =
1324                  vlib_add_trace (vm, node, b0, sizeof (*t));
1325               t->sw_if_index = sw_if_index0;
1326               t->next_index = next0;
1327               t->session_index = ~0;
1328               if (s0)
1329                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1330             }
1331
1332           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1333
1334
1335           ip1 = vlib_buffer_get_current (b1);
1336           udp1 = ip4_next_header (ip1);
1337           tcp1 = (tcp_header_t *) udp1;
1338           icmp1 = (icmp46_header_t *) udp1;
1339
1340           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1341           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1342                                    sw_if_index1);
1343
1344           if (PREDICT_FALSE(ip1->ttl == 1))
1345             {
1346               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1347               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1348                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1349                                            0);
1350               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1351               goto trace1;
1352             }
1353
1354           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1355
1356           if (PREDICT_FALSE (proto1 == ~0))
1357             {
1358               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1359                                              thread_index, now, vm, node);
1360               if (!sm->forwarding_enabled)
1361                 if (!s1)
1362                   next1 = SNAT_OUT2IN_NEXT_DROP;
1363               goto trace1;
1364             }
1365
1366           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1367             {
1368               next1 = icmp_out2in_slow_path
1369                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1370                  next1, now, thread_index, &s1);
1371               goto trace1;
1372             }
1373
1374           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1375             {
1376               next1 = SNAT_OUT2IN_NEXT_REASS;
1377               goto trace1;
1378             }
1379
1380           key1.addr = ip1->dst_address;
1381           key1.port = udp1->dst_port;
1382           key1.protocol = proto1;
1383           key1.fib_index = rx_fib_index1;
1384
1385           kv1.key = key1.as_u64;
1386
1387           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1388                                       &kv1, &value1))
1389             {
1390               /* Try to match static mapping by external address and port,
1391                  destination address and port in packet */
1392               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1393                 {
1394                   /*
1395                    * Send DHCP packets to the ipv4 stack, or we won't
1396                    * be able to use dhcp client on the outside interface
1397                    */
1398                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1399                       && (udp1->dst_port ==
1400                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1401                     {
1402                       vnet_feature_next
1403                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1404                       goto trace1;
1405                     }
1406
1407                   if (!sm->forwarding_enabled)
1408                     {
1409                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1410                       next1 = SNAT_OUT2IN_NEXT_DROP;
1411                       goto trace1;
1412                     }
1413                   else
1414                     {
1415                       if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1416                         {
1417                           next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1418                           goto trace1;
1419                         }
1420                       create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1421                       goto trace1;
1422                     }
1423                 }
1424
1425               /* Create session initiated by host from external network */
1426               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1427                                                      thread_index);
1428               if (!s1)
1429                 {
1430                   next1 = SNAT_OUT2IN_NEXT_DROP;
1431                   goto trace1;
1432                 }
1433             }
1434           else
1435             {
1436               if (PREDICT_FALSE (value1.value == ~0ULL))
1437                 {
1438                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1439                                       now, vm, node);
1440                   if (!s1)
1441                     next1 = SNAT_OUT2IN_NEXT_DROP;
1442                   goto trace1;
1443                 }
1444               else
1445                 {
1446                   s1 = pool_elt_at_index (
1447                     sm->per_thread_data[thread_index].sessions,
1448                     value1.value);
1449                 }
1450             }
1451
1452           old_addr1 = ip1->dst_address.as_u32;
1453           ip1->dst_address = s1->in2out.addr;
1454           new_addr1 = ip1->dst_address.as_u32;
1455           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1456
1457           sum1 = ip1->checksum;
1458           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1459                                  ip4_header_t,
1460                                  dst_address /* changed member */);
1461           ip1->checksum = ip_csum_fold (sum1);
1462
1463           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1464             {
1465               old_port1 = tcp1->dst_port;
1466               tcp1->dst_port = s1->in2out.port;
1467               new_port1 = tcp1->dst_port;
1468
1469               sum1 = tcp1->checksum;
1470               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1471                                      ip4_header_t,
1472                                      dst_address /* changed member */);
1473
1474               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1475                                      ip4_header_t /* cheat */,
1476                                      length /* changed member */);
1477               tcp1->checksum = ip_csum_fold(sum1);
1478             }
1479           else
1480             {
1481               old_port1 = udp1->dst_port;
1482               udp1->dst_port = s1->in2out.port;
1483               udp1->checksum = 0;
1484             }
1485
1486           /* Accounting */
1487           nat44_session_update_counters (s1, now,
1488                                          vlib_buffer_length_in_chain (vm, b1));
1489           /* Per-user LRU list maintenance */
1490           nat44_session_update_lru (sm, s1, thread_index);
1491         trace1:
1492
1493           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1494                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1495             {
1496               snat_out2in_trace_t *t =
1497                  vlib_add_trace (vm, node, b1, sizeof (*t));
1498               t->sw_if_index = sw_if_index1;
1499               t->next_index = next1;
1500               t->session_index = ~0;
1501               if (s1)
1502                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1503             }
1504
1505           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1506
1507           /* verify speculative enqueues, maybe switch current next frame */
1508           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1509                                            to_next, n_left_to_next,
1510                                            bi0, bi1, next0, next1);
1511         }
1512
1513       while (n_left_from > 0 && n_left_to_next > 0)
1514         {
1515           u32 bi0;
1516           vlib_buffer_t * b0;
1517           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1518           u32 sw_if_index0;
1519           ip4_header_t * ip0;
1520           ip_csum_t sum0;
1521           u32 new_addr0, old_addr0;
1522           u16 new_port0, old_port0;
1523           udp_header_t * udp0;
1524           tcp_header_t * tcp0;
1525           icmp46_header_t * icmp0;
1526           snat_session_key_t key0, sm0;
1527           u32 rx_fib_index0;
1528           u32 proto0;
1529           snat_session_t * s0 = 0;
1530           clib_bihash_kv_8_8_t kv0, value0;
1531
1532           /* speculatively enqueue b0 to the current next frame */
1533           bi0 = from[0];
1534           to_next[0] = bi0;
1535           from += 1;
1536           to_next += 1;
1537           n_left_from -= 1;
1538           n_left_to_next -= 1;
1539
1540           b0 = vlib_get_buffer (vm, bi0);
1541
1542           vnet_buffer (b0)->snat.flags = 0;
1543
1544           ip0 = vlib_buffer_get_current (b0);
1545           udp0 = ip4_next_header (ip0);
1546           tcp0 = (tcp_header_t *) udp0;
1547           icmp0 = (icmp46_header_t *) udp0;
1548
1549           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1550           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1551                                    sw_if_index0);
1552
1553           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1554
1555           if (PREDICT_FALSE (proto0 == ~0))
1556             {
1557               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1558                                              thread_index, now, vm, node);
1559               if (!sm->forwarding_enabled)
1560                 if (!s0)
1561                   next0 = SNAT_OUT2IN_NEXT_DROP;
1562               goto trace00;
1563             }
1564
1565           if (PREDICT_FALSE(ip0->ttl == 1))
1566             {
1567               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1568               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1569                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1570                                            0);
1571               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1572               goto trace00;
1573             }
1574
1575           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1576             {
1577               next0 = icmp_out2in_slow_path
1578                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1579                  next0, now, thread_index, &s0);
1580               goto trace00;
1581             }
1582
1583           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1584             {
1585               next0 = SNAT_OUT2IN_NEXT_REASS;
1586               goto trace00;
1587             }
1588
1589           key0.addr = ip0->dst_address;
1590           key0.port = udp0->dst_port;
1591           key0.protocol = proto0;
1592           key0.fib_index = rx_fib_index0;
1593
1594           kv0.key = key0.as_u64;
1595
1596           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1597                                       &kv0, &value0))
1598             {
1599               /* Try to match static mapping by external address and port,
1600                  destination address and port in packet */
1601               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1602                 {
1603                   /*
1604                    * Send DHCP packets to the ipv4 stack, or we won't
1605                    * be able to use dhcp client on the outside interface
1606                    */
1607                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1608                       && (udp0->dst_port ==
1609                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1610                     {
1611                       vnet_feature_next
1612                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1613                       goto trace00;
1614                     }
1615
1616                   if (!sm->forwarding_enabled)
1617                     {
1618                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1619                       next0 = SNAT_OUT2IN_NEXT_DROP;
1620                       goto trace00;
1621                     }
1622                   else
1623                     {
1624                       if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1625                         {
1626                           next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1627                           goto trace00;
1628                         }
1629                       create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1630                       goto trace00;
1631                     }
1632                 }
1633
1634               /* Create session initiated by host from external network */
1635               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1636                                                      thread_index);
1637               if (!s0)
1638                 {
1639                   next0 = SNAT_OUT2IN_NEXT_DROP;
1640                   goto trace00;
1641                 }
1642             }
1643           else
1644             {
1645               if (PREDICT_FALSE (value0.value == ~0ULL))
1646                 {
1647                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1648                                       now, vm, node);
1649                   if (!s0)
1650                     next0 = SNAT_OUT2IN_NEXT_DROP;
1651                   goto trace00;
1652                 }
1653               else
1654                 {
1655                   s0 = pool_elt_at_index (
1656                     sm->per_thread_data[thread_index].sessions,
1657                     value0.value);
1658                 }
1659             }
1660
1661           old_addr0 = ip0->dst_address.as_u32;
1662           ip0->dst_address = s0->in2out.addr;
1663           new_addr0 = ip0->dst_address.as_u32;
1664           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1665
1666           sum0 = ip0->checksum;
1667           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1668                                  ip4_header_t,
1669                                  dst_address /* changed member */);
1670           ip0->checksum = ip_csum_fold (sum0);
1671
1672           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1673             {
1674               old_port0 = tcp0->dst_port;
1675               tcp0->dst_port = s0->in2out.port;
1676               new_port0 = tcp0->dst_port;
1677
1678               sum0 = tcp0->checksum;
1679               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1680                                      ip4_header_t,
1681                                      dst_address /* changed member */);
1682
1683               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1684                                      ip4_header_t /* cheat */,
1685                                      length /* changed member */);
1686               tcp0->checksum = ip_csum_fold(sum0);
1687             }
1688           else
1689             {
1690               old_port0 = udp0->dst_port;
1691               udp0->dst_port = s0->in2out.port;
1692               udp0->checksum = 0;
1693             }
1694
1695           /* Accounting */
1696           nat44_session_update_counters (s0, now,
1697                                          vlib_buffer_length_in_chain (vm, b0));
1698           /* Per-user LRU list maintenance */
1699           nat44_session_update_lru (sm, s0, thread_index);
1700         trace00:
1701
1702           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1703                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1704             {
1705               snat_out2in_trace_t *t =
1706                  vlib_add_trace (vm, node, b0, sizeof (*t));
1707               t->sw_if_index = sw_if_index0;
1708               t->next_index = next0;
1709               t->session_index = ~0;
1710               if (s0)
1711                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1712             }
1713
1714           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1715
1716           /* verify speculative enqueue, maybe switch current next frame */
1717           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1718                                            to_next, n_left_to_next,
1719                                            bi0, next0);
1720         }
1721
1722       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1723     }
1724
1725   vlib_node_increment_counter (vm, snat_out2in_node.index,
1726                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1727                                pkts_processed);
1728   return frame->n_vectors;
1729 }
1730
1731 VLIB_REGISTER_NODE (snat_out2in_node) = {
1732   .function = snat_out2in_node_fn,
1733   .name = "nat44-out2in",
1734   .vector_size = sizeof (u32),
1735   .format_trace = format_snat_out2in_trace,
1736   .type = VLIB_NODE_TYPE_INTERNAL,
1737
1738   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1739   .error_strings = snat_out2in_error_strings,
1740
1741   .runtime_data_bytes = sizeof (snat_runtime_t),
1742
1743   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1744
1745   /* edit / add dispositions here */
1746   .next_nodes = {
1747     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1748     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1749     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1750     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1751     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1752   },
1753 };
1754 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1755
1756 static uword
1757 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1758                             vlib_node_runtime_t * node,
1759                             vlib_frame_t * frame)
1760 {
1761   u32 n_left_from, *from, *to_next;
1762   snat_out2in_next_t next_index;
1763   u32 pkts_processed = 0;
1764   snat_main_t *sm = &snat_main;
1765   f64 now = vlib_time_now (vm);
1766   u32 thread_index = vlib_get_thread_index ();
1767   snat_main_per_thread_data_t *per_thread_data =
1768     &sm->per_thread_data[thread_index];
1769   u32 *fragments_to_drop = 0;
1770   u32 *fragments_to_loopback = 0;
1771
1772   from = vlib_frame_vector_args (frame);
1773   n_left_from = frame->n_vectors;
1774   next_index = node->cached_next_index;
1775
1776   while (n_left_from > 0)
1777     {
1778       u32 n_left_to_next;
1779
1780       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1781
1782       while (n_left_from > 0 && n_left_to_next > 0)
1783        {
1784           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1785           vlib_buffer_t *b0;
1786           u32 next0;
1787           u8 cached0 = 0;
1788           ip4_header_t *ip0;
1789           nat_reass_ip4_t *reass0;
1790           udp_header_t * udp0;
1791           tcp_header_t * tcp0;
1792           snat_session_key_t key0, sm0;
1793           clib_bihash_kv_8_8_t kv0, value0;
1794           snat_session_t * s0 = 0;
1795           u16 old_port0, new_port0;
1796           ip_csum_t sum0;
1797
1798           /* speculatively enqueue b0 to the current next frame */
1799           bi0 = from[0];
1800           to_next[0] = bi0;
1801           from += 1;
1802           to_next += 1;
1803           n_left_from -= 1;
1804           n_left_to_next -= 1;
1805
1806           b0 = vlib_get_buffer (vm, bi0);
1807           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1808
1809           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1810           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1811                                                                sw_if_index0);
1812
1813           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1814             {
1815               next0 = SNAT_OUT2IN_NEXT_DROP;
1816               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1817               goto trace0;
1818             }
1819
1820           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1821           udp0 = ip4_next_header (ip0);
1822           tcp0 = (tcp_header_t *) udp0;
1823           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1824
1825           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1826                                                  ip0->dst_address,
1827                                                  ip0->fragment_id,
1828                                                  ip0->protocol,
1829                                                  1,
1830                                                  &fragments_to_drop);
1831
1832           if (PREDICT_FALSE (!reass0))
1833             {
1834               next0 = SNAT_OUT2IN_NEXT_DROP;
1835               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1836               nat_log_notice ("maximum reassemblies exceeded");
1837               goto trace0;
1838             }
1839
1840           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1841             {
1842               key0.addr = ip0->dst_address;
1843               key0.port = udp0->dst_port;
1844               key0.protocol = proto0;
1845               key0.fib_index = rx_fib_index0;
1846               kv0.key = key0.as_u64;
1847
1848               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1849                 {
1850                   /* Try to match static mapping by external address and port,
1851                      destination address and port in packet */
1852                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1853                     {
1854                       /*
1855                        * Send DHCP packets to the ipv4 stack, or we won't
1856                        * be able to use dhcp client on the outside interface
1857                        */
1858                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1859                           && (udp0->dst_port
1860                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1861                         {
1862                           vnet_feature_next
1863                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1864                              &next0, b0);
1865                           goto trace0;
1866                         }
1867
1868                       if (!sm->forwarding_enabled)
1869                         {
1870                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1871                           next0 = SNAT_OUT2IN_NEXT_DROP;
1872                           goto trace0;
1873                         }
1874                       else
1875                         {
1876                           if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1877                             {
1878                               next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1879                               goto trace0;
1880                             }
1881                           create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1882                           goto trace0;
1883                         }
1884                     }
1885
1886                   /* Create session initiated by host from external network */
1887                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1888                                                          thread_index);
1889                   if (!s0)
1890                     {
1891                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1892                       next0 = SNAT_OUT2IN_NEXT_DROP;
1893                       goto trace0;
1894                     }
1895                   reass0->sess_index = s0 - per_thread_data->sessions;
1896                   reass0->thread_index = thread_index;
1897                 }
1898               else
1899                 {
1900                   s0 = pool_elt_at_index (per_thread_data->sessions,
1901                                           value0.value);
1902                   reass0->sess_index = value0.value;
1903                 }
1904               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1905             }
1906           else
1907             {
1908               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1909                 {
1910                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1911                     {
1912                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1913                       nat_log_notice ("maximum fragments per reassembly exceeded");
1914                       next0 = SNAT_OUT2IN_NEXT_DROP;
1915                       goto trace0;
1916                     }
1917                   cached0 = 1;
1918                   goto trace0;
1919                 }
1920               s0 = pool_elt_at_index (per_thread_data->sessions,
1921                                       reass0->sess_index);
1922             }
1923
1924           old_addr0 = ip0->dst_address.as_u32;
1925           ip0->dst_address = s0->in2out.addr;
1926           new_addr0 = ip0->dst_address.as_u32;
1927           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1928
1929           sum0 = ip0->checksum;
1930           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1931                                  ip4_header_t,
1932                                  dst_address /* changed member */);
1933           ip0->checksum = ip_csum_fold (sum0);
1934
1935           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1936             {
1937               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1938                 {
1939                   old_port0 = tcp0->dst_port;
1940                   tcp0->dst_port = s0->in2out.port;
1941                   new_port0 = tcp0->dst_port;
1942
1943                   sum0 = tcp0->checksum;
1944                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1945                                          ip4_header_t,
1946                                          dst_address /* changed member */);
1947
1948                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1949                                          ip4_header_t /* cheat */,
1950                                          length /* changed member */);
1951                   tcp0->checksum = ip_csum_fold(sum0);
1952                 }
1953               else
1954                 {
1955                   old_port0 = udp0->dst_port;
1956                   udp0->dst_port = s0->in2out.port;
1957                   udp0->checksum = 0;
1958                 }
1959             }
1960
1961           /* Accounting */
1962           nat44_session_update_counters (s0, now,
1963                                          vlib_buffer_length_in_chain (vm, b0));
1964           /* Per-user LRU list maintenance */
1965           nat44_session_update_lru (sm, s0, thread_index);
1966
1967         trace0:
1968           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1969                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1970             {
1971               nat44_out2in_reass_trace_t *t =
1972                  vlib_add_trace (vm, node, b0, sizeof (*t));
1973               t->cached = cached0;
1974               t->sw_if_index = sw_if_index0;
1975               t->next_index = next0;
1976             }
1977
1978           if (cached0)
1979             {
1980               n_left_to_next++;
1981               to_next--;
1982             }
1983           else
1984             {
1985               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1986
1987               /* verify speculative enqueue, maybe switch current next frame */
1988               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1989                                                to_next, n_left_to_next,
1990                                                bi0, next0);
1991             }
1992
1993           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1994             {
1995               from = vlib_frame_vector_args (frame);
1996               u32 len = vec_len (fragments_to_loopback);
1997               if (len <= VLIB_FRAME_SIZE)
1998                 {
1999                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2000                   n_left_from = len;
2001                   vec_reset_length (fragments_to_loopback);
2002                 }
2003               else
2004                 {
2005                   clib_memcpy (from,
2006                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2007                                sizeof (u32) * VLIB_FRAME_SIZE);
2008                   n_left_from = VLIB_FRAME_SIZE;
2009                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2010                 }
2011             }
2012        }
2013
2014       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2015     }
2016
2017   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2018                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2019                                pkts_processed);
2020
2021   nat_send_all_to_node (vm, fragments_to_drop, node,
2022                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2023                         SNAT_OUT2IN_NEXT_DROP);
2024
2025   vec_free (fragments_to_drop);
2026   vec_free (fragments_to_loopback);
2027   return frame->n_vectors;
2028 }
2029
2030 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2031   .function = nat44_out2in_reass_node_fn,
2032   .name = "nat44-out2in-reass",
2033   .vector_size = sizeof (u32),
2034   .format_trace = format_nat44_out2in_reass_trace,
2035   .type = VLIB_NODE_TYPE_INTERNAL,
2036
2037   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2038   .error_strings = snat_out2in_error_strings,
2039
2040   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2041
2042   /* edit / add dispositions here */
2043   .next_nodes = {
2044     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2045     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2046     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2047     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2048     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2049   },
2050 };
2051 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2052                               nat44_out2in_reass_node_fn);
2053
2054 /**************************/
2055 /*** deterministic mode ***/
2056 /**************************/
2057 static uword
2058 snat_det_out2in_node_fn (vlib_main_t * vm,
2059                          vlib_node_runtime_t * node,
2060                          vlib_frame_t * frame)
2061 {
2062   u32 n_left_from, * from, * to_next;
2063   snat_out2in_next_t next_index;
2064   u32 pkts_processed = 0;
2065   snat_main_t * sm = &snat_main;
2066   u32 thread_index = vlib_get_thread_index ();
2067
2068   from = vlib_frame_vector_args (frame);
2069   n_left_from = frame->n_vectors;
2070   next_index = node->cached_next_index;
2071
2072   while (n_left_from > 0)
2073     {
2074       u32 n_left_to_next;
2075
2076       vlib_get_next_frame (vm, node, next_index,
2077                            to_next, n_left_to_next);
2078
2079       while (n_left_from >= 4 && n_left_to_next >= 2)
2080         {
2081           u32 bi0, bi1;
2082           vlib_buffer_t * b0, * b1;
2083           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2084           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2085           u32 sw_if_index0, sw_if_index1;
2086           ip4_header_t * ip0, * ip1;
2087           ip_csum_t sum0, sum1;
2088           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2089           u16 new_port0, old_port0, old_port1, new_port1;
2090           udp_header_t * udp0, * udp1;
2091           tcp_header_t * tcp0, * tcp1;
2092           u32 proto0, proto1;
2093           snat_det_out_key_t key0, key1;
2094           snat_det_map_t * dm0, * dm1;
2095           snat_det_session_t * ses0 = 0, * ses1 = 0;
2096           u32 rx_fib_index0, rx_fib_index1;
2097           icmp46_header_t * icmp0, * icmp1;
2098
2099           /* Prefetch next iteration. */
2100           {
2101             vlib_buffer_t * p2, * p3;
2102
2103             p2 = vlib_get_buffer (vm, from[2]);
2104             p3 = vlib_get_buffer (vm, from[3]);
2105
2106             vlib_prefetch_buffer_header (p2, LOAD);
2107             vlib_prefetch_buffer_header (p3, LOAD);
2108
2109             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2110             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2111           }
2112
2113           /* speculatively enqueue b0 and b1 to the current next frame */
2114           to_next[0] = bi0 = from[0];
2115           to_next[1] = bi1 = from[1];
2116           from += 2;
2117           to_next += 2;
2118           n_left_from -= 2;
2119           n_left_to_next -= 2;
2120
2121           b0 = vlib_get_buffer (vm, bi0);
2122           b1 = vlib_get_buffer (vm, bi1);
2123
2124           ip0 = vlib_buffer_get_current (b0);
2125           udp0 = ip4_next_header (ip0);
2126           tcp0 = (tcp_header_t *) udp0;
2127
2128           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2129
2130           if (PREDICT_FALSE(ip0->ttl == 1))
2131             {
2132               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2133               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2134                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2135                                            0);
2136               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2137               goto trace0;
2138             }
2139
2140           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2141
2142           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2143             {
2144               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2145               icmp0 = (icmp46_header_t *) udp0;
2146
2147               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2148                                   rx_fib_index0, node, next0, thread_index,
2149                                   &ses0, &dm0);
2150               goto trace0;
2151             }
2152
2153           key0.ext_host_addr = ip0->src_address;
2154           key0.ext_host_port = tcp0->src;
2155           key0.out_port = tcp0->dst;
2156
2157           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2158           if (PREDICT_FALSE(!dm0))
2159             {
2160               nat_log_info ("unknown dst address:  %U",
2161                             format_ip4_address, &ip0->dst_address);
2162               next0 = SNAT_OUT2IN_NEXT_DROP;
2163               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2164               goto trace0;
2165             }
2166
2167           snat_det_reverse(dm0, &ip0->dst_address,
2168                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2169
2170           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2171           if (PREDICT_FALSE(!ses0))
2172             {
2173               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2174                             format_ip4_address, &ip0->src_address,
2175                             clib_net_to_host_u16 (tcp0->src),
2176                             format_ip4_address, &ip0->dst_address,
2177                             clib_net_to_host_u16 (tcp0->dst),
2178                             format_ip4_address, &new_addr0);
2179               next0 = SNAT_OUT2IN_NEXT_DROP;
2180               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2181               goto trace0;
2182             }
2183           new_port0 = ses0->in_port;
2184
2185           old_addr0 = ip0->dst_address;
2186           ip0->dst_address = new_addr0;
2187           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2188
2189           sum0 = ip0->checksum;
2190           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2191                                  ip4_header_t,
2192                                  dst_address /* changed member */);
2193           ip0->checksum = ip_csum_fold (sum0);
2194
2195           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2196             {
2197               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2198                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2199               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2200                 snat_det_ses_close(dm0, ses0);
2201
2202               old_port0 = tcp0->dst;
2203               tcp0->dst = new_port0;
2204
2205               sum0 = tcp0->checksum;
2206               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2207                                      ip4_header_t,
2208                                      dst_address /* changed member */);
2209
2210               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2211                                      ip4_header_t /* cheat */,
2212                                      length /* changed member */);
2213               tcp0->checksum = ip_csum_fold(sum0);
2214             }
2215           else
2216             {
2217               old_port0 = udp0->dst_port;
2218               udp0->dst_port = new_port0;
2219               udp0->checksum = 0;
2220             }
2221
2222         trace0:
2223
2224           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2225                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2226             {
2227               snat_out2in_trace_t *t =
2228                  vlib_add_trace (vm, node, b0, sizeof (*t));
2229               t->sw_if_index = sw_if_index0;
2230               t->next_index = next0;
2231               t->session_index = ~0;
2232               if (ses0)
2233                 t->session_index = ses0 - dm0->sessions;
2234             }
2235
2236           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2237
2238           b1 = vlib_get_buffer (vm, bi1);
2239
2240           ip1 = vlib_buffer_get_current (b1);
2241           udp1 = ip4_next_header (ip1);
2242           tcp1 = (tcp_header_t *) udp1;
2243
2244           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2245
2246           if (PREDICT_FALSE(ip1->ttl == 1))
2247             {
2248               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2249               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2250                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2251                                            0);
2252               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2253               goto trace1;
2254             }
2255
2256           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2257
2258           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2259             {
2260               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2261               icmp1 = (icmp46_header_t *) udp1;
2262
2263               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2264                                   rx_fib_index1, node, next1, thread_index,
2265                                   &ses1, &dm1);
2266               goto trace1;
2267             }
2268
2269           key1.ext_host_addr = ip1->src_address;
2270           key1.ext_host_port = tcp1->src;
2271           key1.out_port = tcp1->dst;
2272
2273           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2274           if (PREDICT_FALSE(!dm1))
2275             {
2276               nat_log_info ("unknown dst address:  %U",
2277                             format_ip4_address, &ip1->dst_address);
2278               next1 = SNAT_OUT2IN_NEXT_DROP;
2279               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2280               goto trace1;
2281             }
2282
2283           snat_det_reverse(dm1, &ip1->dst_address,
2284                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2285
2286           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2287           if (PREDICT_FALSE(!ses1))
2288             {
2289               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2290                             format_ip4_address, &ip1->src_address,
2291                             clib_net_to_host_u16 (tcp1->src),
2292                             format_ip4_address, &ip1->dst_address,
2293                             clib_net_to_host_u16 (tcp1->dst),
2294                             format_ip4_address, &new_addr1);
2295               next1 = SNAT_OUT2IN_NEXT_DROP;
2296               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2297               goto trace1;
2298             }
2299           new_port1 = ses1->in_port;
2300
2301           old_addr1 = ip1->dst_address;
2302           ip1->dst_address = new_addr1;
2303           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2304
2305           sum1 = ip1->checksum;
2306           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2307                                  ip4_header_t,
2308                                  dst_address /* changed member */);
2309           ip1->checksum = ip_csum_fold (sum1);
2310
2311           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2312             {
2313               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2314                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2315               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2316                 snat_det_ses_close(dm1, ses1);
2317
2318               old_port1 = tcp1->dst;
2319               tcp1->dst = new_port1;
2320
2321               sum1 = tcp1->checksum;
2322               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2323                                      ip4_header_t,
2324                                      dst_address /* changed member */);
2325
2326               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2327                                      ip4_header_t /* cheat */,
2328                                      length /* changed member */);
2329               tcp1->checksum = ip_csum_fold(sum1);
2330             }
2331           else
2332             {
2333               old_port1 = udp1->dst_port;
2334               udp1->dst_port = new_port1;
2335               udp1->checksum = 0;
2336             }
2337
2338         trace1:
2339
2340           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2341                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2342             {
2343               snat_out2in_trace_t *t =
2344                  vlib_add_trace (vm, node, b1, sizeof (*t));
2345               t->sw_if_index = sw_if_index1;
2346               t->next_index = next1;
2347               t->session_index = ~0;
2348               if (ses1)
2349                 t->session_index = ses1 - dm1->sessions;
2350             }
2351
2352           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2353
2354           /* verify speculative enqueues, maybe switch current next frame */
2355           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2356                                            to_next, n_left_to_next,
2357                                            bi0, bi1, next0, next1);
2358          }
2359
2360       while (n_left_from > 0 && n_left_to_next > 0)
2361         {
2362           u32 bi0;
2363           vlib_buffer_t * b0;
2364           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2365           u32 sw_if_index0;
2366           ip4_header_t * ip0;
2367           ip_csum_t sum0;
2368           ip4_address_t new_addr0, old_addr0;
2369           u16 new_port0, old_port0;
2370           udp_header_t * udp0;
2371           tcp_header_t * tcp0;
2372           u32 proto0;
2373           snat_det_out_key_t key0;
2374           snat_det_map_t * dm0;
2375           snat_det_session_t * ses0 = 0;
2376           u32 rx_fib_index0;
2377           icmp46_header_t * icmp0;
2378
2379           /* speculatively enqueue b0 to the current next frame */
2380           bi0 = from[0];
2381           to_next[0] = bi0;
2382           from += 1;
2383           to_next += 1;
2384           n_left_from -= 1;
2385           n_left_to_next -= 1;
2386
2387           b0 = vlib_get_buffer (vm, bi0);
2388
2389           ip0 = vlib_buffer_get_current (b0);
2390           udp0 = ip4_next_header (ip0);
2391           tcp0 = (tcp_header_t *) udp0;
2392
2393           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2394
2395           if (PREDICT_FALSE(ip0->ttl == 1))
2396             {
2397               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2398               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2399                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2400                                            0);
2401               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2402               goto trace00;
2403             }
2404
2405           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2406
2407           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2408             {
2409               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2410               icmp0 = (icmp46_header_t *) udp0;
2411
2412               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2413                                   rx_fib_index0, node, next0, thread_index,
2414                                   &ses0, &dm0);
2415               goto trace00;
2416             }
2417
2418           key0.ext_host_addr = ip0->src_address;
2419           key0.ext_host_port = tcp0->src;
2420           key0.out_port = tcp0->dst;
2421
2422           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2423           if (PREDICT_FALSE(!dm0))
2424             {
2425               nat_log_info ("unknown dst address:  %U",
2426                             format_ip4_address, &ip0->dst_address);
2427               next0 = SNAT_OUT2IN_NEXT_DROP;
2428               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2429               goto trace00;
2430             }
2431
2432           snat_det_reverse(dm0, &ip0->dst_address,
2433                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2434
2435           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2436           if (PREDICT_FALSE(!ses0))
2437             {
2438               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2439                             format_ip4_address, &ip0->src_address,
2440                             clib_net_to_host_u16 (tcp0->src),
2441                             format_ip4_address, &ip0->dst_address,
2442                             clib_net_to_host_u16 (tcp0->dst),
2443                             format_ip4_address, &new_addr0);
2444               next0 = SNAT_OUT2IN_NEXT_DROP;
2445               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2446               goto trace00;
2447             }
2448           new_port0 = ses0->in_port;
2449
2450           old_addr0 = ip0->dst_address;
2451           ip0->dst_address = new_addr0;
2452           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2453
2454           sum0 = ip0->checksum;
2455           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2456                                  ip4_header_t,
2457                                  dst_address /* changed member */);
2458           ip0->checksum = ip_csum_fold (sum0);
2459
2460           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2461             {
2462               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2463                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2464               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2465                 snat_det_ses_close(dm0, ses0);
2466
2467               old_port0 = tcp0->dst;
2468               tcp0->dst = new_port0;
2469
2470               sum0 = tcp0->checksum;
2471               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2472                                      ip4_header_t,
2473                                      dst_address /* changed member */);
2474
2475               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2476                                      ip4_header_t /* cheat */,
2477                                      length /* changed member */);
2478               tcp0->checksum = ip_csum_fold(sum0);
2479             }
2480           else
2481             {
2482               old_port0 = udp0->dst_port;
2483               udp0->dst_port = new_port0;
2484               udp0->checksum = 0;
2485             }
2486
2487         trace00:
2488
2489           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2490                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2491             {
2492               snat_out2in_trace_t *t =
2493                  vlib_add_trace (vm, node, b0, sizeof (*t));
2494               t->sw_if_index = sw_if_index0;
2495               t->next_index = next0;
2496               t->session_index = ~0;
2497               if (ses0)
2498                 t->session_index = ses0 - dm0->sessions;
2499             }
2500
2501           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2502
2503           /* verify speculative enqueue, maybe switch current next frame */
2504           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2505                                            to_next, n_left_to_next,
2506                                            bi0, next0);
2507         }
2508
2509       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2510     }
2511
2512   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2513                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2514                                pkts_processed);
2515   return frame->n_vectors;
2516 }
2517
2518 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2519   .function = snat_det_out2in_node_fn,
2520   .name = "nat44-det-out2in",
2521   .vector_size = sizeof (u32),
2522   .format_trace = format_snat_out2in_trace,
2523   .type = VLIB_NODE_TYPE_INTERNAL,
2524
2525   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2526   .error_strings = snat_out2in_error_strings,
2527
2528   .runtime_data_bytes = sizeof (snat_runtime_t),
2529
2530   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2531
2532   /* edit / add dispositions here */
2533   .next_nodes = {
2534     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2535     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2536     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2537     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2538     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2539   },
2540 };
2541 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2542
2543 /**
2544  * Get address and port values to be used for ICMP packet translation
2545  * and create session if needed
2546  *
2547  * @param[in,out] sm             NAT main
2548  * @param[in,out] node           NAT node runtime
2549  * @param[in] thread_index       thread index
2550  * @param[in,out] b0             buffer containing packet to be translated
2551  * @param[out] p_proto           protocol used for matching
2552  * @param[out] p_value           address and port after NAT translation
2553  * @param[out] p_dont_translate  if packet should not be translated
2554  * @param d                      optional parameter
2555  * @param e                      optional parameter
2556  */
2557 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2558                           u32 thread_index, vlib_buffer_t *b0,
2559                           ip4_header_t *ip0, u8 *p_proto,
2560                           snat_session_key_t *p_value,
2561                           u8 *p_dont_translate, void *d, void *e)
2562 {
2563   icmp46_header_t *icmp0;
2564   u32 sw_if_index0;
2565   u8 protocol;
2566   snat_det_out_key_t key0;
2567   u8 dont_translate = 0;
2568   u32 next0 = ~0;
2569   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2570   ip4_header_t *inner_ip0;
2571   void *l4_header = 0;
2572   icmp46_header_t *inner_icmp0;
2573   snat_det_map_t * dm0 = 0;
2574   ip4_address_t new_addr0 = {{0}};
2575   snat_det_session_t * ses0 = 0;
2576   ip4_address_t out_addr;
2577
2578   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2579   echo0 = (icmp_echo_header_t *)(icmp0+1);
2580   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2581
2582   if (!icmp_is_error_message (icmp0))
2583     {
2584       protocol = SNAT_PROTOCOL_ICMP;
2585       key0.ext_host_addr = ip0->src_address;
2586       key0.ext_host_port = 0;
2587       key0.out_port = echo0->identifier;
2588       out_addr = ip0->dst_address;
2589     }
2590   else
2591     {
2592       inner_ip0 = (ip4_header_t *)(echo0+1);
2593       l4_header = ip4_next_header (inner_ip0);
2594       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2595       key0.ext_host_addr = inner_ip0->dst_address;
2596       out_addr = inner_ip0->src_address;
2597       switch (protocol)
2598         {
2599         case SNAT_PROTOCOL_ICMP:
2600           inner_icmp0 = (icmp46_header_t*)l4_header;
2601           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2602           key0.ext_host_port = 0;
2603           key0.out_port = inner_echo0->identifier;
2604           break;
2605         case SNAT_PROTOCOL_UDP:
2606         case SNAT_PROTOCOL_TCP:
2607           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2608           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2609           break;
2610         default:
2611           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2612           next0 = SNAT_OUT2IN_NEXT_DROP;
2613           goto out;
2614         }
2615     }
2616
2617   dm0 = snat_det_map_by_out(sm, &out_addr);
2618   if (PREDICT_FALSE(!dm0))
2619     {
2620       /* Don't NAT packet aimed at the intfc address */
2621       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2622                                           ip0->dst_address.as_u32)))
2623         {
2624           dont_translate = 1;
2625           goto out;
2626         }
2627       nat_log_info ("unknown dst address:  %U",
2628                     format_ip4_address, &ip0->dst_address);
2629       goto out;
2630     }
2631
2632   snat_det_reverse(dm0, &ip0->dst_address,
2633                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2634
2635   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2636   if (PREDICT_FALSE(!ses0))
2637     {
2638       /* Don't NAT packet aimed at the intfc address */
2639       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2640                                           ip0->dst_address.as_u32)))
2641         {
2642           dont_translate = 1;
2643           goto out;
2644         }
2645       nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2646                     format_ip4_address, &key0.ext_host_addr,
2647                     clib_net_to_host_u16 (key0.ext_host_port),
2648                     format_ip4_address, &out_addr,
2649                     clib_net_to_host_u16 (key0.out_port),
2650                     format_ip4_address, &new_addr0);
2651       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2652       next0 = SNAT_OUT2IN_NEXT_DROP;
2653       goto out;
2654     }
2655
2656   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2657                     !icmp_is_error_message (icmp0)))
2658     {
2659       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2660       next0 = SNAT_OUT2IN_NEXT_DROP;
2661       goto out;
2662     }
2663
2664   goto out;
2665
2666 out:
2667   *p_proto = protocol;
2668   if (ses0)
2669     {
2670       p_value->addr = new_addr0;
2671       p_value->fib_index = sm->inside_fib_index;
2672       p_value->port = ses0->in_port;
2673     }
2674   *p_dont_translate = dont_translate;
2675   if (d)
2676     *(snat_det_session_t**)d = ses0;
2677   if (e)
2678     *(snat_det_map_t**)e = dm0;
2679   return next0;
2680 }
2681
2682 /**********************/
2683 /*** worker handoff ***/
2684 /**********************/
2685 static uword
2686 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2687                                vlib_node_runtime_t * node,
2688                                vlib_frame_t * frame)
2689 {
2690   snat_main_t *sm = &snat_main;
2691   vlib_thread_main_t *tm = vlib_get_thread_main ();
2692   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
2693   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2694   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2695     = 0;
2696   vlib_frame_queue_elt_t *hf = 0;
2697   vlib_frame_queue_t *fq;
2698   vlib_frame_t *f = 0;
2699   int i;
2700   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2701   u32 next_worker_index = 0;
2702   u32 current_worker_index = ~0;
2703   u32 thread_index = vlib_get_thread_index ();
2704   vlib_frame_t *d = 0;
2705
2706   ASSERT (vec_len (sm->workers));
2707
2708   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2709     {
2710       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2711
2712       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2713                                tm->n_vlib_mains - 1,
2714                                (vlib_frame_queue_t *) (~0));
2715     }
2716
2717   from = vlib_frame_vector_args (frame);
2718   n_left_from = frame->n_vectors;
2719
2720   while (n_left_from > 0)
2721     {
2722       u32 bi0;
2723       vlib_buffer_t *b0;
2724       u32 sw_if_index0;
2725       u32 rx_fib_index0;
2726       ip4_header_t * ip0;
2727       u8 do_handoff;
2728
2729       bi0 = from[0];
2730       from += 1;
2731       n_left_from -= 1;
2732
2733       b0 = vlib_get_buffer (vm, bi0);
2734
2735       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2736       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2737
2738       ip0 = vlib_buffer_get_current (b0);
2739
2740       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2741
2742       if (PREDICT_FALSE (next_worker_index != thread_index))
2743         {
2744           do_handoff = 1;
2745
2746           if (next_worker_index != current_worker_index)
2747             {
2748               fq = is_vlib_frame_queue_congested (
2749                 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
2750                 congested_handoff_queue_by_worker_index);
2751
2752               if (fq)
2753                 {
2754                   /* if this is 1st frame */
2755                   if (!d)
2756                     {
2757                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
2758                       to_next_drop = vlib_frame_vector_args (d);
2759                     }
2760
2761                   to_next_drop[0] = bi0;
2762                   to_next_drop += 1;
2763                   d->n_vectors++;
2764                   b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
2765                   goto trace0;
2766                 }
2767
2768               if (hf)
2769                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2770
2771               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2772                                                       next_worker_index,
2773                                                       handoff_queue_elt_by_worker_index);
2774
2775               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2776               to_next_worker = &hf->buffer_index[hf->n_vectors];
2777               current_worker_index = next_worker_index;
2778             }
2779
2780           /* enqueue to correct worker thread */
2781           to_next_worker[0] = bi0;
2782           to_next_worker++;
2783           n_left_to_next_worker--;
2784
2785           if (n_left_to_next_worker == 0)
2786             {
2787               hf->n_vectors = VLIB_FRAME_SIZE;
2788               vlib_put_frame_queue_elt (hf);
2789               current_worker_index = ~0;
2790               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2791               hf = 0;
2792             }
2793         }
2794       else
2795         {
2796           do_handoff = 0;
2797           /* if this is 1st frame */
2798           if (!f)
2799             {
2800               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2801               to_next = vlib_frame_vector_args (f);
2802             }
2803
2804           to_next[0] = bi0;
2805           to_next += 1;
2806           f->n_vectors++;
2807         }
2808
2809 trace0:
2810       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2811                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2812         {
2813           snat_out2in_worker_handoff_trace_t *t =
2814             vlib_add_trace (vm, node, b0, sizeof (*t));
2815           t->next_worker_index = next_worker_index;
2816           t->do_handoff = do_handoff;
2817         }
2818     }
2819
2820   if (f)
2821     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2822
2823   if (d)
2824     vlib_put_frame_to_node (vm, sm->error_node_index, d);
2825
2826   if (hf)
2827     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2828
2829   /* Ship frames to the worker nodes */
2830   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2831     {
2832       if (handoff_queue_elt_by_worker_index[i])
2833         {
2834           hf = handoff_queue_elt_by_worker_index[i];
2835           /*
2836            * It works better to let the handoff node
2837            * rate-adapt, always ship the handoff queue element.
2838            */
2839           if (1 || hf->n_vectors == hf->last_n_vectors)
2840             {
2841               vlib_put_frame_queue_elt (hf);
2842               handoff_queue_elt_by_worker_index[i] = 0;
2843             }
2844           else
2845             hf->last_n_vectors = hf->n_vectors;
2846         }
2847       congested_handoff_queue_by_worker_index[i] =
2848         (vlib_frame_queue_t *) (~0);
2849     }
2850   hf = 0;
2851   current_worker_index = ~0;
2852   return frame->n_vectors;
2853 }
2854
2855 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2856   .function = snat_out2in_worker_handoff_fn,
2857   .name = "nat44-out2in-worker-handoff",
2858   .vector_size = sizeof (u32),
2859   .format_trace = format_snat_out2in_worker_handoff_trace,
2860   .type = VLIB_NODE_TYPE_INTERNAL,
2861
2862   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2863   .error_strings = snat_out2in_error_strings,
2864
2865   .n_next_nodes = 1,
2866
2867   .next_nodes = {
2868     [0] = "error-drop",
2869   },
2870 };
2871
2872 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2873
2874 static uword
2875 snat_out2in_fast_node_fn (vlib_main_t * vm,
2876                           vlib_node_runtime_t * node,
2877                           vlib_frame_t * frame)
2878 {
2879   u32 n_left_from, * from, * to_next;
2880   snat_out2in_next_t next_index;
2881   u32 pkts_processed = 0;
2882   snat_main_t * sm = &snat_main;
2883
2884   from = vlib_frame_vector_args (frame);
2885   n_left_from = frame->n_vectors;
2886   next_index = node->cached_next_index;
2887
2888   while (n_left_from > 0)
2889     {
2890       u32 n_left_to_next;
2891
2892       vlib_get_next_frame (vm, node, next_index,
2893                            to_next, n_left_to_next);
2894
2895       while (n_left_from > 0 && n_left_to_next > 0)
2896         {
2897           u32 bi0;
2898           vlib_buffer_t * b0;
2899           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2900           u32 sw_if_index0;
2901           ip4_header_t * ip0;
2902           ip_csum_t sum0;
2903           u32 new_addr0, old_addr0;
2904           u16 new_port0, old_port0;
2905           udp_header_t * udp0;
2906           tcp_header_t * tcp0;
2907           icmp46_header_t * icmp0;
2908           snat_session_key_t key0, sm0;
2909           u32 proto0;
2910           u32 rx_fib_index0;
2911
2912           /* speculatively enqueue b0 to the current next frame */
2913           bi0 = from[0];
2914           to_next[0] = bi0;
2915           from += 1;
2916           to_next += 1;
2917           n_left_from -= 1;
2918           n_left_to_next -= 1;
2919
2920           b0 = vlib_get_buffer (vm, bi0);
2921
2922           ip0 = vlib_buffer_get_current (b0);
2923           udp0 = ip4_next_header (ip0);
2924           tcp0 = (tcp_header_t *) udp0;
2925           icmp0 = (icmp46_header_t *) udp0;
2926
2927           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2928           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2929
2930           vnet_feature_next (sw_if_index0, &next0, b0);
2931
2932           if (PREDICT_FALSE(ip0->ttl == 1))
2933             {
2934               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2935               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2936                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2937                                            0);
2938               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2939               goto trace00;
2940             }
2941
2942           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2943
2944           if (PREDICT_FALSE (proto0 == ~0))
2945               goto trace00;
2946
2947           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2948             {
2949               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2950                                   rx_fib_index0, node, next0, ~0, 0, 0);
2951               goto trace00;
2952             }
2953
2954           key0.addr = ip0->dst_address;
2955           key0.port = udp0->dst_port;
2956           key0.fib_index = rx_fib_index0;
2957
2958           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2959             {
2960               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2961               goto trace00;
2962             }
2963
2964           new_addr0 = sm0.addr.as_u32;
2965           new_port0 = sm0.port;
2966           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2967           old_addr0 = ip0->dst_address.as_u32;
2968           ip0->dst_address.as_u32 = new_addr0;
2969
2970           sum0 = ip0->checksum;
2971           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2972                                  ip4_header_t,
2973                                  dst_address /* changed member */);
2974           ip0->checksum = ip_csum_fold (sum0);
2975
2976           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2977             {
2978                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2979                 {
2980                   old_port0 = tcp0->dst_port;
2981                   tcp0->dst_port = new_port0;
2982
2983                   sum0 = tcp0->checksum;
2984                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2985                                          ip4_header_t,
2986                                          dst_address /* changed member */);
2987
2988                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2989                                          ip4_header_t /* cheat */,
2990                                          length /* changed member */);
2991                   tcp0->checksum = ip_csum_fold(sum0);
2992                 }
2993               else
2994                 {
2995                   old_port0 = udp0->dst_port;
2996                   udp0->dst_port = new_port0;
2997                   udp0->checksum = 0;
2998                 }
2999             }
3000           else
3001             {
3002               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3003                 {
3004                   sum0 = tcp0->checksum;
3005                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3006                                          ip4_header_t,
3007                                          dst_address /* changed member */);
3008
3009                   tcp0->checksum = ip_csum_fold(sum0);
3010                 }
3011             }
3012
3013         trace00:
3014
3015           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3016                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3017             {
3018               snat_out2in_trace_t *t =
3019                  vlib_add_trace (vm, node, b0, sizeof (*t));
3020               t->sw_if_index = sw_if_index0;
3021               t->next_index = next0;
3022             }
3023
3024           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3025
3026           /* verify speculative enqueue, maybe switch current next frame */
3027           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3028                                            to_next, n_left_to_next,
3029                                            bi0, next0);
3030         }
3031
3032       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3033     }
3034
3035   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3036                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3037                                pkts_processed);
3038   return frame->n_vectors;
3039 }
3040
3041 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3042   .function = snat_out2in_fast_node_fn,
3043   .name = "nat44-out2in-fast",
3044   .vector_size = sizeof (u32),
3045   .format_trace = format_snat_out2in_fast_trace,
3046   .type = VLIB_NODE_TYPE_INTERNAL,
3047
3048   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3049   .error_strings = snat_out2in_error_strings,
3050
3051   .runtime_data_bytes = sizeof (snat_runtime_t),
3052
3053   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3054
3055   /* edit / add dispositions here */
3056   .next_nodes = {
3057     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3058     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3059     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3060     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3061     [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3062   },
3063 };
3064 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);