src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28 #include <nat/nat_inlines.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38   u32 is_slow_path;
  39 } snat_in2out_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_in2out_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  52   char * tag;
  53
  54   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  55
  56   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  57               t->sw_if_index, t->next_index, t->session_index);
  58
  59   return s;
  60 }
  61
  62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  75 {
  76   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  77   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  78   snat_in2out_worker_handoff_trace_t * t =
  79     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  80   char * m;
  81
  82   m = t->do_handoff ? "next worker" : "same worker";
  83   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  84
  85   return s;
  86 }
  87
  88 typedef struct {
  89   u32 sw_if_index;
  90   u32 next_index;
  91   u8 cached;
  92 } nat44_in2out_reass_trace_t;
  93
  94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  95 {
  96   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  97   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  98   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  99
 100   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 101               t->sw_if_index, t->next_index,
 102               t->cached ? "cached" : "translated");
 103
 104   return s;
 105 }
 106
 107 vlib_node_registration_t snat_in2out_node;
 108 vlib_node_registration_t snat_in2out_slowpath_node;
 109 vlib_node_registration_t snat_in2out_fast_node;
 110 vlib_node_registration_t snat_in2out_worker_handoff_node;
 111 vlib_node_registration_t snat_det_in2out_node;
 112 vlib_node_registration_t snat_in2out_output_node;
 113 vlib_node_registration_t snat_in2out_output_slowpath_node;
 114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 115 vlib_node_registration_t snat_hairpin_dst_node;
 116 vlib_node_registration_t snat_hairpin_src_node;
 117 vlib_node_registration_t nat44_hairpinning_node;
 118 vlib_node_registration_t nat44_in2out_reass_node;
 119 vlib_node_registration_t nat44_ed_in2out_node;
 120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
 121 vlib_node_registration_t nat44_ed_in2out_output_node;
 122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
 123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
 124 vlib_node_registration_t nat44_ed_hairpin_src_node;
 125 vlib_node_registration_t nat44_ed_hairpinning_node;
 126
 127 #define foreach_snat_in2out_error                       \
 128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 129 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 130 _(OUT_OF_PORTS, "Out of ports")                         \
 131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 132 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 133 _(NO_TRANSLATION, "No translation")                     \
 134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 135 _(DROP_FRAGMENT, "Drop fragment")                       \
 136 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 138 _(FQ_CONGESTED, "Handoff frame queue congested")
 139
 140 typedef enum {
 141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 142   foreach_snat_in2out_error
 143 #undef _
 144   SNAT_IN2OUT_N_ERROR,
 145 } snat_in2out_error_t;
 146
 147 static char * snat_in2out_error_strings[] = {
 148 #define _(sym,string) string,
 149   foreach_snat_in2out_error
 150 #undef _
 151 };
 152
 153 typedef enum {
 154   SNAT_IN2OUT_NEXT_LOOKUP,
 155   SNAT_IN2OUT_NEXT_DROP,
 156   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 157   SNAT_IN2OUT_NEXT_SLOW_PATH,
 158   SNAT_IN2OUT_NEXT_REASS,
 159   SNAT_IN2OUT_N_NEXT,
 160 } snat_in2out_next_t;
 161
 162 typedef enum {
 163   SNAT_HAIRPIN_SRC_NEXT_DROP,
 164   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 165   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 166   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 167   SNAT_HAIRPIN_SRC_N_NEXT,
 168 } snat_hairpin_next_t;
 169
 170 /**
 171  * @brief Check if packet should be translated
 172  *
 173  * Packets aimed at outside interface and external address with active session
 174  * should be translated.
 175  *
 176  * @param sm            NAT main
 177  * @param rt            NAT runtime data
 178  * @param sw_if_index0  index of the inside interface
 179  * @param ip0           IPv4 header
 180  * @param proto0        NAT protocol
 181  * @param rx_fib_index0 RX FIB index
 182  *
 183  * @returns 0 if packet should be translated otherwise 1
 184  */
 185 static inline int
 186 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 187                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 188                          u32 rx_fib_index0)
 189 {
 190   if (sm->out2in_dpo)
 191     return 0;
 192
 193   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 194   nat_outside_fib_t *outside_fib;
 195   fib_prefix_t pfx = {
 196     .fp_proto = FIB_PROTOCOL_IP4,
 197     .fp_len = 32,
 198     .fp_addr = {
 199         .ip4.as_u32 = ip0->dst_address.as_u32,
 200     },
 201   };
 202
 203   /* Don't NAT packet aimed at the intfc address */
 204   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 205                                       ip0->dst_address.as_u32)))
 206     return 1;
 207
 208   fei = fib_table_lookup (rx_fib_index0, &pfx);
 209   if (FIB_NODE_INDEX_INVALID != fei)
 210     {
 211       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 212       if (sw_if_index == ~0)
 213         {
 214           vec_foreach (outside_fib, sm->outside_fibs)
 215             {
 216               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 217               if (FIB_NODE_INDEX_INVALID != fei)
 218                 {
 219                   sw_if_index = fib_entry_get_resolving_interface (fei);
 220                   if (sw_if_index != ~0)
 221                     break;
 222                 }
 223             }
 224         }
 225       if (sw_if_index == ~0)
 226         return 1;
 227
 228       snat_interface_t *i;
 229       pool_foreach (i, sm->interfaces,
 230       ({
 231         /* NAT packet aimed at outside interface */
 232         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 233           return 0;
 234       }));
 235     }
 236
 237   return 1;
 238 }
 239
 240 static inline int
 241 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 242                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 243                     u32 rx_fib_index0, u32 thread_index)
 244 {
 245   udp_header_t * udp0 = ip4_next_header (ip0);
 246   snat_session_key_t key0, sm0;
 247   clib_bihash_kv_8_8_t kv0, value0;
 248
 249   key0.addr = ip0->dst_address;
 250   key0.port = udp0->dst_port;
 251   key0.protocol = proto0;
 252   key0.fib_index = sm->outside_fib_index;
 253   kv0.key = key0.as_u64;
 254
 255   /* NAT packet aimed at external address if */
 256   /* has active sessions */
 257   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 258                               &value0))
 259     {
 260       /* or is static mappings */
 261       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 262         return 0;
 263     }
 264   else
 265     return 0;
 266
 267   if (sm->forwarding_enabled)
 268     return 1;
 269
 270   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 271                                  rx_fib_index0);
 272 }
 273
 274 static inline int
 275 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 276                                   u32 proto0, u16 src_port, u16 dst_port,
 277                                   u32 thread_index, u32 sw_if_index)
 278 {
 279   snat_session_key_t key0;
 280   clib_bihash_kv_8_8_t kv0, value0;
 281   snat_interface_t *i;
 282
 283   /* src NAT check */
 284   key0.addr = ip0->src_address;
 285   key0.port = src_port;
 286   key0.protocol = proto0;
 287   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 288   kv0.key = key0.as_u64;
 289
 290   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 291                                &value0))
 292     return 1;
 293
 294   /* dst NAT check */
 295   key0.addr = ip0->dst_address;
 296   key0.port = dst_port;
 297   key0.protocol = proto0;
 298   kv0.key = key0.as_u64;
 299   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 300                                &value0))
 301   {
 302     /* hairpinning */
 303     pool_foreach (i, sm->output_feature_interfaces,
 304     ({
 305       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 306         return 0;
 307     }));
 308     return 1;
 309   }
 310
 311   return 0;
 312 }
 313
 314 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 315                       ip4_header_t * ip0,
 316                       u32 rx_fib_index0,
 317                       snat_session_key_t * key0,
 318                       snat_session_t ** sessionp,
 319                       vlib_node_runtime_t * node,
 320                       u32 next0,
 321                       u32 thread_index)
 322 {
 323   snat_user_t *u;
 324   snat_session_t *s;
 325   clib_bihash_kv_8_8_t kv0;
 326   snat_session_key_t key1;
 327   u32 address_index = ~0;
 328   udp_header_t * udp0 = ip4_next_header (ip0);
 329   u8 is_sm = 0;
 330   nat_outside_fib_t *outside_fib;
 331   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 332   fib_prefix_t pfx = {
 333     .fp_proto = FIB_PROTOCOL_IP4,
 334     .fp_len = 32,
 335     .fp_addr = {
 336         .ip4.as_u32 = ip0->dst_address.as_u32,
 337     },
 338   };
 339
 340   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 341     {
 342       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 343       nat_ipfix_logging_max_sessions(sm->max_translations);
 344       nat_log_notice ("maximum sessions exceeded");
 345       return SNAT_IN2OUT_NEXT_DROP;
 346     }
 347
 348   key1.protocol = key0->protocol;
 349
 350   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 351                               thread_index);
 352   if (!u)
 353     {
 354       nat_log_warn ("create NAT user failed");
 355       return SNAT_IN2OUT_NEXT_DROP;
 356     }
 357
 358   /* First try to match static mapping by local address and port */
 359   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 360     {
 361       /* Try to create dynamic translation */
 362       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 363                                                thread_index, &key1,
 364                                                &address_index,
 365                                                sm->port_per_thread,
 366                                                sm->per_thread_data[thread_index].snat_thread_index))
 367         {
 368           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 369           return SNAT_IN2OUT_NEXT_DROP;
 370         }
 371     }
 372   else
 373     is_sm = 1;
 374
 375   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 376   if (!s)
 377     {
 378       nat_log_warn ("create NAT session failed");
 379       return SNAT_IN2OUT_NEXT_DROP;
 380     }
 381
 382   if (is_sm)
 383     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 384   user_session_increment (sm, u, is_sm);
 385   s->outside_address_index = address_index;
 386   s->in2out = *key0;
 387   s->out2in = key1;
 388   s->out2in.protocol = key0->protocol;
 389   s->out2in.fib_index = sm->outside_fib_index;
 390   switch (vec_len (sm->outside_fibs))
 391     {
 392     case 0:
 393       s->out2in.fib_index = sm->outside_fib_index;
 394       break;
 395     case 1:
 396       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 397       break;
 398     default:
 399       vec_foreach (outside_fib, sm->outside_fibs)
 400         {
 401           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 402           if (FIB_NODE_INDEX_INVALID != fei)
 403             {
 404               if (fib_entry_get_resolving_interface (fei) != ~0)
 405                 {
 406                   s->out2in.fib_index = outside_fib->fib_index;
 407                   break;
 408                 }
 409             }
 410         }
 411       break;
 412     }
 413   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 414   s->ext_host_port = udp0->dst_port;
 415   *sessionp = s;
 416
 417   /* Add to translation hashes */
 418   kv0.key = s->in2out.as_u64;
 419   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 420   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 421                                1 /* is_add */))
 422       nat_log_notice ("in2out key add failed");
 423
 424   kv0.key = s->out2in.as_u64;
 425   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 426
 427   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 428                                1 /* is_add */))
 429       nat_log_notice ("out2in key add failed");
 430
 431   /* log NAT event */
 432   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 433                                       s->out2in.addr.as_u32,
 434                                       s->in2out.protocol,
 435                                       s->in2out.port,
 436                                       s->out2in.port,
 437                                       s->in2out.fib_index);
 438   return next0;
 439 }
 440
 441 static_always_inline
 442 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 443                                  snat_session_key_t *p_key0)
 444 {
 445   icmp46_header_t *icmp0;
 446   snat_session_key_t key0;
 447   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 448   ip4_header_t *inner_ip0 = 0;
 449   void *l4_header = 0;
 450   icmp46_header_t *inner_icmp0;
 451
 452   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 453   echo0 = (icmp_echo_header_t *)(icmp0+1);
 454
 455   if (!icmp_is_error_message (icmp0))
 456     {
 457       key0.protocol = SNAT_PROTOCOL_ICMP;
 458       key0.addr = ip0->src_address;
 459       key0.port = echo0->identifier;
 460     }
 461   else
 462     {
 463       inner_ip0 = (ip4_header_t *)(echo0+1);
 464       l4_header = ip4_next_header (inner_ip0);
 465       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 466       key0.addr = inner_ip0->dst_address;
 467       switch (key0.protocol)
 468         {
 469         case SNAT_PROTOCOL_ICMP:
 470           inner_icmp0 = (icmp46_header_t*)l4_header;
 471           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 472           key0.port = inner_echo0->identifier;
 473           break;
 474         case SNAT_PROTOCOL_UDP:
 475         case SNAT_PROTOCOL_TCP:
 476           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 477           break;
 478         default:
 479           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 480         }
 481     }
 482   *p_key0 = key0;
 483   return -1; /* success */
 484 }
 485
 486 /**
 487  * Get address and port values to be used for ICMP packet translation
 488  * and create session if needed
 489  *
 490  * @param[in,out] sm             NAT main
 491  * @param[in,out] node           NAT node runtime
 492  * @param[in] thread_index       thread index
 493  * @param[in,out] b0             buffer containing packet to be translated
 494  * @param[out] p_proto           protocol used for matching
 495  * @param[out] p_value           address and port after NAT translation
 496  * @param[out] p_dont_translate  if packet should not be translated
 497  * @param d                      optional parameter
 498  * @param e                      optional parameter
 499  */
 500 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 501                            u32 thread_index, vlib_buffer_t *b0,
 502                            ip4_header_t *ip0, u8 *p_proto,
 503                            snat_session_key_t *p_value,
 504                            u8 *p_dont_translate, void *d, void *e)
 505 {
 506   icmp46_header_t *icmp0;
 507   u32 sw_if_index0;
 508   u32 rx_fib_index0;
 509   snat_session_key_t key0;
 510   snat_session_t *s0 = 0;
 511   u8 dont_translate = 0;
 512   clib_bihash_kv_8_8_t kv0, value0;
 513   u32 next0 = ~0;
 514   int err;
 515
 516   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 517   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 518   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 519
 520   err = icmp_get_key (ip0, &key0);
 521   if (err != -1)
 522     {
 523       b0->error = node->errors[err];
 524       next0 = SNAT_IN2OUT_NEXT_DROP;
 525       goto out;
 526     }
 527   key0.fib_index = rx_fib_index0;
 528
 529   kv0.key = key0.as_u64;
 530
 531   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 532                               &value0))
 533     {
 534       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 535         {
 536           if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
 537               key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
 538             {
 539               dont_translate = 1;
 540               goto out;
 541             }
 542         }
 543       else
 544         {
 545           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 546               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 547             {
 548               dont_translate = 1;
 549               goto out;
 550             }
 551         }
 552
 553       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 554         {
 555           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 556           next0 = SNAT_IN2OUT_NEXT_DROP;
 557           goto out;
 558         }
 559
 560       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 561                          &s0, node, next0, thread_index);
 562
 563       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 564         goto out;
 565     }
 566   else
 567     {
 568       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 569                         icmp0->type != ICMP4_echo_reply &&
 570                         !icmp_is_error_message (icmp0)))
 571         {
 572           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 573           next0 = SNAT_IN2OUT_NEXT_DROP;
 574           goto out;
 575         }
 576
 577       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 578                               value0.value);
 579     }
 580
 581 out:
 582   *p_proto = key0.protocol;
 583   if (s0)
 584     *p_value = s0->out2in;
 585   *p_dont_translate = dont_translate;
 586   if (d)
 587     *(snat_session_t**)d = s0;
 588   return next0;
 589 }
 590
 591 /**
 592  * Get address and port values to be used for ICMP packet translation
 593  *
 594  * @param[in] sm                 NAT main
 595  * @param[in,out] node           NAT node runtime
 596  * @param[in] thread_index       thread index
 597  * @param[in,out] b0             buffer containing packet to be translated
 598  * @param[out] p_proto           protocol used for matching
 599  * @param[out] p_value           address and port after NAT translation
 600  * @param[out] p_dont_translate  if packet should not be translated
 601  * @param d                      optional parameter
 602  * @param e                      optional parameter
 603  */
 604 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 605                            u32 thread_index, vlib_buffer_t *b0,
 606                            ip4_header_t *ip0, u8 *p_proto,
 607                            snat_session_key_t *p_value,
 608                            u8 *p_dont_translate, void *d, void *e)
 609 {
 610   icmp46_header_t *icmp0;
 611   u32 sw_if_index0;
 612   u32 rx_fib_index0;
 613   snat_session_key_t key0;
 614   snat_session_key_t sm0;
 615   u8 dont_translate = 0;
 616   u8 is_addr_only;
 617   u32 next0 = ~0;
 618   int err;
 619
 620   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 621   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 622   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 623
 624   err = icmp_get_key (ip0, &key0);
 625   if (err != -1)
 626     {
 627       b0->error = node->errors[err];
 628       next0 = SNAT_IN2OUT_NEXT_DROP;
 629       goto out2;
 630     }
 631   key0.fib_index = rx_fib_index0;
 632
 633   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 634     {
 635       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 636           IP_PROTOCOL_ICMP, rx_fib_index0)))
 637         {
 638           dont_translate = 1;
 639           goto out;
 640         }
 641
 642       if (icmp_is_error_message (icmp0))
 643         {
 644           next0 = SNAT_IN2OUT_NEXT_DROP;
 645           goto out;
 646         }
 647
 648       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 649       next0 = SNAT_IN2OUT_NEXT_DROP;
 650       goto out;
 651     }
 652
 653   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 654                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 655                     !icmp_is_error_message (icmp0)))
 656     {
 657       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 658       next0 = SNAT_IN2OUT_NEXT_DROP;
 659       goto out;
 660     }
 661
 662 out:
 663   *p_value = sm0;
 664 out2:
 665   *p_proto = key0.protocol;
 666   *p_dont_translate = dont_translate;
 667   return next0;
 668 }
 669
 670 static inline u32 icmp_in2out (snat_main_t *sm,
 671                                vlib_buffer_t * b0,
 672                                ip4_header_t * ip0,
 673                                icmp46_header_t * icmp0,
 674                                u32 sw_if_index0,
 675                                u32 rx_fib_index0,
 676                                vlib_node_runtime_t * node,
 677                                u32 next0,
 678                                u32 thread_index,
 679                                void *d,
 680                                void *e)
 681 {
 682   snat_session_key_t sm0;
 683   u8 protocol;
 684   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 685   ip4_header_t *inner_ip0;
 686   void *l4_header = 0;
 687   icmp46_header_t *inner_icmp0;
 688   u8 dont_translate;
 689   u32 new_addr0, old_addr0;
 690   u16 old_id0, new_id0;
 691   ip_csum_t sum0;
 692   u16 checksum0;
 693   u32 next0_tmp;
 694
 695   echo0 = (icmp_echo_header_t *)(icmp0+1);
 696
 697   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 698                                        &protocol, &sm0, &dont_translate, d, e);
 699   if (next0_tmp != ~0)
 700     next0 = next0_tmp;
 701   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 702     goto out;
 703
 704   sum0 = ip_incremental_checksum (0, icmp0,
 705                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 706   checksum0 = ~ip_csum_fold (sum0);
 707   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 708     {
 709       next0 = SNAT_IN2OUT_NEXT_DROP;
 710       goto out;
 711     }
 712
 713   old_addr0 = ip0->src_address.as_u32;
 714   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 715   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 716     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 717
 718   sum0 = ip0->checksum;
 719   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 720                          src_address /* changed member */);
 721   ip0->checksum = ip_csum_fold (sum0);
 722
 723   if (icmp0->checksum == 0)
 724     icmp0->checksum = 0xffff;
 725
 726   if (!icmp_is_error_message (icmp0))
 727     {
 728       new_id0 = sm0.port;
 729       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 730         {
 731           old_id0 = echo0->identifier;
 732           new_id0 = sm0.port;
 733           echo0->identifier = new_id0;
 734
 735           sum0 = icmp0->checksum;
 736           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 737                                  identifier);
 738           icmp0->checksum = ip_csum_fold (sum0);
 739         }
 740     }
 741   else
 742     {
 743       inner_ip0 = (ip4_header_t *)(echo0+1);
 744       l4_header = ip4_next_header (inner_ip0);
 745
 746       if (!ip4_header_checksum_is_valid (inner_ip0))
 747         {
 748           next0 = SNAT_IN2OUT_NEXT_DROP;
 749           goto out;
 750         }
 751
 752       old_addr0 = inner_ip0->dst_address.as_u32;
 753       inner_ip0->dst_address = sm0.addr;
 754       new_addr0 = inner_ip0->dst_address.as_u32;
 755
 756       sum0 = icmp0->checksum;
 757       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 758                              dst_address /* changed member */);
 759       icmp0->checksum = ip_csum_fold (sum0);
 760
 761       switch (protocol)
 762         {
 763           case SNAT_PROTOCOL_ICMP:
 764             inner_icmp0 = (icmp46_header_t*)l4_header;
 765             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 766
 767             old_id0 = inner_echo0->identifier;
 768             new_id0 = sm0.port;
 769             inner_echo0->identifier = new_id0;
 770
 771             sum0 = icmp0->checksum;
 772             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 773                                    identifier);
 774             icmp0->checksum = ip_csum_fold (sum0);
 775             break;
 776           case SNAT_PROTOCOL_UDP:
 777           case SNAT_PROTOCOL_TCP:
 778             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 779             new_id0 = sm0.port;
 780             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 781
 782             sum0 = icmp0->checksum;
 783             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 784                                    dst_port);
 785             icmp0->checksum = ip_csum_fold (sum0);
 786             break;
 787           default:
 788             ASSERT(0);
 789         }
 790     }
 791
 792 out:
 793   return next0;
 794 }
 795
 796 /**
 797  * @brief Hairpinning
 798  *
 799  * Hairpinning allows two endpoints on the internal side of the NAT to
 800  * communicate even if they only use each other's external IP addresses
 801  * and ports.
 802  *
 803  * @param sm     NAT main.
 804  * @param b0     Vlib buffer.
 805  * @param ip0    IP header.
 806  * @param udp0   UDP header.
 807  * @param tcp0   TCP header.
 808  * @param proto0 NAT protocol.
 809  */
 810 static inline int
 811 snat_hairpinning (snat_main_t *sm,
 812                   vlib_buffer_t * b0,
 813                   ip4_header_t * ip0,
 814                   udp_header_t * udp0,
 815                   tcp_header_t * tcp0,
 816                   u32 proto0,
 817                   int is_ed)
 818 {
 819   snat_session_key_t key0, sm0;
 820   snat_session_t * s0;
 821   clib_bihash_kv_8_8_t kv0, value0;
 822   ip_csum_t sum0;
 823   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 824   u16 new_dst_port0, old_dst_port0;
 825   int rv;
 826
 827   key0.addr = ip0->dst_address;
 828   key0.port = udp0->dst_port;
 829   key0.protocol = proto0;
 830   key0.fib_index = sm->outside_fib_index;
 831   kv0.key = key0.as_u64;
 832
 833   /* Check if destination is static mappings */
 834   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 835     {
 836       new_dst_addr0 = sm0.addr.as_u32;
 837       new_dst_port0 = sm0.port;
 838       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 839     }
 840   /* or active session */
 841   else
 842     {
 843       if (sm->num_workers > 1)
 844         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 845       else
 846         ti = sm->num_workers;
 847
 848       if (is_ed)
 849         {
 850           clib_bihash_kv_16_8_t ed_kv, ed_value;
 851           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 852                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 853                       udp0->src_port);
 854           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 855                                         &ed_kv, &ed_value);
 856           si = ed_value.value;
 857         }
 858       else
 859         {
 860           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 861                                        &value0);
 862           si = value0.value;
 863         }
 864       if (rv)
 865         return 0;
 866
 867       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 868       new_dst_addr0 = s0->in2out.addr.as_u32;
 869       new_dst_port0 = s0->in2out.port;
 870       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 871     }
 872
 873   /* Destination is behind the same NAT, use internal address and port */
 874   if (new_dst_addr0)
 875     {
 876       old_dst_addr0 = ip0->dst_address.as_u32;
 877       ip0->dst_address.as_u32 = new_dst_addr0;
 878       sum0 = ip0->checksum;
 879       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 880                              ip4_header_t, dst_address);
 881       ip0->checksum = ip_csum_fold (sum0);
 882
 883       old_dst_port0 = tcp0->dst;
 884       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 885         {
 886           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 887             {
 888               tcp0->dst = new_dst_port0;
 889               sum0 = tcp0->checksum;
 890               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 891                                      ip4_header_t, dst_address);
 892               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 893                                      ip4_header_t /* cheat */, length);
 894               tcp0->checksum = ip_csum_fold(sum0);
 895             }
 896           else
 897             {
 898               udp0->dst_port = new_dst_port0;
 899               udp0->checksum = 0;
 900             }
 901         }
 902       else
 903         {
 904           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 905             {
 906               sum0 = tcp0->checksum;
 907               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 908                                      ip4_header_t, dst_address);
 909               tcp0->checksum = ip_csum_fold(sum0);
 910             }
 911         }
 912       return 1;
 913     }
 914   return 0;
 915 }
 916
 917 static inline void
 918 snat_icmp_hairpinning (snat_main_t *sm,
 919                        vlib_buffer_t * b0,
 920                        ip4_header_t * ip0,
 921                        icmp46_header_t * icmp0,
 922                        int is_ed)
 923 {
 924   snat_session_key_t key0, sm0;
 925   clib_bihash_kv_8_8_t kv0, value0;
 926   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 927   ip_csum_t sum0;
 928   snat_session_t *s0;
 929   int rv;
 930
 931   if (!icmp_is_error_message (icmp0))
 932     {
 933       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 934       u16 icmp_id0 = echo0->identifier;
 935       key0.addr = ip0->dst_address;
 936       key0.port = icmp_id0;
 937       key0.protocol = SNAT_PROTOCOL_ICMP;
 938       key0.fib_index = sm->outside_fib_index;
 939       kv0.key = key0.as_u64;
 940
 941       if (sm->num_workers > 1)
 942         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 943       else
 944         ti = sm->num_workers;
 945
 946       /* Check if destination is in active sessions */
 947       if (is_ed)
 948         {
 949           clib_bihash_kv_16_8_t ed_kv, ed_value;
 950           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 951                       IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
 952           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 953                                         &ed_kv, &ed_value);
 954           si = ed_value.value;
 955         }
 956       else
 957         {
 958           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 959                                        &value0);
 960           si = value0.value;
 961         }
 962       if (rv)
 963         {
 964           /* or static mappings */
 965           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 966             {
 967               new_dst_addr0 = sm0.addr.as_u32;
 968               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 969             }
 970         }
 971       else
 972         {
 973           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 974           new_dst_addr0 = s0->in2out.addr.as_u32;
 975           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 976           echo0->identifier = s0->in2out.port;
 977           sum0 = icmp0->checksum;
 978           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 979                                  icmp_echo_header_t, identifier);
 980           icmp0->checksum = ip_csum_fold (sum0);
 981         }
 982
 983       /* Destination is behind the same NAT, use internal address and port */
 984       if (new_dst_addr0)
 985         {
 986           old_dst_addr0 = ip0->dst_address.as_u32;
 987           ip0->dst_address.as_u32 = new_dst_addr0;
 988           sum0 = ip0->checksum;
 989           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 990                                  ip4_header_t, dst_address);
 991           ip0->checksum = ip_csum_fold (sum0);
 992         }
 993     }
 994
 995 }
 996
 997 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 998                                          vlib_buffer_t * b0,
 999                                          ip4_header_t * ip0,
1000                                          icmp46_header_t * icmp0,
1001                                          u32 sw_if_index0,
1002                                          u32 rx_fib_index0,
1003                                          vlib_node_runtime_t * node,
1004                                          u32 next0,
1005                                          f64 now,
1006                                          u32 thread_index,
1007                                          snat_session_t ** p_s0)
1008 {
1009   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1010                       next0, thread_index, p_s0, 0);
1011   snat_session_t * s0 = *p_s0;
1012   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1013     {
1014       /* Hairpinning */
1015       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1016         snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1017       /* Accounting */
1018       nat44_session_update_counters (s0, now,
1019                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
1020       /* Per-user LRU list maintenance */
1021       nat44_session_update_lru (sm, s0, thread_index);
1022     }
1023   return next0;
1024 }
1025
1026 static inline void
1027 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1028                                   vlib_buffer_t * b,
1029                                   ip4_header_t * ip)
1030 {
1031   clib_bihash_kv_8_8_t kv, value;
1032   snat_static_mapping_t *m;
1033   u32 old_addr, new_addr;
1034   ip_csum_t sum;
1035
1036   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1037   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1038     return;
1039
1040   m = pool_elt_at_index (sm->static_mappings, value.value);
1041
1042   old_addr = ip->dst_address.as_u32;
1043   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1044   sum = ip->checksum;
1045   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1046   ip->checksum = ip_csum_fold (sum);
1047
1048   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1049     vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1050 }
1051
1052 static int
1053 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1054                              vlib_buffer_t * b,
1055                              ip4_header_t * ip,
1056                              u32 rx_fib_index)
1057 {
1058   clib_bihash_kv_8_8_t kv, value;
1059   snat_static_mapping_t *m;
1060   snat_session_key_t m_key;
1061   u32 old_addr, new_addr;
1062   ip_csum_t sum;
1063
1064   m_key.addr = ip->src_address;
1065   m_key.port = 0;
1066   m_key.protocol = 0;
1067   m_key.fib_index = rx_fib_index;
1068   kv.key = m_key.as_u64;
1069   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1070     return 1;
1071
1072   m = pool_elt_at_index (sm->static_mappings, value.value);
1073
1074   old_addr = ip->src_address.as_u32;
1075   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1076   sum = ip->checksum;
1077   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1078   ip->checksum = ip_csum_fold (sum);
1079
1080
1081   /* Hairpinning */
1082   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1083     {
1084       vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1085       nat_hairpinning_sm_unknown_proto (sm, b, ip);
1086     }
1087
1088   return 0;
1089 }
1090
1091 static inline uword
1092 snat_in2out_node_fn_inline (vlib_main_t * vm,
1093                             vlib_node_runtime_t * node,
1094                             vlib_frame_t * frame, int is_slow_path,
1095                             int is_output_feature)
1096 {
1097   u32 n_left_from, * from, * to_next;
1098   snat_in2out_next_t next_index;
1099   u32 pkts_processed = 0;
1100   snat_main_t * sm = &snat_main;
1101   f64 now = vlib_time_now (vm);
1102   u32 stats_node_index;
1103   u32 thread_index = vm->thread_index;
1104
1105   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1106     snat_in2out_node.index;
1107
1108   from = vlib_frame_vector_args (frame);
1109   n_left_from = frame->n_vectors;
1110   next_index = node->cached_next_index;
1111
1112   while (n_left_from > 0)
1113     {
1114       u32 n_left_to_next;
1115
1116       vlib_get_next_frame (vm, node, next_index,
1117                            to_next, n_left_to_next);
1118
1119       while (n_left_from >= 4 && n_left_to_next >= 2)
1120         {
1121           u32 bi0, bi1;
1122           vlib_buffer_t * b0, * b1;
1123           u32 next0, next1;
1124           u32 sw_if_index0, sw_if_index1;
1125           ip4_header_t * ip0, * ip1;
1126           ip_csum_t sum0, sum1;
1127           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1128           u16 old_port0, new_port0, old_port1, new_port1;
1129           udp_header_t * udp0, * udp1;
1130           tcp_header_t * tcp0, * tcp1;
1131           icmp46_header_t * icmp0, * icmp1;
1132           snat_session_key_t key0, key1;
1133           u32 rx_fib_index0, rx_fib_index1;
1134           u32 proto0, proto1;
1135           snat_session_t * s0 = 0, * s1 = 0;
1136           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1137           u32 iph_offset0 = 0, iph_offset1 = 0;
1138
1139           /* Prefetch next iteration. */
1140           {
1141             vlib_buffer_t * p2, * p3;
1142
1143             p2 = vlib_get_buffer (vm, from[2]);
1144             p3 = vlib_get_buffer (vm, from[3]);
1145
1146             vlib_prefetch_buffer_header (p2, LOAD);
1147             vlib_prefetch_buffer_header (p3, LOAD);
1148
1149             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1150             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1151           }
1152
1153           /* speculatively enqueue b0 and b1 to the current next frame */
1154           to_next[0] = bi0 = from[0];
1155           to_next[1] = bi1 = from[1];
1156           from += 2;
1157           to_next += 2;
1158           n_left_from -= 2;
1159           n_left_to_next -= 2;
1160
1161           b0 = vlib_get_buffer (vm, bi0);
1162           b1 = vlib_get_buffer (vm, bi1);
1163
1164           if (is_output_feature)
1165             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1166
1167           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1168                  iph_offset0);
1169
1170           udp0 = ip4_next_header (ip0);
1171           tcp0 = (tcp_header_t *) udp0;
1172           icmp0 = (icmp46_header_t *) udp0;
1173
1174           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1175           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1176                                    sw_if_index0);
1177
1178           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1179
1180           if (PREDICT_FALSE(ip0->ttl == 1))
1181             {
1182               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1183               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1184                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1185                                            0);
1186               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1187               goto trace00;
1188             }
1189
1190           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1191
1192           /* Next configured feature, probably ip4-lookup */
1193           if (is_slow_path)
1194             {
1195               if (PREDICT_FALSE (proto0 == ~0))
1196                 {
1197                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1198                     {
1199                       next0 = SNAT_IN2OUT_NEXT_DROP;
1200                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1201                     }
1202                   goto trace00;
1203                 }
1204
1205               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1206                 {
1207                   next0 = icmp_in2out_slow_path
1208                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1209                      node, next0, now, thread_index, &s0);
1210                   goto trace00;
1211                 }
1212             }
1213           else
1214             {
1215               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1216                 {
1217                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1218                   goto trace00;
1219                 }
1220
1221               if (ip4_is_fragment (ip0))
1222                 {
1223                   next0 = SNAT_IN2OUT_NEXT_REASS;
1224                   goto trace00;
1225                 }
1226             }
1227
1228           key0.addr = ip0->src_address;
1229           key0.port = udp0->src_port;
1230           key0.protocol = proto0;
1231           key0.fib_index = rx_fib_index0;
1232
1233           kv0.key = key0.as_u64;
1234
1235           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1236               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1237             {
1238               if (is_slow_path)
1239                 {
1240                   if (is_output_feature)
1241                     {
1242                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1243                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1244                         goto trace00;
1245                     }
1246                   else
1247                     {
1248                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1249                           ip0, proto0, rx_fib_index0, thread_index)))
1250                         goto trace00;
1251                     }
1252
1253                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1254                                      &s0, node, next0, thread_index);
1255                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1256                     goto trace00;
1257                 }
1258               else
1259                 {
1260                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1261                   goto trace00;
1262                 }
1263             }
1264           else
1265             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1266                                     value0.value);
1267
1268           b0->flags |= VNET_BUFFER_F_IS_NATED;
1269
1270           old_addr0 = ip0->src_address.as_u32;
1271           ip0->src_address = s0->out2in.addr;
1272           new_addr0 = ip0->src_address.as_u32;
1273           if (!is_output_feature)
1274             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1275
1276           sum0 = ip0->checksum;
1277           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1278                                  ip4_header_t,
1279                                  src_address /* changed member */);
1280           ip0->checksum = ip_csum_fold (sum0);
1281
1282           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1283             {
1284               old_port0 = tcp0->src_port;
1285               tcp0->src_port = s0->out2in.port;
1286               new_port0 = tcp0->src_port;
1287
1288               sum0 = tcp0->checksum;
1289               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1290                                      ip4_header_t,
1291                                      dst_address /* changed member */);
1292               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1293                                      ip4_header_t /* cheat */,
1294                                      length /* changed member */);
1295               tcp0->checksum = ip_csum_fold(sum0);
1296             }
1297           else
1298             {
1299               old_port0 = udp0->src_port;
1300               udp0->src_port = s0->out2in.port;
1301               udp0->checksum = 0;
1302             }
1303
1304           /* Accounting */
1305           nat44_session_update_counters (s0, now,
1306                                          vlib_buffer_length_in_chain (vm, b0));
1307           /* Per-user LRU list maintenance */
1308           nat44_session_update_lru (sm, s0, thread_index);
1309         trace00:
1310
1311           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1312                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1313             {
1314               snat_in2out_trace_t *t =
1315                  vlib_add_trace (vm, node, b0, sizeof (*t));
1316               t->is_slow_path = is_slow_path;
1317               t->sw_if_index = sw_if_index0;
1318               t->next_index = next0;
1319                   t->session_index = ~0;
1320               if (s0)
1321                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1322             }
1323
1324           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1325
1326           if (is_output_feature)
1327             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1328
1329           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1330                  iph_offset1);
1331
1332           udp1 = ip4_next_header (ip1);
1333           tcp1 = (tcp_header_t *) udp1;
1334           icmp1 = (icmp46_header_t *) udp1;
1335
1336           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1337           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1338                                    sw_if_index1);
1339
1340           if (PREDICT_FALSE(ip1->ttl == 1))
1341             {
1342               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1343               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1344                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1345                                            0);
1346               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1347               goto trace01;
1348             }
1349
1350           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1351
1352           /* Next configured feature, probably ip4-lookup */
1353           if (is_slow_path)
1354             {
1355               if (PREDICT_FALSE (proto1 == ~0))
1356                 {
1357                   if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1358                     {
1359                       next1 = SNAT_IN2OUT_NEXT_DROP;
1360                       b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1361                     }
1362                   goto trace01;
1363                 }
1364
1365               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1366                 {
1367                   next1 = icmp_in2out_slow_path
1368                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1369                      next1, now, thread_index, &s1);
1370                   goto trace01;
1371                 }
1372             }
1373           else
1374             {
1375               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1376                 {
1377                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1378                   goto trace01;
1379                 }
1380
1381               if (ip4_is_fragment (ip1))
1382                 {
1383                   next1 = SNAT_IN2OUT_NEXT_REASS;
1384                   goto trace01;
1385                 }
1386             }
1387
1388           key1.addr = ip1->src_address;
1389           key1.port = udp1->src_port;
1390           key1.protocol = proto1;
1391           key1.fib_index = rx_fib_index1;
1392
1393           kv1.key = key1.as_u64;
1394
1395             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1396                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1397             {
1398               if (is_slow_path)
1399                 {
1400                   if (is_output_feature)
1401                     {
1402                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1403                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1404                         goto trace01;
1405                     }
1406                   else
1407                     {
1408                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1409                           ip1, proto1, rx_fib_index1, thread_index)))
1410                         goto trace01;
1411                     }
1412
1413                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1414                                      &s1, node, next1, thread_index);
1415                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1416                     goto trace01;
1417                 }
1418               else
1419                 {
1420                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1421                   goto trace01;
1422                 }
1423             }
1424           else
1425             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1426                                     value1.value);
1427
1428           b1->flags |= VNET_BUFFER_F_IS_NATED;
1429
1430           old_addr1 = ip1->src_address.as_u32;
1431           ip1->src_address = s1->out2in.addr;
1432           new_addr1 = ip1->src_address.as_u32;
1433           if (!is_output_feature)
1434             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1435
1436           sum1 = ip1->checksum;
1437           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1438                                  ip4_header_t,
1439                                  src_address /* changed member */);
1440           ip1->checksum = ip_csum_fold (sum1);
1441
1442           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1443             {
1444               old_port1 = tcp1->src_port;
1445               tcp1->src_port = s1->out2in.port;
1446               new_port1 = tcp1->src_port;
1447
1448               sum1 = tcp1->checksum;
1449               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1450                                      ip4_header_t,
1451                                      dst_address /* changed member */);
1452               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1453                                      ip4_header_t /* cheat */,
1454                                      length /* changed member */);
1455               tcp1->checksum = ip_csum_fold(sum1);
1456             }
1457           else
1458             {
1459               old_port1 = udp1->src_port;
1460               udp1->src_port = s1->out2in.port;
1461               udp1->checksum = 0;
1462             }
1463
1464           /* Accounting */
1465           nat44_session_update_counters (s1, now,
1466                                          vlib_buffer_length_in_chain (vm, b1));
1467           /* Per-user LRU list maintenance */
1468           nat44_session_update_lru (sm, s1, thread_index);
1469         trace01:
1470
1471           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1472                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1473             {
1474               snat_in2out_trace_t *t =
1475                  vlib_add_trace (vm, node, b1, sizeof (*t));
1476               t->sw_if_index = sw_if_index1;
1477               t->next_index = next1;
1478               t->session_index = ~0;
1479               if (s1)
1480                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1481             }
1482
1483           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1484
1485           /* verify speculative enqueues, maybe switch current next frame */
1486           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1487                                            to_next, n_left_to_next,
1488                                            bi0, bi1, next0, next1);
1489         }
1490
1491       while (n_left_from > 0 && n_left_to_next > 0)
1492         {
1493           u32 bi0;
1494           vlib_buffer_t * b0;
1495           u32 next0;
1496           u32 sw_if_index0;
1497           ip4_header_t * ip0;
1498           ip_csum_t sum0;
1499           u32 new_addr0, old_addr0;
1500           u16 old_port0, new_port0;
1501           udp_header_t * udp0;
1502           tcp_header_t * tcp0;
1503           icmp46_header_t * icmp0;
1504           snat_session_key_t key0;
1505           u32 rx_fib_index0;
1506           u32 proto0;
1507           snat_session_t * s0 = 0;
1508           clib_bihash_kv_8_8_t kv0, value0;
1509           u32 iph_offset0 = 0;
1510
1511           /* speculatively enqueue b0 to the current next frame */
1512           bi0 = from[0];
1513           to_next[0] = bi0;
1514           from += 1;
1515           to_next += 1;
1516           n_left_from -= 1;
1517           n_left_to_next -= 1;
1518
1519           b0 = vlib_get_buffer (vm, bi0);
1520           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1521
1522           if (is_output_feature)
1523             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1524
1525           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1526                  iph_offset0);
1527
1528           udp0 = ip4_next_header (ip0);
1529           tcp0 = (tcp_header_t *) udp0;
1530           icmp0 = (icmp46_header_t *) udp0;
1531
1532           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1533           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1534                                    sw_if_index0);
1535
1536           if (PREDICT_FALSE(ip0->ttl == 1))
1537             {
1538               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1539               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1540                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1541                                            0);
1542               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1543               goto trace0;
1544             }
1545
1546           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1547
1548           /* Next configured feature, probably ip4-lookup */
1549           if (is_slow_path)
1550             {
1551               if (PREDICT_FALSE (proto0 == ~0))
1552                 {
1553                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1554                     {
1555                       next0 = SNAT_IN2OUT_NEXT_DROP;
1556                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1557                     }
1558                   goto trace0;
1559                 }
1560
1561               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1562                 {
1563                   next0 = icmp_in2out_slow_path
1564                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1565                      next0, now, thread_index, &s0);
1566                   goto trace0;
1567                 }
1568             }
1569           else
1570             {
1571               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1572                 {
1573                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1574                   goto trace0;
1575                 }
1576
1577               if (ip4_is_fragment (ip0))
1578                 {
1579                   next0 = SNAT_IN2OUT_NEXT_REASS;
1580                   goto trace0;
1581                 }
1582             }
1583
1584           key0.addr = ip0->src_address;
1585           key0.port = udp0->src_port;
1586           key0.protocol = proto0;
1587           key0.fib_index = rx_fib_index0;
1588
1589           kv0.key = key0.as_u64;
1590
1591           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1592                                       &kv0, &value0))
1593             {
1594               if (is_slow_path)
1595                 {
1596                   if (is_output_feature)
1597                     {
1598                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1599                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1600                         goto trace0;
1601                     }
1602                   else
1603                     {
1604                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1605                           ip0, proto0, rx_fib_index0, thread_index)))
1606                         goto trace0;
1607                     }
1608
1609                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1610                                      &s0, node, next0, thread_index);
1611
1612                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1613                     goto trace0;
1614                 }
1615               else
1616                 {
1617                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1618                   goto trace0;
1619                 }
1620             }
1621           else
1622           s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1623                                   value0.value);
1624
1625           b0->flags |= VNET_BUFFER_F_IS_NATED;
1626
1627           old_addr0 = ip0->src_address.as_u32;
1628           ip0->src_address = s0->out2in.addr;
1629           new_addr0 = ip0->src_address.as_u32;
1630           if (!is_output_feature)
1631             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1632
1633           sum0 = ip0->checksum;
1634           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1635                                  ip4_header_t,
1636                                  src_address /* changed member */);
1637           ip0->checksum = ip_csum_fold (sum0);
1638
1639           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1640             {
1641               old_port0 = tcp0->src_port;
1642               tcp0->src_port = s0->out2in.port;
1643               new_port0 = tcp0->src_port;
1644
1645               sum0 = tcp0->checksum;
1646               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1647                                      ip4_header_t,
1648                                      dst_address /* changed member */);
1649               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1650                                      ip4_header_t /* cheat */,
1651                                      length /* changed member */);
1652               tcp0->checksum = ip_csum_fold(sum0);
1653             }
1654           else
1655             {
1656               old_port0 = udp0->src_port;
1657               udp0->src_port = s0->out2in.port;
1658               udp0->checksum = 0;
1659             }
1660
1661           /* Accounting */
1662           nat44_session_update_counters (s0, now,
1663                                          vlib_buffer_length_in_chain (vm, b0));
1664           /* Per-user LRU list maintenance */
1665           nat44_session_update_lru (sm, s0, thread_index);
1666
1667         trace0:
1668           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1669                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1670             {
1671               snat_in2out_trace_t *t =
1672                  vlib_add_trace (vm, node, b0, sizeof (*t));
1673               t->is_slow_path = is_slow_path;
1674               t->sw_if_index = sw_if_index0;
1675               t->next_index = next0;
1676                   t->session_index = ~0;
1677               if (s0)
1678                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1679             }
1680
1681           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1682
1683           /* verify speculative enqueue, maybe switch current next frame */
1684           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1685                                            to_next, n_left_to_next,
1686                                            bi0, next0);
1687         }
1688
1689       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1690     }
1691
1692   vlib_node_increment_counter (vm, stats_node_index,
1693                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1694                                pkts_processed);
1695   return frame->n_vectors;
1696 }
1697
1698 static uword
1699 snat_in2out_fast_path_fn (vlib_main_t * vm,
1700                           vlib_node_runtime_t * node,
1701                           vlib_frame_t * frame)
1702 {
1703   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1704 }
1705
1706 VLIB_REGISTER_NODE (snat_in2out_node) = {
1707   .function = snat_in2out_fast_path_fn,
1708   .name = "nat44-in2out",
1709   .vector_size = sizeof (u32),
1710   .format_trace = format_snat_in2out_trace,
1711   .type = VLIB_NODE_TYPE_INTERNAL,
1712
1713   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1714   .error_strings = snat_in2out_error_strings,
1715
1716   .runtime_data_bytes = sizeof (snat_runtime_t),
1717
1718   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1719
1720   /* edit / add dispositions here */
1721   .next_nodes = {
1722     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1723     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1724     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1725     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1726     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1727   },
1728 };
1729
1730 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1731
1732 static uword
1733 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1734                                  vlib_node_runtime_t * node,
1735                                  vlib_frame_t * frame)
1736 {
1737   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1738 }
1739
1740 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1741   .function = snat_in2out_output_fast_path_fn,
1742   .name = "nat44-in2out-output",
1743   .vector_size = sizeof (u32),
1744   .format_trace = format_snat_in2out_trace,
1745   .type = VLIB_NODE_TYPE_INTERNAL,
1746
1747   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1748   .error_strings = snat_in2out_error_strings,
1749
1750   .runtime_data_bytes = sizeof (snat_runtime_t),
1751
1752   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1753
1754   /* edit / add dispositions here */
1755   .next_nodes = {
1756     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1757     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1758     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1759     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1760     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1761   },
1762 };
1763
1764 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1765                               snat_in2out_output_fast_path_fn);
1766
1767 static uword
1768 snat_in2out_slow_path_fn (vlib_main_t * vm,
1769                           vlib_node_runtime_t * node,
1770                           vlib_frame_t * frame)
1771 {
1772   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1773 }
1774
1775 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1776   .function = snat_in2out_slow_path_fn,
1777   .name = "nat44-in2out-slowpath",
1778   .vector_size = sizeof (u32),
1779   .format_trace = format_snat_in2out_trace,
1780   .type = VLIB_NODE_TYPE_INTERNAL,
1781
1782   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1783   .error_strings = snat_in2out_error_strings,
1784
1785   .runtime_data_bytes = sizeof (snat_runtime_t),
1786
1787   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1788
1789   /* edit / add dispositions here */
1790   .next_nodes = {
1791     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1792     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1793     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1794     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1795     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1796   },
1797 };
1798
1799 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1800                               snat_in2out_slow_path_fn);
1801
1802 static uword
1803 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1804                                  vlib_node_runtime_t * node,
1805                                  vlib_frame_t * frame)
1806 {
1807   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1808 }
1809
1810 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1811   .function = snat_in2out_output_slow_path_fn,
1812   .name = "nat44-in2out-output-slowpath",
1813   .vector_size = sizeof (u32),
1814   .format_trace = format_snat_in2out_trace,
1815   .type = VLIB_NODE_TYPE_INTERNAL,
1816
1817   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1818   .error_strings = snat_in2out_error_strings,
1819
1820   .runtime_data_bytes = sizeof (snat_runtime_t),
1821
1822   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1823
1824   /* edit / add dispositions here */
1825   .next_nodes = {
1826     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1827     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1828     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1829     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1830     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1831   },
1832 };
1833
1834 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1835                               snat_in2out_output_slow_path_fn);
1836
1837 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1838
1839 static inline uword
1840 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1841                              vlib_node_runtime_t * node,
1842                              vlib_frame_t * frame,
1843                              int is_ed)
1844 {
1845   u32 n_left_from, * from, * to_next, stats_node_index;
1846   snat_in2out_next_t next_index;
1847   u32 pkts_processed = 0;
1848   snat_main_t * sm = &snat_main;
1849   vnet_feature_main_t *fm = &feature_main;
1850   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1851   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1852
1853   stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1854     nat44_hairpinning_node.index;
1855   from = vlib_frame_vector_args (frame);
1856   n_left_from = frame->n_vectors;
1857   next_index = node->cached_next_index;
1858
1859   while (n_left_from > 0)
1860     {
1861       u32 n_left_to_next;
1862
1863       vlib_get_next_frame (vm, node, next_index,
1864                            to_next, n_left_to_next);
1865
1866       while (n_left_from > 0 && n_left_to_next > 0)
1867         {
1868           u32 bi0;
1869           vlib_buffer_t * b0;
1870           u32 next0;
1871           ip4_header_t * ip0;
1872           u32 proto0;
1873           udp_header_t * udp0;
1874           tcp_header_t * tcp0;
1875
1876           /* speculatively enqueue b0 to the current next frame */
1877           bi0 = from[0];
1878           to_next[0] = bi0;
1879           from += 1;
1880           to_next += 1;
1881           n_left_from -= 1;
1882           n_left_to_next -= 1;
1883
1884           b0 = vlib_get_buffer (vm, bi0);
1885           ip0 = vlib_buffer_get_current (b0);
1886           udp0 = ip4_next_header (ip0);
1887           tcp0 = (tcp_header_t *) udp0;
1888
1889           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1890
1891           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
1892                                 &next0, 0);
1893
1894           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1895             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1896
1897           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1898
1899           /* verify speculative enqueue, maybe switch current next frame */
1900           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1901                                            to_next, n_left_to_next,
1902                                            bi0, next0);
1903          }
1904
1905       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1906     }
1907
1908   vlib_node_increment_counter (vm, stats_node_index,
1909                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1910                                pkts_processed);
1911   return frame->n_vectors;
1912 }
1913
1914 static uword
1915 nat44_hairpinning_fn (vlib_main_t * vm,
1916                       vlib_node_runtime_t * node,
1917                       vlib_frame_t * frame)
1918 {
1919   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1920 }
1921
1922 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
1923   .function = nat44_hairpinning_fn,
1924   .name = "nat44-hairpinning",
1925   .vector_size = sizeof (u32),
1926   .type = VLIB_NODE_TYPE_INTERNAL,
1927   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1928   .error_strings = snat_in2out_error_strings,
1929   .n_next_nodes = 2,
1930   .next_nodes = {
1931     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1932     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1933   },
1934 };
1935
1936 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
1937                               nat44_hairpinning_fn);
1938
1939 static uword
1940 nat44_ed_hairpinning_fn (vlib_main_t * vm,
1941                          vlib_node_runtime_t * node,
1942                          vlib_frame_t * frame)
1943 {
1944   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1945 }
1946
1947 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
1948   .function = nat44_ed_hairpinning_fn,
1949   .name = "nat44-ed-hairpinning",
1950   .vector_size = sizeof (u32),
1951   .type = VLIB_NODE_TYPE_INTERNAL,
1952   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1953   .error_strings = snat_in2out_error_strings,
1954   .n_next_nodes = 2,
1955   .next_nodes = {
1956     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1957     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1958   },
1959 };
1960
1961 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
1962                               nat44_ed_hairpinning_fn);
1963
1964 static inline void
1965 nat44_reass_hairpinning (snat_main_t *sm,
1966                          vlib_buffer_t * b0,
1967                          ip4_header_t * ip0,
1968                          u16 sport,
1969                          u16 dport,
1970                          u32 proto0)
1971 {
1972   snat_session_key_t key0, sm0;
1973   snat_session_t * s0;
1974   clib_bihash_kv_8_8_t kv0, value0;
1975   ip_csum_t sum0;
1976   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
1977   u16 new_dst_port0, old_dst_port0;
1978   udp_header_t * udp0;
1979   tcp_header_t * tcp0;
1980
1981   key0.addr = ip0->dst_address;
1982   key0.port = dport;
1983   key0.protocol = proto0;
1984   key0.fib_index = sm->outside_fib_index;
1985   kv0.key = key0.as_u64;
1986
1987   udp0 = ip4_next_header (ip0);
1988
1989   /* Check if destination is static mappings */
1990   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1991     {
1992       new_dst_addr0 = sm0.addr.as_u32;
1993       new_dst_port0 = sm0.port;
1994       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1995     }
1996   /* or active sessions */
1997   else
1998     {
1999       if (sm->num_workers > 1)
2000         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2001       else
2002         ti = sm->num_workers;
2003
2004       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2005         {
2006           si = value0.value;
2007           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2008           new_dst_addr0 = s0->in2out.addr.as_u32;
2009           new_dst_port0 = s0->in2out.port;
2010           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2011         }
2012     }
2013
2014   /* Destination is behind the same NAT, use internal address and port */
2015   if (new_dst_addr0)
2016     {
2017       old_dst_addr0 = ip0->dst_address.as_u32;
2018       ip0->dst_address.as_u32 = new_dst_addr0;
2019       sum0 = ip0->checksum;
2020       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2021                              ip4_header_t, dst_address);
2022       ip0->checksum = ip_csum_fold (sum0);
2023
2024       old_dst_port0 = dport;
2025       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2026                        ip4_is_first_fragment (ip0)))
2027         {
2028           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2029             {
2030               tcp0 = ip4_next_header (ip0);
2031               tcp0->dst = new_dst_port0;
2032               sum0 = tcp0->checksum;
2033               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2034                                      ip4_header_t, dst_address);
2035               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2036                                      ip4_header_t /* cheat */, length);
2037               tcp0->checksum = ip_csum_fold(sum0);
2038             }
2039           else
2040             {
2041               udp0->dst_port = new_dst_port0;
2042               udp0->checksum = 0;
2043             }
2044         }
2045       else
2046         {
2047           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2048             {
2049               tcp0 = ip4_next_header (ip0);
2050               sum0 = tcp0->checksum;
2051               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2052                                      ip4_header_t, dst_address);
2053               tcp0->checksum = ip_csum_fold(sum0);
2054             }
2055         }
2056     }
2057 }
2058
2059 static uword
2060 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2061                             vlib_node_runtime_t * node,
2062                             vlib_frame_t * frame)
2063 {
2064   u32 n_left_from, *from, *to_next;
2065   snat_in2out_next_t next_index;
2066   u32 pkts_processed = 0;
2067   snat_main_t *sm = &snat_main;
2068   f64 now = vlib_time_now (vm);
2069   u32 thread_index = vm->thread_index;
2070   snat_main_per_thread_data_t *per_thread_data =
2071     &sm->per_thread_data[thread_index];
2072   u32 *fragments_to_drop = 0;
2073   u32 *fragments_to_loopback = 0;
2074
2075   from = vlib_frame_vector_args (frame);
2076   n_left_from = frame->n_vectors;
2077   next_index = node->cached_next_index;
2078
2079   while (n_left_from > 0)
2080     {
2081       u32 n_left_to_next;
2082
2083       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2084
2085       while (n_left_from > 0 && n_left_to_next > 0)
2086        {
2087           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2088           vlib_buffer_t *b0;
2089           u32 next0;
2090           u8 cached0 = 0;
2091           ip4_header_t *ip0;
2092           nat_reass_ip4_t *reass0;
2093           udp_header_t * udp0;
2094           tcp_header_t * tcp0;
2095           snat_session_key_t key0;
2096           clib_bihash_kv_8_8_t kv0, value0;
2097           snat_session_t * s0 = 0;
2098           u16 old_port0, new_port0;
2099           ip_csum_t sum0;
2100
2101           /* speculatively enqueue b0 to the current next frame */
2102           bi0 = from[0];
2103           to_next[0] = bi0;
2104           from += 1;
2105           to_next += 1;
2106           n_left_from -= 1;
2107           n_left_to_next -= 1;
2108
2109           b0 = vlib_get_buffer (vm, bi0);
2110           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2111
2112           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2113           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2114                                                                sw_if_index0);
2115
2116           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2117             {
2118               next0 = SNAT_IN2OUT_NEXT_DROP;
2119               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2120               goto trace0;
2121             }
2122
2123           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2124           udp0 = ip4_next_header (ip0);
2125           tcp0 = (tcp_header_t *) udp0;
2126           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2127
2128           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2129                                                  ip0->dst_address,
2130                                                  ip0->fragment_id,
2131                                                  ip0->protocol,
2132                                                  1,
2133                                                  &fragments_to_drop);
2134
2135           if (PREDICT_FALSE (!reass0))
2136             {
2137               next0 = SNAT_IN2OUT_NEXT_DROP;
2138               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2139               nat_log_notice ("maximum reassemblies exceeded");
2140               goto trace0;
2141             }
2142
2143           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2144             {
2145               key0.addr = ip0->src_address;
2146               key0.port = udp0->src_port;
2147               key0.protocol = proto0;
2148               key0.fib_index = rx_fib_index0;
2149               kv0.key = key0.as_u64;
2150
2151               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2152                 {
2153                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2154                       ip0, proto0, rx_fib_index0, thread_index)))
2155                     goto trace0;
2156
2157                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2158                                      &s0, node, next0, thread_index);
2159
2160                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2161                     goto trace0;
2162
2163                   reass0->sess_index = s0 - per_thread_data->sessions;
2164                 }
2165               else
2166                 {
2167                   s0 = pool_elt_at_index (per_thread_data->sessions,
2168                                           value0.value);
2169                   reass0->sess_index = value0.value;
2170                 }
2171               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2172             }
2173           else
2174             {
2175               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2176                 {
2177                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2178                     {
2179                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2180                       nat_log_notice ("maximum fragments per reassembly exceeded");
2181                       next0 = SNAT_IN2OUT_NEXT_DROP;
2182                       goto trace0;
2183                     }
2184                   cached0 = 1;
2185                   goto trace0;
2186                 }
2187               s0 = pool_elt_at_index (per_thread_data->sessions,
2188                                       reass0->sess_index);
2189             }
2190
2191           old_addr0 = ip0->src_address.as_u32;
2192           ip0->src_address = s0->out2in.addr;
2193           new_addr0 = ip0->src_address.as_u32;
2194           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2195
2196           sum0 = ip0->checksum;
2197           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2198                                  ip4_header_t,
2199                                  src_address /* changed member */);
2200           ip0->checksum = ip_csum_fold (sum0);
2201
2202           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2203             {
2204               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2205                 {
2206                   old_port0 = tcp0->src_port;
2207                   tcp0->src_port = s0->out2in.port;
2208                   new_port0 = tcp0->src_port;
2209
2210                   sum0 = tcp0->checksum;
2211                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2212                                          ip4_header_t,
2213                                          dst_address /* changed member */);
2214                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2215                                          ip4_header_t /* cheat */,
2216                                          length /* changed member */);
2217                   tcp0->checksum = ip_csum_fold(sum0);
2218                 }
2219               else
2220                 {
2221                   old_port0 = udp0->src_port;
2222                   udp0->src_port = s0->out2in.port;
2223                   udp0->checksum = 0;
2224                 }
2225             }
2226
2227           /* Hairpinning */
2228           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2229                                    s0->ext_host_port, proto0);
2230
2231           /* Accounting */
2232           nat44_session_update_counters (s0, now,
2233                                          vlib_buffer_length_in_chain (vm, b0));
2234           /* Per-user LRU list maintenance */
2235           nat44_session_update_lru (sm, s0, thread_index);
2236
2237         trace0:
2238           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2239                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2240             {
2241               nat44_in2out_reass_trace_t *t =
2242                  vlib_add_trace (vm, node, b0, sizeof (*t));
2243               t->cached = cached0;
2244               t->sw_if_index = sw_if_index0;
2245               t->next_index = next0;
2246             }
2247
2248           if (cached0)
2249             {
2250               n_left_to_next++;
2251               to_next--;
2252             }
2253           else
2254             {
2255               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2256
2257               /* verify speculative enqueue, maybe switch current next frame */
2258               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2259                                                to_next, n_left_to_next,
2260                                                bi0, next0);
2261             }
2262
2263           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2264             {
2265               from = vlib_frame_vector_args (frame);
2266               u32 len = vec_len (fragments_to_loopback);
2267               if (len <= VLIB_FRAME_SIZE)
2268                 {
2269                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2270                   n_left_from = len;
2271                   vec_reset_length (fragments_to_loopback);
2272                 }
2273               else
2274                 {
2275                   clib_memcpy (from,
2276                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2277                                sizeof (u32) * VLIB_FRAME_SIZE);
2278                   n_left_from = VLIB_FRAME_SIZE;
2279                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2280                 }
2281             }
2282        }
2283
2284       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2285     }
2286
2287   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2288                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2289                                pkts_processed);
2290
2291   nat_send_all_to_node (vm, fragments_to_drop, node,
2292                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2293                         SNAT_IN2OUT_NEXT_DROP);
2294
2295   vec_free (fragments_to_drop);
2296   vec_free (fragments_to_loopback);
2297   return frame->n_vectors;
2298 }
2299
2300 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2301   .function = nat44_in2out_reass_node_fn,
2302   .name = "nat44-in2out-reass",
2303   .vector_size = sizeof (u32),
2304   .format_trace = format_nat44_in2out_reass_trace,
2305   .type = VLIB_NODE_TYPE_INTERNAL,
2306
2307   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2308   .error_strings = snat_in2out_error_strings,
2309
2310   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2311   .next_nodes = {
2312     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2313     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2314     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2315     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2316     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2317   },
2318 };
2319
2320 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2321                               nat44_in2out_reass_node_fn);
2322
2323 /*******************************/
2324 /*** endpoint-dependent mode ***/
2325 /*******************************/
2326
2327 static_always_inline int
2328 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2329 {
2330   icmp46_header_t *icmp0;
2331   nat_ed_ses_key_t key0;
2332   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2333   ip4_header_t *inner_ip0 = 0;
2334   void *l4_header = 0;
2335   icmp46_header_t *inner_icmp0;
2336
2337   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2338   echo0 = (icmp_echo_header_t *)(icmp0+1);
2339
2340   if (!icmp_is_error_message (icmp0))
2341     {
2342       key0.proto = IP_PROTOCOL_ICMP;
2343       key0.l_addr = ip0->src_address;
2344       key0.r_addr = ip0->dst_address;
2345       key0.l_port = echo0->identifier;
2346       key0.r_port = 0;
2347     }
2348   else
2349     {
2350       inner_ip0 = (ip4_header_t *)(echo0+1);
2351       l4_header = ip4_next_header (inner_ip0);
2352       key0.proto = inner_ip0->protocol;
2353       key0.r_addr = inner_ip0->src_address;
2354       key0.l_addr = inner_ip0->dst_address;
2355       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2356         {
2357         case SNAT_PROTOCOL_ICMP:
2358           inner_icmp0 = (icmp46_header_t*)l4_header;
2359           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2360           key0.r_port = 0;
2361           key0.l_port = inner_echo0->identifier;
2362           break;
2363         case SNAT_PROTOCOL_UDP:
2364         case SNAT_PROTOCOL_TCP:
2365           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2366           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2367           break;
2368         default:
2369           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2370         }
2371     }
2372   *p_key0 = key0;
2373   return 0;
2374 }
2375
2376 static u32
2377 slow_path_ed (snat_main_t *sm,
2378               vlib_buffer_t *b,
2379               u32 rx_fib_index,
2380               clib_bihash_kv_16_8_t *kv,
2381               snat_session_t ** sessionp,
2382               vlib_node_runtime_t * node,
2383               u32 next,
2384               u32 thread_index)
2385 {
2386   snat_session_t *s;
2387   snat_user_t *u;
2388   snat_session_key_t key0, key1;
2389   u8 lb = 0, is_sm = 0;
2390   u32 address_index = ~0;
2391   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2392   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2393   u32 proto = ip_proto_to_snat_proto (key->proto);
2394   nat_outside_fib_t *outside_fib;
2395   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2396   fib_prefix_t pfx = {
2397     .fp_proto = FIB_PROTOCOL_IP4,
2398     .fp_len = 32,
2399     .fp_addr = {
2400         .ip4.as_u32 = key->r_addr.as_u32,
2401     },
2402   };
2403
2404   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2405     {
2406       b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2407       nat_ipfix_logging_max_sessions(sm->max_translations);
2408       nat_log_notice ("maximum sessions exceeded");
2409       return SNAT_IN2OUT_NEXT_DROP;
2410     }
2411
2412   key0.addr = key->l_addr;
2413   key0.port = key->l_port;
2414   key1.protocol = key0.protocol = proto;
2415   key0.fib_index = rx_fib_index;
2416   key1.fib_index = sm->outside_fib_index;
2417   /* First try to match static mapping by local address and port */
2418   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb))
2419     {
2420       /* Try to create dynamic translation */
2421       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2422                                                thread_index, &key1,
2423                                                &address_index,
2424                                                sm->port_per_thread,
2425                                                tsm->snat_thread_index))
2426         {
2427           nat_log_notice ("addresses exhausted");
2428           b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2429           return SNAT_IN2OUT_NEXT_DROP;
2430         }
2431     }
2432   else
2433     is_sm = 1;
2434
2435   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2436   if (!u)
2437     {
2438       nat_log_warn ("create NAT user failed");
2439       return SNAT_IN2OUT_NEXT_DROP;
2440     }
2441
2442   s = nat_session_alloc_or_recycle (sm, u, thread_index);
2443   if (!s)
2444     {
2445       nat_log_warn ("create NAT session failed");
2446       return SNAT_IN2OUT_NEXT_DROP;
2447     }
2448
2449   user_session_increment (sm, u, is_sm);
2450   if (is_sm)
2451     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2452   if (lb)
2453     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2454   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2455   s->outside_address_index = address_index;
2456   s->ext_host_addr = key->r_addr;
2457   s->ext_host_port = key->r_port;
2458   s->in2out = key0;
2459   s->out2in = key1;
2460   s->out2in.protocol = key0.protocol;
2461
2462   switch (vec_len (sm->outside_fibs))
2463     {
2464     case 0:
2465       s->out2in.fib_index = sm->outside_fib_index;
2466       break;
2467     case 1:
2468       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2469       break;
2470     default:
2471       vec_foreach (outside_fib, sm->outside_fibs)
2472         {
2473           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2474           if (FIB_NODE_INDEX_INVALID != fei)
2475             {
2476               if (fib_entry_get_resolving_interface (fei) != ~0)
2477                 {
2478                   s->out2in.fib_index = outside_fib->fib_index;
2479                   break;
2480                 }
2481             }
2482         }
2483       break;
2484     }
2485
2486   /* Add to lookup tables */
2487   kv->value = s - tsm->sessions;
2488   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, kv, 1))
2489     nat_log_notice ("in2out-ed key add failed");
2490
2491   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2492               key1.port, key->r_port);
2493   kv->value = s - tsm->sessions;
2494   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, kv, 1))
2495     nat_log_notice ("out2in-ed key add failed");
2496
2497   *sessionp = s;
2498
2499   /* log NAT event */
2500   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2501                                       s->out2in.addr.as_u32,
2502                                       s->in2out.protocol,
2503                                       s->in2out.port,
2504                                       s->out2in.port,
2505                                       s->in2out.fib_index);
2506   return next;
2507 }
2508
2509 static_always_inline int
2510 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2511                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
2512                         u32 rx_fib_index, u32 thread_index)
2513 {
2514   udp_header_t *udp = ip4_next_header (ip);
2515   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2516   clib_bihash_kv_16_8_t kv, value;
2517   snat_session_key_t key0, key1;
2518
2519   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2520               sm->outside_fib_index, udp->dst_port, udp->src_port);
2521
2522   /* NAT packet aimed at external address if */
2523   /* has active sessions */
2524   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2525     {
2526       key0.addr = ip->dst_address;
2527       key0.port = udp->dst_port;
2528       key0.protocol = proto;
2529       key0.fib_index = sm->outside_fib_index;
2530       /* or is static mappings */
2531       if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0))
2532         return 0;
2533     }
2534   else
2535     return 0;
2536
2537   if (sm->forwarding_enabled)
2538     return 1;
2539
2540   return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2541 }
2542
2543 static_always_inline int
2544 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2545                                       u32 thread_index, f64 now,
2546                                       vlib_main_t * vm, vlib_buffer_t * b)
2547 {
2548   nat_ed_ses_key_t key;
2549   clib_bihash_kv_16_8_t kv, value;
2550   udp_header_t *udp;
2551   snat_session_t *s = 0;
2552   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2553
2554   if (!sm->forwarding_enabled)
2555     return 0;
2556
2557   if (ip->protocol == IP_PROTOCOL_ICMP)
2558     {
2559       key.as_u64[0] = key.as_u64[1] = 0;
2560       if (icmp_get_ed_key (ip, &key))
2561         return 0;
2562       key.fib_index = 0;
2563       kv.key[0] = key.as_u64[0];
2564       kv.key[1] = key.as_u64[1];
2565     }
2566   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2567     {
2568       udp = ip4_next_header(ip);
2569       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2570                   udp->src_port, udp->dst_port);
2571     }
2572   else
2573     {
2574       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2575                   0);
2576     }
2577
2578   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2579     {
2580       s = pool_elt_at_index (tsm->sessions, value.value);
2581       if (is_fwd_bypass_session (s))
2582         {
2583           if (ip->protocol == IP_PROTOCOL_TCP)
2584             {
2585               tcp_header_t *tcp = ip4_next_header(ip);
2586               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2587                 return 1;
2588             }
2589           /* Per-user LRU list maintenance */
2590           nat44_session_update_lru (sm, s, thread_index);
2591           /* Accounting */
2592           nat44_session_update_counters (s, now,
2593                                          vlib_buffer_length_in_chain (vm, b));
2594           return 1;
2595         }
2596       else
2597         return 0;
2598     }
2599
2600   return 0;
2601 }
2602
2603 static_always_inline int
2604 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2605                                        u8 proto, u16 src_port, u16 dst_port,
2606                                        u32 thread_index, u32 sw_if_index)
2607 {
2608   clib_bihash_kv_16_8_t kv, value;
2609   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2610   snat_interface_t *i;
2611   snat_session_t *s;
2612   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2613
2614   /* src NAT check */
2615   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, fib_index,
2616               src_port, dst_port);
2617   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2618     return 1;
2619
2620   /* dst NAT check */
2621   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, fib_index,
2622               dst_port, src_port);
2623   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2624   {
2625     s = pool_elt_at_index (tsm->sessions, value.value);
2626     if (is_fwd_bypass_session (s))
2627       return 0;
2628
2629     /* hairpinning */
2630     pool_foreach (i, sm->output_feature_interfaces,
2631     ({
2632       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
2633         return 0;
2634     }));
2635     return 1;
2636   }
2637
2638   return 0;
2639 }
2640
2641 u32
2642 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2643                      u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2644                      u8 *p_proto, snat_session_key_t *p_value,
2645                      u8 *p_dont_translate, void *d, void *e)
2646 {
2647   icmp46_header_t *icmp;
2648   u32 sw_if_index;
2649   u32 rx_fib_index;
2650   nat_ed_ses_key_t key;
2651   snat_session_t *s = 0;
2652   u8 dont_translate = 0;
2653   clib_bihash_kv_16_8_t kv, value;
2654   u32 next = ~0;
2655   int err;
2656   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2657
2658   icmp = (icmp46_header_t *) ip4_next_header (ip);
2659   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2660   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2661
2662   key.as_u64[0] = key.as_u64[1] = 0;
2663   err = icmp_get_ed_key (ip, &key);
2664   if (err != 0)
2665     {
2666       b->error = node->errors[err];
2667       next = SNAT_IN2OUT_NEXT_DROP;
2668       goto out;
2669     }
2670   key.fib_index = rx_fib_index;
2671
2672   kv.key[0] = key.as_u64[0];
2673   kv.key[1] = key.as_u64[1];
2674
2675   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2676     {
2677       if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2678         {
2679           if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2680               key.proto, key.l_port, key.r_port, thread_index, sw_if_index)))
2681             {
2682               dont_translate = 1;
2683               goto out;
2684             }
2685         }
2686       else
2687         {
2688           if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2689               ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2690             {
2691               dont_translate = 1;
2692               goto out;
2693             }
2694         }
2695
2696       if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2697         {
2698           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2699           next = SNAT_IN2OUT_NEXT_DROP;
2700           goto out;
2701         }
2702
2703       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2704                            thread_index);
2705
2706       if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2707         goto out;
2708     }
2709   else
2710     {
2711       if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2712                         icmp->type != ICMP4_echo_reply &&
2713                         !icmp_is_error_message (icmp)))
2714         {
2715           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2716           next = SNAT_IN2OUT_NEXT_DROP;
2717           goto out;
2718         }
2719
2720       s = pool_elt_at_index (tsm->sessions, value.value);
2721     }
2722
2723   *p_proto = ip_proto_to_snat_proto (key.proto);
2724 out:
2725   if (s)
2726     *p_value = s->out2in;
2727   *p_dont_translate = dont_translate;
2728   if (d)
2729     *(snat_session_t**)d = s;
2730   return next;
2731 }
2732
2733 static inline void
2734 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
2735                                     vlib_buffer_t * b,
2736                                     ip4_header_t * ip)
2737 {
2738   u32 old_addr, new_addr = 0, ti = 0;
2739   clib_bihash_kv_8_8_t kv, value;
2740   clib_bihash_kv_16_8_t s_kv, s_value;
2741   snat_static_mapping_t *m;
2742   ip_csum_t sum;
2743   snat_session_t *s;
2744   snat_main_per_thread_data_t *tsm;
2745
2746   if (sm->num_workers > 1)
2747     ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2748   else
2749     ti = sm->num_workers;
2750   tsm = &sm->per_thread_data[ti];
2751
2752   old_addr = ip->dst_address.as_u32;
2753   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2754               sm->outside_fib_index, 0, 0);
2755   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2756     {
2757       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2758       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2759         return;
2760
2761       m = pool_elt_at_index (sm->static_mappings, value.value);
2762       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2763         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2764       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2765     }
2766   else
2767     {
2768       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2769       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2770         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2771       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2772     }
2773   sum = ip->checksum;
2774   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2775   ip->checksum = ip_csum_fold (sum);
2776 }
2777
2778 static snat_session_t *
2779 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
2780                                vlib_buffer_t * b,
2781                                ip4_header_t * ip,
2782                                u32 rx_fib_index,
2783                                u32 thread_index,
2784                                f64 now,
2785                                vlib_main_t * vm,
2786                                vlib_node_runtime_t * node)
2787 {
2788   clib_bihash_kv_8_8_t kv, value;
2789   clib_bihash_kv_16_8_t s_kv, s_value;
2790   snat_static_mapping_t *m;
2791   u32 old_addr, new_addr = 0;
2792   ip_csum_t sum;
2793   snat_user_t *u;
2794   dlist_elt_t *head, *elt;
2795   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2796   u32 elt_index, head_index, ses_index;
2797   snat_session_t * s;
2798   u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2799   int i;
2800   u8 is_sm = 0;
2801   nat_outside_fib_t *outside_fib;
2802   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2803   fib_prefix_t pfx = {
2804     .fp_proto = FIB_PROTOCOL_IP4,
2805     .fp_len = 32,
2806     .fp_addr = {
2807         .ip4.as_u32 = ip->dst_address.as_u32,
2808     },
2809   };
2810
2811   switch (vec_len (sm->outside_fibs))
2812     {
2813     case 0:
2814       outside_fib_index = sm->outside_fib_index;
2815       break;
2816     case 1:
2817       outside_fib_index = sm->outside_fibs[0].fib_index;
2818       break;
2819     default:
2820       vec_foreach (outside_fib, sm->outside_fibs)
2821         {
2822           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2823           if (FIB_NODE_INDEX_INVALID != fei)
2824             {
2825               if (fib_entry_get_resolving_interface (fei) != ~0)
2826                 {
2827                   outside_fib_index = outside_fib->fib_index;
2828                   break;
2829                 }
2830             }
2831         }
2832       break;
2833     }
2834   old_addr = ip->src_address.as_u32;
2835
2836   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
2837               rx_fib_index, 0, 0);
2838
2839   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
2840     {
2841       s = pool_elt_at_index (tsm->sessions, s_value.value);
2842       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2843     }
2844   else
2845     {
2846       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
2847         {
2848           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2849           nat_ipfix_logging_max_sessions(sm->max_translations);
2850           nat_log_notice ("maximum sessions exceeded");
2851           return 0;
2852         }
2853
2854       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
2855                                   thread_index);
2856       if (!u)
2857         {
2858           nat_log_warn ("create NAT user failed");
2859           return 0;
2860         }
2861
2862       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
2863
2864       /* Try to find static mapping first */
2865       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
2866         {
2867           m = pool_elt_at_index (sm->static_mappings, value.value);
2868           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
2869           is_sm = 1;
2870           goto create_ses;
2871         }
2872       /* Fallback to 3-tuple key */
2873       else
2874         {
2875           /* Choose same out address as for TCP/UDP session to same destination */
2876           head_index = u->sessions_per_user_list_head_index;
2877           head = pool_elt_at_index (tsm->list_pool, head_index);
2878           elt_index = head->next;
2879           if (PREDICT_FALSE (elt_index == ~0))
2880             ses_index = ~0;
2881           else
2882             {
2883               elt = pool_elt_at_index (tsm->list_pool, elt_index);
2884               ses_index = elt->value;
2885             }
2886
2887           while (ses_index != ~0)
2888             {
2889               s =  pool_elt_at_index (tsm->sessions, ses_index);
2890               elt_index = elt->next;
2891               elt = pool_elt_at_index (tsm->list_pool, elt_index);
2892               ses_index = elt->value;
2893
2894               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
2895                 {
2896                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2897                   address_index = s->outside_address_index;
2898
2899                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
2900                               ip->protocol, outside_fib_index, 0, 0);
2901                   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2902                     goto create_ses;
2903
2904                   break;
2905                 }
2906             }
2907
2908           for (i = 0; i < vec_len (sm->addresses); i++)
2909             {
2910               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
2911                           ip->protocol, outside_fib_index, 0, 0);
2912               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2913                 {
2914                   new_addr = ip->src_address.as_u32 =
2915                     sm->addresses[i].addr.as_u32;
2916                   address_index = i;
2917                   goto create_ses;
2918                 }
2919             }
2920           return 0;
2921         }
2922
2923 create_ses:
2924       s = nat_session_alloc_or_recycle (sm, u, thread_index);
2925       if (!s)
2926         {
2927           nat_log_warn ("create NAT session failed");
2928           return 0;
2929         }
2930
2931       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
2932       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
2933       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2934       s->outside_address_index = address_index;
2935       s->out2in.addr.as_u32 = new_addr;
2936       s->out2in.fib_index = outside_fib_index;
2937       s->in2out.addr.as_u32 = old_addr;
2938       s->in2out.fib_index = rx_fib_index;
2939       s->in2out.port = s->out2in.port = ip->protocol;
2940       if (is_sm)
2941         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2942       user_session_increment (sm, u, is_sm);
2943
2944       /* Add to lookup tables */
2945       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
2946                   rx_fib_index, 0, 0);
2947       s_kv.value = s - tsm->sessions;
2948       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2949         nat_log_notice ("in2out key add failed");
2950
2951       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
2952                   outside_fib_index, 0, 0);
2953       s_kv.value = s - tsm->sessions;
2954       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2955         nat_log_notice ("out2in key add failed");
2956   }
2957
2958   /* Update IP checksum */
2959   sum = ip->checksum;
2960   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
2961   ip->checksum = ip_csum_fold (sum);
2962
2963   /* Accounting */
2964   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
2965   /* Per-user LRU list maintenance */
2966   nat44_session_update_lru (sm, s, thread_index);
2967
2968   /* Hairpinning */
2969   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2970     nat44_ed_hairpinning_unknown_proto(sm, b, ip);
2971
2972   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2973     vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
2974
2975   return s;
2976 }
2977
2978 static inline uword
2979 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
2980                                 vlib_node_runtime_t * node,
2981                                 vlib_frame_t * frame, int is_slow_path,
2982                                 int is_output_feature)
2983 {
2984   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2985   snat_in2out_next_t next_index;
2986   snat_main_t *sm = &snat_main;
2987   f64 now = vlib_time_now (vm);
2988   u32 thread_index = vm->thread_index;
2989   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2990
2991   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
2992     nat44_ed_in2out_node.index;
2993
2994   from = vlib_frame_vector_args (frame);
2995   n_left_from = frame->n_vectors;
2996   next_index = node->cached_next_index;
2997
2998   while (n_left_from > 0)
2999     {
3000       u32 n_left_to_next;
3001
3002       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3003
3004       while (n_left_from >= 4 && n_left_to_next >= 2)
3005         {
3006           u32 bi0, bi1;
3007           vlib_buffer_t *b0, *b1;
3008           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3009               new_addr0, old_addr0;
3010           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3011               new_addr1, old_addr1;
3012           u16 old_port0, new_port0, old_port1, new_port1;
3013           ip4_header_t *ip0, *ip1;
3014           udp_header_t *udp0, *udp1;
3015           tcp_header_t *tcp0, *tcp1;
3016           icmp46_header_t *icmp0, *icmp1;
3017           snat_session_t *s0 = 0, *s1 = 0;
3018           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3019           ip_csum_t sum0, sum1;
3020
3021           /* Prefetch next iteration. */
3022           {
3023             vlib_buffer_t * p2, * p3;
3024
3025             p2 = vlib_get_buffer (vm, from[2]);
3026             p3 = vlib_get_buffer (vm, from[3]);
3027
3028             vlib_prefetch_buffer_header (p2, LOAD);
3029             vlib_prefetch_buffer_header (p3, LOAD);
3030
3031             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3032             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3033           }
3034
3035           /* speculatively enqueue b0 and b1 to the current next frame */
3036           to_next[0] = bi0 = from[0];
3037           to_next[1] = bi1 = from[1];
3038           from += 2;
3039           to_next += 2;
3040           n_left_from -= 2;
3041           n_left_to_next -= 2;
3042
3043           b0 = vlib_get_buffer (vm, bi0);
3044           b1 = vlib_get_buffer (vm, bi1);
3045
3046           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3047
3048           if (is_output_feature)
3049             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3050
3051           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3052                  iph_offset0);
3053
3054           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3055           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3056                                                                sw_if_index0);
3057
3058           if (PREDICT_FALSE(ip0->ttl == 1))
3059             {
3060               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3061               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3062                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3063                                            0);
3064               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3065               goto trace00;
3066             }
3067
3068           udp0 = ip4_next_header (ip0);
3069           tcp0 = (tcp_header_t *) udp0;
3070           icmp0 = (icmp46_header_t *) udp0;
3071           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3072
3073           if (is_slow_path)
3074             {
3075               if (PREDICT_FALSE (proto0 == ~0))
3076                 {
3077                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3078                                                       rx_fib_index0,
3079                                                       thread_index, now, vm,
3080                                                       node);
3081                   if (!s0)
3082                     next0 = SNAT_IN2OUT_NEXT_DROP;
3083                   goto trace00;
3084                 }
3085
3086               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3087                 {
3088                   next0 = icmp_in2out_slow_path
3089                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3090                      next0, now, thread_index, &s0);
3091                   goto trace00;
3092                 }
3093             }
3094           else
3095             {
3096                if (is_output_feature)
3097                 {
3098                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3099                       sm, ip0, thread_index, now, vm, b0)))
3100                     goto trace00;
3101                 }
3102
3103               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3104                 {
3105                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3106                   goto trace00;
3107                 }
3108
3109               if (ip4_is_fragment (ip0))
3110                 {
3111                   b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3112                   next0 = SNAT_IN2OUT_NEXT_DROP;
3113                   goto trace00;
3114                 }
3115             }
3116
3117           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3118                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3119
3120           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3121             {
3122               if (is_slow_path)
3123                 {
3124                   if (is_output_feature)
3125                     {
3126                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3127                           sm, ip0, ip0->protocol, udp0->src_port,
3128                           udp0->dst_port, thread_index, sw_if_index0)))
3129                         goto trace00;
3130                     }
3131                   else
3132                     {
3133                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3134                           sw_if_index0, ip0, proto0, rx_fib_index0,
3135                           thread_index)))
3136                         goto trace00;
3137                     }
3138
3139                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3140                                         next0, thread_index);
3141
3142                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3143                     goto trace00;
3144                 }
3145               else
3146                 {
3147                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3148                   goto trace00;
3149                 }
3150             }
3151           else
3152             {
3153               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3154             }
3155
3156           b0->flags |= VNET_BUFFER_F_IS_NATED;
3157
3158           if (!is_output_feature)
3159             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3160
3161           old_addr0 = ip0->src_address.as_u32;
3162           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3163           sum0 = ip0->checksum;
3164           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3165                                  src_address);
3166           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3167             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3168                                    s0->ext_host_addr.as_u32, ip4_header_t,
3169                                    dst_address);
3170           ip0->checksum = ip_csum_fold (sum0);
3171
3172           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3173             {
3174               old_port0 = tcp0->src_port;
3175               new_port0 = tcp0->src_port = s0->out2in.port;
3176
3177               sum0 = tcp0->checksum;
3178               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3179                                      dst_address);
3180               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3181                                      length);
3182               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3183                 {
3184                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3185                                          s0->ext_host_addr.as_u32,
3186                                          ip4_header_t, dst_address);
3187                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3188                                          s0->ext_host_port, ip4_header_t,
3189                                          length);
3190                   tcp0->dst_port = s0->ext_host_port;
3191                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3192                 }
3193               tcp0->checksum = ip_csum_fold(sum0);
3194               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3195                 goto trace00;
3196             }
3197           else
3198             {
3199               udp0->src_port = s0->out2in.port;
3200               udp0->checksum = 0;
3201               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3202                 {
3203                   udp0->dst_port = s0->ext_host_port;
3204                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3205                 }
3206             }
3207
3208           /* Accounting */
3209           nat44_session_update_counters (s0, now,
3210                                          vlib_buffer_length_in_chain (vm, b0));
3211           /* Per-user LRU list maintenance */
3212           nat44_session_update_lru (sm, s0, thread_index);
3213
3214         trace00:
3215           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3216                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3217             {
3218               snat_in2out_trace_t *t =
3219                 vlib_add_trace (vm, node, b0, sizeof (*t));
3220               t->is_slow_path = is_slow_path;
3221               t->sw_if_index = sw_if_index0;
3222               t->next_index = next0;
3223               t->session_index = ~0;
3224               if (s0)
3225                 t->session_index = s0 - tsm->sessions;
3226             }
3227
3228           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3229
3230
3231           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3232
3233           if (is_output_feature)
3234             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3235
3236           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3237                  iph_offset1);
3238
3239           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3240           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3241                                                                sw_if_index1);
3242
3243           if (PREDICT_FALSE(ip1->ttl == 1))
3244             {
3245               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3246               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3247                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3248                                            0);
3249               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3250               goto trace01;
3251             }
3252
3253           udp1 = ip4_next_header (ip1);
3254           tcp1 = (tcp_header_t *) udp1;
3255           icmp1 = (icmp46_header_t *) udp1;
3256           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3257
3258           if (is_slow_path)
3259             {
3260               if (PREDICT_FALSE (proto1 == ~0))
3261                 {
3262                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3263                                                       rx_fib_index1,
3264                                                       thread_index, now, vm,
3265                                                       node);
3266                   if (!s1)
3267                     next1 = SNAT_IN2OUT_NEXT_DROP;
3268                   goto trace01;
3269                 }
3270
3271               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3272                 {
3273                   next1 = icmp_in2out_slow_path
3274                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3275                      next1, now, thread_index, &s1);
3276                   goto trace01;
3277                 }
3278             }
3279           else
3280             {
3281                if (is_output_feature)
3282                 {
3283                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3284                       sm, ip1, thread_index, now, vm, b1)))
3285                     goto trace01;
3286                 }
3287
3288               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3289                 {
3290                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3291                   goto trace01;
3292                 }
3293
3294               if (ip4_is_fragment (ip1))
3295                 {
3296                   b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3297                   next1 = SNAT_IN2OUT_NEXT_DROP;
3298                   goto trace01;
3299                 }
3300             }
3301
3302           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3303                       rx_fib_index1, udp1->src_port, udp1->dst_port);
3304
3305           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3306             {
3307               if (is_slow_path)
3308                 {
3309                   if (is_output_feature)
3310                     {
3311                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3312                           sm, ip1, ip1->protocol, udp1->src_port,
3313                           udp1->dst_port, thread_index, sw_if_index1)))
3314                         goto trace01;
3315                     }
3316                   else
3317                     {
3318                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3319                           sw_if_index1, ip1, proto1, rx_fib_index1,
3320                           thread_index)))
3321                         goto trace01;
3322                     }
3323
3324                   next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3325                                         next1, thread_index);
3326
3327                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3328                     goto trace01;
3329                 }
3330               else
3331                 {
3332                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3333                   goto trace01;
3334                 }
3335             }
3336           else
3337             {
3338               s1 = pool_elt_at_index (tsm->sessions, value1.value);
3339             }
3340
3341           b1->flags |= VNET_BUFFER_F_IS_NATED;
3342
3343           if (!is_output_feature)
3344             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3345
3346           old_addr1 = ip1->src_address.as_u32;
3347           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3348           sum1 = ip1->checksum;
3349           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3350                                  src_address);
3351           if (PREDICT_FALSE (is_twice_nat_session (s1)))
3352             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3353                                    s1->ext_host_addr.as_u32, ip4_header_t,
3354                                    dst_address);
3355           ip1->checksum = ip_csum_fold (sum1);
3356
3357           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3358             {
3359               old_port1 = tcp1->src_port;
3360               new_port1 = tcp1->src_port = s1->out2in.port;
3361
3362               sum1 = tcp1->checksum;
3363               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3364                                      dst_address);
3365               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3366                                      length);
3367               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3368                 {
3369                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3370                                          s1->ext_host_addr.as_u32,
3371                                          ip4_header_t, dst_address);
3372                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
3373                                          s1->ext_host_port, ip4_header_t,
3374                                          length);
3375                   tcp1->dst_port = s1->ext_host_port;
3376                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3377                 }
3378               tcp1->checksum = ip_csum_fold(sum1);
3379               if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3380                 goto trace01;
3381             }
3382           else
3383             {
3384               udp1->src_port = s1->out2in.port;
3385               udp1->checksum = 0;
3386               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3387                 {
3388                   udp1->dst_port = s1->ext_host_port;
3389                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3390                 }
3391             }
3392
3393           /* Accounting */
3394           nat44_session_update_counters (s1, now,
3395                                          vlib_buffer_length_in_chain (vm, b1));
3396           /* Per-user LRU list maintenance */
3397           nat44_session_update_lru (sm, s1, thread_index);
3398
3399         trace01:
3400           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3401                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3402             {
3403               snat_in2out_trace_t *t =
3404                 vlib_add_trace (vm, node, b1, sizeof (*t));
3405               t->is_slow_path = is_slow_path;
3406               t->sw_if_index = sw_if_index1;
3407               t->next_index = next1;
3408               t->session_index = ~0;
3409               if (s1)
3410                 t->session_index = s1 - tsm->sessions;
3411             }
3412
3413           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3414
3415           /* verify speculative enqueues, maybe switch current next frame */
3416           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3417                                            to_next, n_left_to_next,
3418                                            bi0, bi1, next0, next1);
3419         }
3420
3421       while (n_left_from > 0 && n_left_to_next > 0)
3422         {
3423           u32 bi0;
3424           vlib_buffer_t *b0;
3425           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3426               new_addr0, old_addr0;
3427           u16 old_port0, new_port0;
3428           ip4_header_t *ip0;
3429           udp_header_t *udp0;
3430           tcp_header_t *tcp0;
3431           icmp46_header_t * icmp0;
3432           snat_session_t *s0 = 0;
3433           clib_bihash_kv_16_8_t kv0, value0;
3434           ip_csum_t sum0;
3435
3436           /* speculatively enqueue b0 to the current next frame */
3437           bi0 = from[0];
3438           to_next[0] = bi0;
3439           from += 1;
3440           to_next += 1;
3441           n_left_from -= 1;
3442           n_left_to_next -= 1;
3443
3444           b0 = vlib_get_buffer (vm, bi0);
3445           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3446
3447           if (is_output_feature)
3448             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3449
3450           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3451                  iph_offset0);
3452
3453           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3454           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3455                                                                sw_if_index0);
3456
3457           if (PREDICT_FALSE(ip0->ttl == 1))
3458             {
3459               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3460               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3461                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3462                                            0);
3463               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3464               goto trace0;
3465             }
3466
3467           udp0 = ip4_next_header (ip0);
3468           tcp0 = (tcp_header_t *) udp0;
3469           icmp0 = (icmp46_header_t *) udp0;
3470           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3471
3472           if (is_slow_path)
3473             {
3474               if (PREDICT_FALSE (proto0 == ~0))
3475                 {
3476                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3477                                                       rx_fib_index0,
3478                                                       thread_index, now, vm,
3479                                                       node);
3480                   if (!s0)
3481                     next0 = SNAT_IN2OUT_NEXT_DROP;
3482                   goto trace0;
3483                 }
3484
3485               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3486                 {
3487                   next0 = icmp_in2out_slow_path
3488                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3489                      next0, now, thread_index, &s0);
3490                   goto trace0;
3491                 }
3492             }
3493           else
3494             {
3495                if (is_output_feature)
3496                 {
3497                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3498                       sm, ip0, thread_index, now, vm, b0)))
3499                     goto trace0;
3500                 }
3501
3502               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3503                 {
3504                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3505                   goto trace0;
3506                 }
3507
3508               if (ip4_is_fragment (ip0))
3509                 {
3510                   b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3511                   next0 = SNAT_IN2OUT_NEXT_DROP;
3512                   goto trace0;
3513                 }
3514             }
3515
3516           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3517                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3518
3519           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3520             {
3521               if (is_slow_path)
3522                 {
3523                   if (is_output_feature)
3524                     {
3525                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3526                           sm, ip0, ip0->protocol, udp0->src_port,
3527                           udp0->dst_port, thread_index, sw_if_index0)))
3528                         goto trace0;
3529                     }
3530                   else
3531                     {
3532                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3533                           sw_if_index0, ip0, proto0, rx_fib_index0,
3534                           thread_index)))
3535                         goto trace0;
3536                     }
3537
3538                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3539                                         next0, thread_index);
3540
3541                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3542                     goto trace0;
3543                 }
3544               else
3545                 {
3546                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3547                   goto trace0;
3548                 }
3549             }
3550           else
3551             {
3552               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3553             }
3554
3555           b0->flags |= VNET_BUFFER_F_IS_NATED;
3556
3557           if (!is_output_feature)
3558             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3559
3560           old_addr0 = ip0->src_address.as_u32;
3561           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3562           sum0 = ip0->checksum;
3563           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3564                                  src_address);
3565           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3566             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3567                                    s0->ext_host_addr.as_u32, ip4_header_t,
3568                                    dst_address);
3569           ip0->checksum = ip_csum_fold (sum0);
3570
3571           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3572             {
3573               old_port0 = tcp0->src_port;
3574               new_port0 = tcp0->src_port = s0->out2in.port;
3575
3576               sum0 = tcp0->checksum;
3577               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3578                                      dst_address);
3579               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3580                                      length);
3581               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3582                 {
3583                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3584                                          s0->ext_host_addr.as_u32,
3585                                          ip4_header_t, dst_address);
3586                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3587                                          s0->ext_host_port, ip4_header_t,
3588                                          length);
3589                   tcp0->dst_port = s0->ext_host_port;
3590                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3591                 }
3592               tcp0->checksum = ip_csum_fold(sum0);
3593               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3594                 goto trace0;
3595             }
3596           else
3597             {
3598               udp0->src_port = s0->out2in.port;
3599               udp0->checksum = 0;
3600               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3601                 {
3602                   udp0->dst_port = s0->ext_host_port;
3603                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3604                 }
3605             }
3606
3607           /* Accounting */
3608           nat44_session_update_counters (s0, now,
3609                                          vlib_buffer_length_in_chain (vm, b0));
3610           /* Per-user LRU list maintenance */
3611           nat44_session_update_lru (sm, s0, thread_index);
3612
3613         trace0:
3614           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3615                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3616             {
3617               snat_in2out_trace_t *t =
3618                 vlib_add_trace (vm, node, b0, sizeof (*t));
3619               t->is_slow_path = is_slow_path;
3620               t->sw_if_index = sw_if_index0;
3621               t->next_index = next0;
3622               t->session_index = ~0;
3623               if (s0)
3624                 t->session_index = s0 - tsm->sessions;
3625             }
3626
3627           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3628
3629           /* verify speculative enqueue, maybe switch current next frame */
3630           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3631                                            to_next, n_left_to_next,
3632                                            bi0, next0);
3633         }
3634
3635       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3636     }
3637
3638   vlib_node_increment_counter (vm, stats_node_index,
3639                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3640                                pkts_processed);
3641   return frame->n_vectors;
3642 }
3643
3644 static uword
3645 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3646                               vlib_node_runtime_t * node,
3647                               vlib_frame_t * frame)
3648 {
3649   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3650 }
3651
3652 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3653   .function = nat44_ed_in2out_fast_path_fn,
3654   .name = "nat44-ed-in2out",
3655   .vector_size = sizeof (u32),
3656   .format_trace = format_snat_in2out_trace,
3657   .type = VLIB_NODE_TYPE_INTERNAL,
3658
3659   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3660   .error_strings = snat_in2out_error_strings,
3661
3662   .runtime_data_bytes = sizeof (snat_runtime_t),
3663
3664   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3665
3666   /* edit / add dispositions here */
3667   .next_nodes = {
3668     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3669     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3670     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3671     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3672     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3673   },
3674 };
3675
3676 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3677
3678 static uword
3679 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3680                                      vlib_node_runtime_t * node,
3681                                      vlib_frame_t * frame)
3682 {
3683   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3684 }
3685
3686 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3687   .function = nat44_ed_in2out_output_fast_path_fn,
3688   .name = "nat44-ed-in2out-output",
3689   .vector_size = sizeof (u32),
3690   .format_trace = format_snat_in2out_trace,
3691   .type = VLIB_NODE_TYPE_INTERNAL,
3692
3693   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3694   .error_strings = snat_in2out_error_strings,
3695
3696   .runtime_data_bytes = sizeof (snat_runtime_t),
3697
3698   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3699
3700   /* edit / add dispositions here */
3701   .next_nodes = {
3702     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3703     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3704     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3705     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3706     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3707   },
3708 };
3709
3710 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
3711                               nat44_ed_in2out_output_fast_path_fn);
3712
3713 static uword
3714 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
3715                               vlib_node_runtime_t * node,
3716                               vlib_frame_t * frame)
3717 {
3718   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3719 }
3720
3721 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
3722   .function = nat44_ed_in2out_slow_path_fn,
3723   .name = "nat44-ed-in2out-slowpath",
3724   .vector_size = sizeof (u32),
3725   .format_trace = format_snat_in2out_trace,
3726   .type = VLIB_NODE_TYPE_INTERNAL,
3727
3728   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3729   .error_strings = snat_in2out_error_strings,
3730
3731   .runtime_data_bytes = sizeof (snat_runtime_t),
3732
3733   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3734
3735   /* edit / add dispositions here */
3736   .next_nodes = {
3737     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3738     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3739     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3740     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3741     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3742   },
3743 };
3744
3745 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
3746                               nat44_ed_in2out_slow_path_fn);
3747
3748 static uword
3749 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
3750                                      vlib_node_runtime_t * node,
3751                                      vlib_frame_t * frame)
3752 {
3753   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3754 }
3755
3756 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
3757   .function = nat44_ed_in2out_output_slow_path_fn,
3758   .name = "nat44-ed-in2out-output-slowpath",
3759   .vector_size = sizeof (u32),
3760   .format_trace = format_snat_in2out_trace,
3761   .type = VLIB_NODE_TYPE_INTERNAL,
3762
3763   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3764   .error_strings = snat_in2out_error_strings,
3765
3766   .runtime_data_bytes = sizeof (snat_runtime_t),
3767
3768   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3769
3770   /* edit / add dispositions here */
3771   .next_nodes = {
3772     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3773     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3774     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3775     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3776     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3777   },
3778 };
3779
3780 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
3781                               nat44_ed_in2out_output_slow_path_fn);
3782
3783 /**************************/
3784 /*** deterministic mode ***/
3785 /**************************/
3786 static uword
3787 snat_det_in2out_node_fn (vlib_main_t * vm,
3788                          vlib_node_runtime_t * node,
3789                          vlib_frame_t * frame)
3790 {
3791   u32 n_left_from, * from, * to_next;
3792   snat_in2out_next_t next_index;
3793   u32 pkts_processed = 0;
3794   snat_main_t * sm = &snat_main;
3795   u32 now = (u32) vlib_time_now (vm);
3796   u32 thread_index = vm->thread_index;
3797
3798   from = vlib_frame_vector_args (frame);
3799   n_left_from = frame->n_vectors;
3800   next_index = node->cached_next_index;
3801
3802   while (n_left_from > 0)
3803     {
3804       u32 n_left_to_next;
3805
3806       vlib_get_next_frame (vm, node, next_index,
3807                            to_next, n_left_to_next);
3808
3809       while (n_left_from >= 4 && n_left_to_next >= 2)
3810         {
3811           u32 bi0, bi1;
3812           vlib_buffer_t * b0, * b1;
3813           u32 next0, next1;
3814           u32 sw_if_index0, sw_if_index1;
3815           ip4_header_t * ip0, * ip1;
3816           ip_csum_t sum0, sum1;
3817           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3818           u16 old_port0, new_port0, lo_port0, i0;
3819           u16 old_port1, new_port1, lo_port1, i1;
3820           udp_header_t * udp0, * udp1;
3821           tcp_header_t * tcp0, * tcp1;
3822           u32 proto0, proto1;
3823           snat_det_out_key_t key0, key1;
3824           snat_det_map_t * dm0, * dm1;
3825           snat_det_session_t * ses0 = 0, * ses1 = 0;
3826           u32 rx_fib_index0, rx_fib_index1;
3827           icmp46_header_t * icmp0, * icmp1;
3828
3829           /* Prefetch next iteration. */
3830           {
3831             vlib_buffer_t * p2, * p3;
3832
3833             p2 = vlib_get_buffer (vm, from[2]);
3834             p3 = vlib_get_buffer (vm, from[3]);
3835
3836             vlib_prefetch_buffer_header (p2, LOAD);
3837             vlib_prefetch_buffer_header (p3, LOAD);
3838
3839             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3840             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3841           }
3842
3843           /* speculatively enqueue b0 and b1 to the current next frame */
3844           to_next[0] = bi0 = from[0];
3845           to_next[1] = bi1 = from[1];
3846           from += 2;
3847           to_next += 2;
3848           n_left_from -= 2;
3849           n_left_to_next -= 2;
3850
3851           b0 = vlib_get_buffer (vm, bi0);
3852           b1 = vlib_get_buffer (vm, bi1);
3853
3854           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3855           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3856
3857           ip0 = vlib_buffer_get_current (b0);
3858           udp0 = ip4_next_header (ip0);
3859           tcp0 = (tcp_header_t *) udp0;
3860
3861           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3862
3863           if (PREDICT_FALSE(ip0->ttl == 1))
3864             {
3865               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3866               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3867                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3868                                            0);
3869               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3870               goto trace0;
3871             }
3872
3873           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3874
3875           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3876             {
3877               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3878               icmp0 = (icmp46_header_t *) udp0;
3879
3880               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3881                                   rx_fib_index0, node, next0, thread_index,
3882                                   &ses0, &dm0);
3883               goto trace0;
3884             }
3885
3886           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3887           if (PREDICT_FALSE(!dm0))
3888             {
3889               nat_log_info ("no match for internal host %U",
3890                             format_ip4_address, &ip0->src_address);
3891               next0 = SNAT_IN2OUT_NEXT_DROP;
3892               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3893               goto trace0;
3894             }
3895
3896           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3897
3898           key0.ext_host_addr = ip0->dst_address;
3899           key0.ext_host_port = tcp0->dst;
3900
3901           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3902           if (PREDICT_FALSE(!ses0))
3903             {
3904               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3905                 {
3906                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3907                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3908
3909                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3910                     continue;
3911
3912                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3913                   break;
3914                 }
3915               if (PREDICT_FALSE(!ses0))
3916                 {
3917                   /* too many sessions for user, send ICMP error packet */
3918
3919                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3920                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3921                                                ICMP4_destination_unreachable_destination_unreachable_host,
3922                                                0);
3923                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3924                   goto trace0;
3925                 }
3926             }
3927
3928           new_port0 = ses0->out.out_port;
3929
3930           old_addr0.as_u32 = ip0->src_address.as_u32;
3931           ip0->src_address.as_u32 = new_addr0.as_u32;
3932           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3933
3934           sum0 = ip0->checksum;
3935           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3936                                  ip4_header_t,
3937                                  src_address /* changed member */);
3938           ip0->checksum = ip_csum_fold (sum0);
3939
3940           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3941             {
3942               if (tcp0->flags & TCP_FLAG_SYN)
3943                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3944               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3945                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3946               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3947                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3948               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3949                 snat_det_ses_close(dm0, ses0);
3950               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3951                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3952               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3953                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3954
3955               old_port0 = tcp0->src;
3956               tcp0->src = new_port0;
3957
3958               sum0 = tcp0->checksum;
3959               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3960                                      ip4_header_t,
3961                                      dst_address /* changed member */);
3962               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3963                                      ip4_header_t /* cheat */,
3964                                      length /* changed member */);
3965               tcp0->checksum = ip_csum_fold(sum0);
3966             }
3967           else
3968             {
3969               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3970               old_port0 = udp0->src_port;
3971               udp0->src_port = new_port0;
3972               udp0->checksum = 0;
3973             }
3974
3975           switch(ses0->state)
3976             {
3977             case SNAT_SESSION_UDP_ACTIVE:
3978                 ses0->expire = now + sm->udp_timeout;
3979                 break;
3980             case SNAT_SESSION_TCP_SYN_SENT:
3981             case SNAT_SESSION_TCP_FIN_WAIT:
3982             case SNAT_SESSION_TCP_CLOSE_WAIT:
3983             case SNAT_SESSION_TCP_LAST_ACK:
3984                 ses0->expire = now + sm->tcp_transitory_timeout;
3985                 break;
3986             case SNAT_SESSION_TCP_ESTABLISHED:
3987                 ses0->expire = now + sm->tcp_established_timeout;
3988                 break;
3989             }
3990
3991         trace0:
3992           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3993                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3994             {
3995               snat_in2out_trace_t *t =
3996                  vlib_add_trace (vm, node, b0, sizeof (*t));
3997               t->is_slow_path = 0;
3998               t->sw_if_index = sw_if_index0;
3999               t->next_index = next0;
4000               t->session_index = ~0;
4001               if (ses0)
4002                 t->session_index = ses0 - dm0->sessions;
4003             }
4004
4005           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4006
4007           ip1 = vlib_buffer_get_current (b1);
4008           udp1 = ip4_next_header (ip1);
4009           tcp1 = (tcp_header_t *) udp1;
4010
4011           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4012
4013           if (PREDICT_FALSE(ip1->ttl == 1))
4014             {
4015               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4016               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4017                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4018                                            0);
4019               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4020               goto trace1;
4021             }
4022
4023           proto1 = ip_proto_to_snat_proto (ip1->protocol);
4024
4025           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4026             {
4027               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4028               icmp1 = (icmp46_header_t *) udp1;
4029
4030               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4031                                   rx_fib_index1, node, next1, thread_index,
4032                                   &ses1, &dm1);
4033               goto trace1;
4034             }
4035
4036           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4037           if (PREDICT_FALSE(!dm1))
4038             {
4039               nat_log_info ("no match for internal host %U",
4040                             format_ip4_address, &ip0->src_address);
4041               next1 = SNAT_IN2OUT_NEXT_DROP;
4042               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4043               goto trace1;
4044             }
4045
4046           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4047
4048           key1.ext_host_addr = ip1->dst_address;
4049           key1.ext_host_port = tcp1->dst;
4050
4051           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4052           if (PREDICT_FALSE(!ses1))
4053             {
4054               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4055                 {
4056                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
4057                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4058
4059                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4060                     continue;
4061
4062                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4063                   break;
4064                 }
4065               if (PREDICT_FALSE(!ses1))
4066                 {
4067                   /* too many sessions for user, send ICMP error packet */
4068
4069                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4070                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4071                                                ICMP4_destination_unreachable_destination_unreachable_host,
4072                                                0);
4073                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4074                   goto trace1;
4075                 }
4076             }
4077
4078           new_port1 = ses1->out.out_port;
4079
4080           old_addr1.as_u32 = ip1->src_address.as_u32;
4081           ip1->src_address.as_u32 = new_addr1.as_u32;
4082           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4083
4084           sum1 = ip1->checksum;
4085           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4086                                  ip4_header_t,
4087                                  src_address /* changed member */);
4088           ip1->checksum = ip_csum_fold (sum1);
4089
4090           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4091             {
4092               if (tcp1->flags & TCP_FLAG_SYN)
4093                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4094               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4095                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4096               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4097                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4098               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4099                 snat_det_ses_close(dm1, ses1);
4100               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4101                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4102               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4103                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4104
4105               old_port1 = tcp1->src;
4106               tcp1->src = new_port1;
4107
4108               sum1 = tcp1->checksum;
4109               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4110                                      ip4_header_t,
4111                                      dst_address /* changed member */);
4112               sum1 = ip_csum_update (sum1, old_port1, new_port1,
4113                                      ip4_header_t /* cheat */,
4114                                      length /* changed member */);
4115               tcp1->checksum = ip_csum_fold(sum1);
4116             }
4117           else
4118             {
4119               ses1->state = SNAT_SESSION_UDP_ACTIVE;
4120               old_port1 = udp1->src_port;
4121               udp1->src_port = new_port1;
4122               udp1->checksum = 0;
4123             }
4124
4125           switch(ses1->state)
4126             {
4127             case SNAT_SESSION_UDP_ACTIVE:
4128                 ses1->expire = now + sm->udp_timeout;
4129                 break;
4130             case SNAT_SESSION_TCP_SYN_SENT:
4131             case SNAT_SESSION_TCP_FIN_WAIT:
4132             case SNAT_SESSION_TCP_CLOSE_WAIT:
4133             case SNAT_SESSION_TCP_LAST_ACK:
4134                 ses1->expire = now + sm->tcp_transitory_timeout;
4135                 break;
4136             case SNAT_SESSION_TCP_ESTABLISHED:
4137                 ses1->expire = now + sm->tcp_established_timeout;
4138                 break;
4139             }
4140
4141         trace1:
4142           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4143                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4144             {
4145               snat_in2out_trace_t *t =
4146                  vlib_add_trace (vm, node, b1, sizeof (*t));
4147               t->is_slow_path = 0;
4148               t->sw_if_index = sw_if_index1;
4149               t->next_index = next1;
4150               t->session_index = ~0;
4151               if (ses1)
4152                 t->session_index = ses1 - dm1->sessions;
4153             }
4154
4155           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4156
4157           /* verify speculative enqueues, maybe switch current next frame */
4158           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4159                                            to_next, n_left_to_next,
4160                                            bi0, bi1, next0, next1);
4161          }
4162
4163       while (n_left_from > 0 && n_left_to_next > 0)
4164         {
4165           u32 bi0;
4166           vlib_buffer_t * b0;
4167           u32 next0;
4168           u32 sw_if_index0;
4169           ip4_header_t * ip0;
4170           ip_csum_t sum0;
4171           ip4_address_t new_addr0, old_addr0;
4172           u16 old_port0, new_port0, lo_port0, i0;
4173           udp_header_t * udp0;
4174           tcp_header_t * tcp0;
4175           u32 proto0;
4176           snat_det_out_key_t key0;
4177           snat_det_map_t * dm0;
4178           snat_det_session_t * ses0 = 0;
4179           u32 rx_fib_index0;
4180           icmp46_header_t * icmp0;
4181
4182           /* speculatively enqueue b0 to the current next frame */
4183           bi0 = from[0];
4184           to_next[0] = bi0;
4185           from += 1;
4186           to_next += 1;
4187           n_left_from -= 1;
4188           n_left_to_next -= 1;
4189
4190           b0 = vlib_get_buffer (vm, bi0);
4191           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4192
4193           ip0 = vlib_buffer_get_current (b0);
4194           udp0 = ip4_next_header (ip0);
4195           tcp0 = (tcp_header_t *) udp0;
4196
4197           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4198
4199           if (PREDICT_FALSE(ip0->ttl == 1))
4200             {
4201               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4202               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4203                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4204                                            0);
4205               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4206               goto trace00;
4207             }
4208
4209           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4210
4211           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4212             {
4213               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4214               icmp0 = (icmp46_header_t *) udp0;
4215
4216               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4217                                   rx_fib_index0, node, next0, thread_index,
4218                                   &ses0, &dm0);
4219               goto trace00;
4220             }
4221
4222           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4223           if (PREDICT_FALSE(!dm0))
4224             {
4225               nat_log_info ("no match for internal host %U",
4226                             format_ip4_address, &ip0->src_address);
4227               next0 = SNAT_IN2OUT_NEXT_DROP;
4228               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4229               goto trace00;
4230             }
4231
4232           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4233
4234           key0.ext_host_addr = ip0->dst_address;
4235           key0.ext_host_port = tcp0->dst;
4236
4237           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4238           if (PREDICT_FALSE(!ses0))
4239             {
4240               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4241                 {
4242                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
4243                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4244
4245                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4246                     continue;
4247
4248                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4249                   break;
4250                 }
4251               if (PREDICT_FALSE(!ses0))
4252                 {
4253                   /* too many sessions for user, send ICMP error packet */
4254
4255                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4256                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4257                                                ICMP4_destination_unreachable_destination_unreachable_host,
4258                                                0);
4259                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4260                   goto trace00;
4261                 }
4262             }
4263
4264           new_port0 = ses0->out.out_port;
4265
4266           old_addr0.as_u32 = ip0->src_address.as_u32;
4267           ip0->src_address.as_u32 = new_addr0.as_u32;
4268           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4269
4270           sum0 = ip0->checksum;
4271           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4272                                  ip4_header_t,
4273                                  src_address /* changed member */);
4274           ip0->checksum = ip_csum_fold (sum0);
4275
4276           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4277             {
4278               if (tcp0->flags & TCP_FLAG_SYN)
4279                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4280               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4281                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4282               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4283                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4284               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4285                 snat_det_ses_close(dm0, ses0);
4286               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4287                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4288               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4289                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4290
4291               old_port0 = tcp0->src;
4292               tcp0->src = new_port0;
4293
4294               sum0 = tcp0->checksum;
4295               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4296                                      ip4_header_t,
4297                                      dst_address /* changed member */);
4298               sum0 = ip_csum_update (sum0, old_port0, new_port0,
4299                                      ip4_header_t /* cheat */,
4300                                      length /* changed member */);
4301               tcp0->checksum = ip_csum_fold(sum0);
4302             }
4303           else
4304             {
4305               ses0->state = SNAT_SESSION_UDP_ACTIVE;
4306               old_port0 = udp0->src_port;
4307               udp0->src_port = new_port0;
4308               udp0->checksum = 0;
4309             }
4310
4311           switch(ses0->state)
4312             {
4313             case SNAT_SESSION_UDP_ACTIVE:
4314                 ses0->expire = now + sm->udp_timeout;
4315                 break;
4316             case SNAT_SESSION_TCP_SYN_SENT:
4317             case SNAT_SESSION_TCP_FIN_WAIT:
4318             case SNAT_SESSION_TCP_CLOSE_WAIT:
4319             case SNAT_SESSION_TCP_LAST_ACK:
4320                 ses0->expire = now + sm->tcp_transitory_timeout;
4321                 break;
4322             case SNAT_SESSION_TCP_ESTABLISHED:
4323                 ses0->expire = now + sm->tcp_established_timeout;
4324                 break;
4325             }
4326
4327         trace00:
4328           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4329                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4330             {
4331               snat_in2out_trace_t *t =
4332                  vlib_add_trace (vm, node, b0, sizeof (*t));
4333               t->is_slow_path = 0;
4334               t->sw_if_index = sw_if_index0;
4335               t->next_index = next0;
4336               t->session_index = ~0;
4337               if (ses0)
4338                 t->session_index = ses0 - dm0->sessions;
4339             }
4340
4341           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4342
4343           /* verify speculative enqueue, maybe switch current next frame */
4344           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4345                                            to_next, n_left_to_next,
4346                                            bi0, next0);
4347         }
4348
4349       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4350     }
4351
4352   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
4353                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4354                                pkts_processed);
4355   return frame->n_vectors;
4356 }
4357
4358 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
4359   .function = snat_det_in2out_node_fn,
4360   .name = "nat44-det-in2out",
4361   .vector_size = sizeof (u32),
4362   .format_trace = format_snat_in2out_trace,
4363   .type = VLIB_NODE_TYPE_INTERNAL,
4364
4365   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4366   .error_strings = snat_in2out_error_strings,
4367
4368   .runtime_data_bytes = sizeof (snat_runtime_t),
4369
4370   .n_next_nodes = 3,
4371
4372   /* edit / add dispositions here */
4373   .next_nodes = {
4374     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4375     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4376     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4377   },
4378 };
4379
4380 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
4381
4382 /**
4383  * Get address and port values to be used for ICMP packet translation
4384  * and create session if needed
4385  *
4386  * @param[in,out] sm             NAT main
4387  * @param[in,out] node           NAT node runtime
4388  * @param[in] thread_index       thread index
4389  * @param[in,out] b0             buffer containing packet to be translated
4390  * @param[out] p_proto           protocol used for matching
4391  * @param[out] p_value           address and port after NAT translation
4392  * @param[out] p_dont_translate  if packet should not be translated
4393  * @param d                      optional parameter
4394  * @param e                      optional parameter
4395  */
4396 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
4397                           u32 thread_index, vlib_buffer_t *b0,
4398                           ip4_header_t *ip0, u8 *p_proto,
4399                           snat_session_key_t *p_value,
4400                           u8 *p_dont_translate, void *d, void *e)
4401 {
4402   icmp46_header_t *icmp0;
4403   u32 sw_if_index0;
4404   u32 rx_fib_index0;
4405   u8 protocol;
4406   snat_det_out_key_t key0;
4407   u8 dont_translate = 0;
4408   u32 next0 = ~0;
4409   icmp_echo_header_t *echo0, *inner_echo0 = 0;
4410   ip4_header_t *inner_ip0;
4411   void *l4_header = 0;
4412   icmp46_header_t *inner_icmp0;
4413   snat_det_map_t * dm0 = 0;
4414   ip4_address_t new_addr0;
4415   u16 lo_port0, i0;
4416   snat_det_session_t * ses0 = 0;
4417   ip4_address_t in_addr;
4418   u16 in_port;
4419
4420   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4421   echo0 = (icmp_echo_header_t *)(icmp0+1);
4422   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4423   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4424
4425   if (!icmp_is_error_message (icmp0))
4426     {
4427       protocol = SNAT_PROTOCOL_ICMP;
4428       in_addr = ip0->src_address;
4429       in_port = echo0->identifier;
4430     }
4431   else
4432     {
4433       inner_ip0 = (ip4_header_t *)(echo0+1);
4434       l4_header = ip4_next_header (inner_ip0);
4435       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4436       in_addr = inner_ip0->dst_address;
4437       switch (protocol)
4438         {
4439         case SNAT_PROTOCOL_ICMP:
4440           inner_icmp0 = (icmp46_header_t*)l4_header;
4441           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4442           in_port = inner_echo0->identifier;
4443           break;
4444         case SNAT_PROTOCOL_UDP:
4445         case SNAT_PROTOCOL_TCP:
4446           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4447           break;
4448         default:
4449           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4450           next0 = SNAT_IN2OUT_NEXT_DROP;
4451           goto out;
4452         }
4453     }
4454
4455   dm0 = snat_det_map_by_user(sm, &in_addr);
4456   if (PREDICT_FALSE(!dm0))
4457     {
4458       nat_log_info ("no match for internal host %U",
4459                     format_ip4_address, &in_addr);
4460       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4461           IP_PROTOCOL_ICMP, rx_fib_index0)))
4462         {
4463           dont_translate = 1;
4464           goto out;
4465         }
4466       next0 = SNAT_IN2OUT_NEXT_DROP;
4467       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4468       goto out;
4469     }
4470
4471   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4472
4473   key0.ext_host_addr = ip0->dst_address;
4474   key0.ext_host_port = 0;
4475
4476   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4477   if (PREDICT_FALSE(!ses0))
4478     {
4479       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4480           IP_PROTOCOL_ICMP, rx_fib_index0)))
4481         {
4482           dont_translate = 1;
4483           goto out;
4484         }
4485       if (icmp0->type != ICMP4_echo_request)
4486         {
4487           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4488           next0 = SNAT_IN2OUT_NEXT_DROP;
4489           goto out;
4490         }
4491       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4492         {
4493           key0.out_port = clib_host_to_net_u16 (lo_port0 +
4494             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4495
4496           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4497             continue;
4498
4499           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4500           break;
4501         }
4502       if (PREDICT_FALSE(!ses0))
4503         {
4504           next0 = SNAT_IN2OUT_NEXT_DROP;
4505           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4506           goto out;
4507         }
4508     }
4509
4510   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4511                     !icmp_is_error_message (icmp0)))
4512     {
4513       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4514       next0 = SNAT_IN2OUT_NEXT_DROP;
4515       goto out;
4516     }
4517
4518   u32 now = (u32) vlib_time_now (sm->vlib_main);
4519
4520   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4521   ses0->expire = now + sm->icmp_timeout;
4522
4523 out:
4524   *p_proto = protocol;
4525   if (ses0)
4526     {
4527       p_value->addr = new_addr0;
4528       p_value->fib_index = sm->outside_fib_index;
4529       p_value->port = ses0->out.out_port;
4530     }
4531   *p_dont_translate = dont_translate;
4532   if (d)
4533     *(snat_det_session_t**)d = ses0;
4534   if (e)
4535     *(snat_det_map_t**)e = dm0;
4536   return next0;
4537 }
4538
4539 /**********************/
4540 /*** worker handoff ***/
4541 /**********************/
4542 static inline uword
4543 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
4544                                       vlib_node_runtime_t * node,
4545                                       vlib_frame_t * frame,
4546                                       u8 is_output)
4547 {
4548   snat_main_t *sm = &snat_main;
4549   vlib_thread_main_t *tm = vlib_get_thread_main ();
4550   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4551   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4552   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4553     = 0;
4554   vlib_frame_queue_elt_t *hf = 0;
4555   vlib_frame_queue_t *fq;
4556   vlib_frame_t *f = 0;
4557   int i;
4558   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4559   u32 next_worker_index = 0;
4560   u32 current_worker_index = ~0;
4561   u32 thread_index = vm->thread_index;
4562   u32 fq_index;
4563   u32 to_node_index;
4564   vlib_frame_t *d = 0;
4565
4566   ASSERT (vec_len (sm->workers));
4567
4568   if (is_output)
4569     {
4570       fq_index = sm->fq_in2out_output_index;
4571       to_node_index = sm->in2out_output_node_index;
4572     }
4573   else
4574     {
4575       fq_index = sm->fq_in2out_index;
4576       to_node_index = sm->in2out_node_index;
4577     }
4578
4579   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4580     {
4581       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4582
4583       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4584                                tm->n_vlib_mains - 1,
4585                                (vlib_frame_queue_t *) (~0));
4586     }
4587
4588   from = vlib_frame_vector_args (frame);
4589   n_left_from = frame->n_vectors;
4590
4591   while (n_left_from > 0)
4592     {
4593       u32 bi0;
4594       vlib_buffer_t *b0;
4595       u32 sw_if_index0;
4596       u32 rx_fib_index0;
4597       ip4_header_t * ip0;
4598       u8 do_handoff;
4599
4600       bi0 = from[0];
4601       from += 1;
4602       n_left_from -= 1;
4603
4604       b0 = vlib_get_buffer (vm, bi0);
4605
4606       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4607       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4608
4609       ip0 = vlib_buffer_get_current (b0);
4610
4611       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4612
4613       if (PREDICT_FALSE (next_worker_index != thread_index))
4614         {
4615           do_handoff = 1;
4616
4617           if (next_worker_index != current_worker_index)
4618             {
4619               fq = is_vlib_frame_queue_congested (
4620                 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4621                 congested_handoff_queue_by_worker_index);
4622
4623               if (fq)
4624                 {
4625                   /* if this is 1st frame */
4626                   if (!d)
4627                     {
4628                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
4629                       to_next_drop = vlib_frame_vector_args (d);
4630                     }
4631
4632                   to_next_drop[0] = bi0;
4633                   to_next_drop += 1;
4634                   d->n_vectors++;
4635                   b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4636                   goto trace0;
4637                 }
4638
4639               if (hf)
4640                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4641
4642               hf = vlib_get_worker_handoff_queue_elt (fq_index,
4643                                                       next_worker_index,
4644                                                       handoff_queue_elt_by_worker_index);
4645
4646               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4647               to_next_worker = &hf->buffer_index[hf->n_vectors];
4648               current_worker_index = next_worker_index;
4649             }
4650
4651           /* enqueue to correct worker thread */
4652           to_next_worker[0] = bi0;
4653           to_next_worker++;
4654           n_left_to_next_worker--;
4655
4656           if (n_left_to_next_worker == 0)
4657             {
4658               hf->n_vectors = VLIB_FRAME_SIZE;
4659               vlib_put_frame_queue_elt (hf);
4660               current_worker_index = ~0;
4661               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4662               hf = 0;
4663             }
4664         }
4665       else
4666         {
4667           do_handoff = 0;
4668           /* if this is 1st frame */
4669           if (!f)
4670             {
4671               f = vlib_get_frame_to_node (vm, to_node_index);
4672               to_next = vlib_frame_vector_args (f);
4673             }
4674
4675           to_next[0] = bi0;
4676           to_next += 1;
4677           f->n_vectors++;
4678         }
4679
4680 trace0:
4681       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
4682                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4683         {
4684           snat_in2out_worker_handoff_trace_t *t =
4685             vlib_add_trace (vm, node, b0, sizeof (*t));
4686           t->next_worker_index = next_worker_index;
4687           t->do_handoff = do_handoff;
4688         }
4689     }
4690
4691   if (f)
4692     vlib_put_frame_to_node (vm, to_node_index, f);
4693
4694   if (d)
4695     vlib_put_frame_to_node (vm, sm->error_node_index, d);
4696
4697   if (hf)
4698     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4699
4700   /* Ship frames to the worker nodes */
4701   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4702     {
4703       if (handoff_queue_elt_by_worker_index[i])
4704         {
4705           hf = handoff_queue_elt_by_worker_index[i];
4706           /*
4707            * It works better to let the handoff node
4708            * rate-adapt, always ship the handoff queue element.
4709            */
4710           if (1 || hf->n_vectors == hf->last_n_vectors)
4711             {
4712               vlib_put_frame_queue_elt (hf);
4713               handoff_queue_elt_by_worker_index[i] = 0;
4714             }
4715           else
4716             hf->last_n_vectors = hf->n_vectors;
4717         }
4718       congested_handoff_queue_by_worker_index[i] =
4719         (vlib_frame_queue_t *) (~0);
4720     }
4721   hf = 0;
4722   current_worker_index = ~0;
4723   return frame->n_vectors;
4724 }
4725
4726 static uword
4727 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
4728                                vlib_node_runtime_t * node,
4729                                vlib_frame_t * frame)
4730 {
4731   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4732 }
4733
4734 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
4735   .function = snat_in2out_worker_handoff_fn,
4736   .name = "nat44-in2out-worker-handoff",
4737   .vector_size = sizeof (u32),
4738   .format_trace = format_snat_in2out_worker_handoff_trace,
4739   .type = VLIB_NODE_TYPE_INTERNAL,
4740
4741   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4742   .error_strings = snat_in2out_error_strings,
4743
4744   .n_next_nodes = 1,
4745
4746   .next_nodes = {
4747     [0] = "error-drop",
4748   },
4749 };
4750
4751 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
4752                               snat_in2out_worker_handoff_fn);
4753
4754 static uword
4755 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
4756                                       vlib_node_runtime_t * node,
4757                                       vlib_frame_t * frame)
4758 {
4759   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4760 }
4761
4762 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
4763   .function = snat_in2out_output_worker_handoff_fn,
4764   .name = "nat44-in2out-output-worker-handoff",
4765   .vector_size = sizeof (u32),
4766   .format_trace = format_snat_in2out_worker_handoff_trace,
4767   .type = VLIB_NODE_TYPE_INTERNAL,
4768
4769   .n_next_nodes = 1,
4770
4771   .next_nodes = {
4772     [0] = "error-drop",
4773   },
4774 };
4775
4776 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
4777                               snat_in2out_output_worker_handoff_fn);
4778
4779 static_always_inline int
4780 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
4781 {
4782   snat_address_t * ap;
4783   clib_bihash_kv_8_8_t kv, value;
4784   snat_session_key_t m_key;
4785
4786   vec_foreach (ap, sm->addresses)
4787     {
4788       if (ap->addr.as_u32 == dst_addr->as_u32)
4789         return 1;
4790     }
4791
4792   m_key.addr.as_u32 = dst_addr->as_u32;
4793   m_key.fib_index = 0;
4794   m_key.port = 0;
4795   m_key.protocol = 0;
4796   kv.key = m_key.as_u64;
4797   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4798     return 1;
4799
4800   return 0;
4801 }
4802
4803 static inline uword
4804 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
4805                             vlib_node_runtime_t * node,
4806                             vlib_frame_t * frame,
4807                             int is_ed)
4808 {
4809   u32 n_left_from, * from, * to_next, stats_node_index;
4810   snat_in2out_next_t next_index;
4811   u32 pkts_processed = 0;
4812   snat_main_t * sm = &snat_main;
4813
4814   stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4815     snat_hairpin_dst_node.index;
4816
4817   from = vlib_frame_vector_args (frame);
4818   n_left_from = frame->n_vectors;
4819   next_index = node->cached_next_index;
4820
4821   while (n_left_from > 0)
4822     {
4823       u32 n_left_to_next;
4824
4825       vlib_get_next_frame (vm, node, next_index,
4826                            to_next, n_left_to_next);
4827
4828       while (n_left_from > 0 && n_left_to_next > 0)
4829         {
4830           u32 bi0;
4831           vlib_buffer_t * b0;
4832           u32 next0;
4833           ip4_header_t * ip0;
4834           u32 proto0;
4835
4836           /* speculatively enqueue b0 to the current next frame */
4837           bi0 = from[0];
4838           to_next[0] = bi0;
4839           from += 1;
4840           to_next += 1;
4841           n_left_from -= 1;
4842           n_left_to_next -= 1;
4843
4844           b0 = vlib_get_buffer (vm, bi0);
4845           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4846           ip0 = vlib_buffer_get_current (b0);
4847
4848           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4849
4850           vnet_buffer (b0)->snat.flags = 0;
4851           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
4852             {
4853               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
4854                 {
4855                   udp_header_t * udp0 = ip4_next_header (ip0);
4856                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
4857
4858                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
4859                 }
4860               else if (proto0 == SNAT_PROTOCOL_ICMP)
4861                 {
4862                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
4863
4864                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
4865                 }
4866               else
4867                 {
4868                   if (is_ed)
4869                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
4870                   else
4871                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
4872                 }
4873
4874               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
4875             }
4876
4877           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4878
4879           /* verify speculative enqueue, maybe switch current next frame */
4880           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4881                                            to_next, n_left_to_next,
4882                                            bi0, next0);
4883          }
4884
4885       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4886     }
4887
4888   vlib_node_increment_counter (vm, stats_node_index,
4889                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4890                                pkts_processed);
4891   return frame->n_vectors;
4892 }
4893
4894 static uword
4895 snat_hairpin_dst_fn (vlib_main_t * vm,
4896                      vlib_node_runtime_t * node,
4897                      vlib_frame_t * frame)
4898 {
4899   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
4900 }
4901
4902 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
4903   .function = snat_hairpin_dst_fn,
4904   .name = "nat44-hairpin-dst",
4905   .vector_size = sizeof (u32),
4906   .type = VLIB_NODE_TYPE_INTERNAL,
4907   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4908   .error_strings = snat_in2out_error_strings,
4909   .n_next_nodes = 2,
4910   .next_nodes = {
4911     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4912     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4913   },
4914 };
4915
4916 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
4917                               snat_hairpin_dst_fn);
4918
4919 static uword
4920 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
4921                          vlib_node_runtime_t * node,
4922                          vlib_frame_t * frame)
4923 {
4924   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
4925 }
4926
4927 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
4928   .function = nat44_ed_hairpin_dst_fn,
4929   .name = "nat44-ed-hairpin-dst",
4930   .vector_size = sizeof (u32),
4931   .type = VLIB_NODE_TYPE_INTERNAL,
4932   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4933   .error_strings = snat_in2out_error_strings,
4934   .n_next_nodes = 2,
4935   .next_nodes = {
4936     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4937     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4938   },
4939 };
4940
4941 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
4942                               nat44_ed_hairpin_dst_fn);
4943
4944 static inline uword
4945 snat_hairpin_src_fn_inline (vlib_main_t * vm,
4946                             vlib_node_runtime_t * node,
4947                             vlib_frame_t * frame,
4948                             int is_ed)
4949 {
4950   u32 n_left_from, * from, * to_next, stats_node_index;
4951   snat_in2out_next_t next_index;
4952   u32 pkts_processed = 0;
4953   snat_main_t *sm = &snat_main;
4954
4955   stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
4956     snat_hairpin_src_node.index;
4957
4958   from = vlib_frame_vector_args (frame);
4959   n_left_from = frame->n_vectors;
4960   next_index = node->cached_next_index;
4961
4962   while (n_left_from > 0)
4963     {
4964       u32 n_left_to_next;
4965
4966       vlib_get_next_frame (vm, node, next_index,
4967                            to_next, n_left_to_next);
4968
4969       while (n_left_from > 0 && n_left_to_next > 0)
4970         {
4971           u32 bi0;
4972           vlib_buffer_t * b0;
4973           u32 next0;
4974           snat_interface_t *i;
4975           u32 sw_if_index0;
4976
4977           /* speculatively enqueue b0 to the current next frame */
4978           bi0 = from[0];
4979           to_next[0] = bi0;
4980           from += 1;
4981           to_next += 1;
4982           n_left_from -= 1;
4983           n_left_to_next -= 1;
4984
4985           b0 = vlib_get_buffer (vm, bi0);
4986           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4987           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
4988
4989           pool_foreach (i, sm->output_feature_interfaces,
4990           ({
4991             /* Only packets from NAT inside interface */
4992             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
4993               {
4994                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
4995                                     SNAT_FLAG_HAIRPINNING))
4996                   {
4997                     if (PREDICT_TRUE (sm->num_workers > 1))
4998                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
4999                     else
5000                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5001                   }
5002                 break;
5003               }
5004           }));
5005
5006           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5007
5008           /* verify speculative enqueue, maybe switch current next frame */
5009           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5010                                            to_next, n_left_to_next,
5011                                            bi0, next0);
5012          }
5013
5014       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5015     }
5016
5017   vlib_node_increment_counter (vm, stats_node_index,
5018                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5019                                pkts_processed);
5020   return frame->n_vectors;
5021 }
5022
5023 static uword
5024 snat_hairpin_src_fn (vlib_main_t * vm,
5025                      vlib_node_runtime_t * node,
5026                      vlib_frame_t * frame)
5027 {
5028   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5029 }
5030
5031 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5032   .function = snat_hairpin_src_fn,
5033   .name = "nat44-hairpin-src",
5034   .vector_size = sizeof (u32),
5035   .type = VLIB_NODE_TYPE_INTERNAL,
5036   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5037   .error_strings = snat_in2out_error_strings,
5038   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5039   .next_nodes = {
5040      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5041      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5042      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5043      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5044   },
5045 };
5046
5047 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5048                               snat_hairpin_src_fn);
5049
5050 static uword
5051 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5052                          vlib_node_runtime_t * node,
5053                          vlib_frame_t * frame)
5054 {
5055   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5056 }
5057
5058 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5059   .function = nat44_ed_hairpin_src_fn,
5060   .name = "nat44-ed-hairpin-src",
5061   .vector_size = sizeof (u32),
5062   .type = VLIB_NODE_TYPE_INTERNAL,
5063   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5064   .error_strings = snat_in2out_error_strings,
5065   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5066   .next_nodes = {
5067      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5068      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5069      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5070      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5071   },
5072 };
5073
5074 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5075                               nat44_ed_hairpin_src_fn);
5076
5077 static uword
5078 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5079                                 vlib_node_runtime_t * node,
5080                                 vlib_frame_t * frame)
5081 {
5082   u32 n_left_from, * from, * to_next;
5083   snat_in2out_next_t next_index;
5084   u32 pkts_processed = 0;
5085   snat_main_t * sm = &snat_main;
5086   u32 stats_node_index;
5087
5088   stats_node_index = snat_in2out_fast_node.index;
5089
5090   from = vlib_frame_vector_args (frame);
5091   n_left_from = frame->n_vectors;
5092   next_index = node->cached_next_index;
5093
5094   while (n_left_from > 0)
5095     {
5096       u32 n_left_to_next;
5097
5098       vlib_get_next_frame (vm, node, next_index,
5099                            to_next, n_left_to_next);
5100
5101       while (n_left_from > 0 && n_left_to_next > 0)
5102         {
5103           u32 bi0;
5104           vlib_buffer_t * b0;
5105           u32 next0;
5106           u32 sw_if_index0;
5107           ip4_header_t * ip0;
5108           ip_csum_t sum0;
5109           u32 new_addr0, old_addr0;
5110           u16 old_port0, new_port0;
5111           udp_header_t * udp0;
5112           tcp_header_t * tcp0;
5113           icmp46_header_t * icmp0;
5114           snat_session_key_t key0, sm0;
5115           u32 proto0;
5116           u32 rx_fib_index0;
5117
5118           /* speculatively enqueue b0 to the current next frame */
5119           bi0 = from[0];
5120           to_next[0] = bi0;
5121           from += 1;
5122           to_next += 1;
5123           n_left_from -= 1;
5124           n_left_to_next -= 1;
5125
5126           b0 = vlib_get_buffer (vm, bi0);
5127           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5128
5129           ip0 = vlib_buffer_get_current (b0);
5130           udp0 = ip4_next_header (ip0);
5131           tcp0 = (tcp_header_t *) udp0;
5132           icmp0 = (icmp46_header_t *) udp0;
5133
5134           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5135           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5136
5137           if (PREDICT_FALSE(ip0->ttl == 1))
5138             {
5139               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5140               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5141                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
5142                                            0);
5143               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5144               goto trace0;
5145             }
5146
5147           proto0 = ip_proto_to_snat_proto (ip0->protocol);
5148
5149           if (PREDICT_FALSE (proto0 == ~0))
5150               goto trace0;
5151
5152           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5153             {
5154               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5155                                   rx_fib_index0, node, next0, ~0, 0, 0);
5156               goto trace0;
5157             }
5158
5159           key0.addr = ip0->src_address;
5160           key0.protocol = proto0;
5161           key0.port = udp0->src_port;
5162           key0.fib_index = rx_fib_index0;
5163
5164           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
5165             {
5166               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5167               next0= SNAT_IN2OUT_NEXT_DROP;
5168               goto trace0;
5169             }
5170
5171           new_addr0 = sm0.addr.as_u32;
5172           new_port0 = sm0.port;
5173           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5174           old_addr0 = ip0->src_address.as_u32;
5175           ip0->src_address.as_u32 = new_addr0;
5176
5177           sum0 = ip0->checksum;
5178           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5179                                  ip4_header_t,
5180                                  src_address /* changed member */);
5181           ip0->checksum = ip_csum_fold (sum0);
5182
5183           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5184             {
5185               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5186                 {
5187                   old_port0 = tcp0->src_port;
5188                   tcp0->src_port = new_port0;
5189
5190                   sum0 = tcp0->checksum;
5191                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5192                                          ip4_header_t,
5193                                          dst_address /* changed member */);
5194                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
5195                                          ip4_header_t /* cheat */,
5196                                          length /* changed member */);
5197                   tcp0->checksum = ip_csum_fold(sum0);
5198                 }
5199               else
5200                 {
5201                   old_port0 = udp0->src_port;
5202                   udp0->src_port = new_port0;
5203                   udp0->checksum = 0;
5204                 }
5205             }
5206           else
5207             {
5208               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5209                 {
5210                   sum0 = tcp0->checksum;
5211                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5212                                          ip4_header_t,
5213                                          dst_address /* changed member */);
5214                   tcp0->checksum = ip_csum_fold(sum0);
5215                 }
5216             }
5217
5218           /* Hairpinning */
5219           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5220
5221         trace0:
5222           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5223                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5224             {
5225               snat_in2out_trace_t *t =
5226                  vlib_add_trace (vm, node, b0, sizeof (*t));
5227               t->sw_if_index = sw_if_index0;
5228               t->next_index = next0;
5229             }
5230
5231           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5232
5233           /* verify speculative enqueue, maybe switch current next frame */
5234           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5235                                            to_next, n_left_to_next,
5236                                            bi0, next0);
5237         }
5238
5239       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5240     }
5241
5242   vlib_node_increment_counter (vm, stats_node_index,
5243                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5244                                pkts_processed);
5245   return frame->n_vectors;
5246 }
5247
5248
5249 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5250   .function = snat_in2out_fast_static_map_fn,
5251   .name = "nat44-in2out-fast",
5252   .vector_size = sizeof (u32),
5253   .format_trace = format_snat_in2out_fast_trace,
5254   .type = VLIB_NODE_TYPE_INTERNAL,
5255
5256   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5257   .error_strings = snat_in2out_error_strings,
5258
5259   .runtime_data_bytes = sizeof (snat_runtime_t),
5260
5261   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5262
5263   /* edit / add dispositions here */
5264   .next_nodes = {
5265     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5266     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5267     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5268     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5269     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5270   },
5271 };
5272
5273 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);