src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28 #include <nat/nat_inlines.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38   u32 is_slow_path;
  39 } snat_in2out_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_in2out_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  52   char * tag;
  53
  54   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  55
  56   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  57               t->sw_if_index, t->next_index, t->session_index);
  58
  59   return s;
  60 }
  61
  62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  75 {
  76   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  77   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  78   snat_in2out_worker_handoff_trace_t * t =
  79     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  80   char * m;
  81
  82   m = t->do_handoff ? "next worker" : "same worker";
  83   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  84
  85   return s;
  86 }
  87
  88 typedef struct {
  89   u32 sw_if_index;
  90   u32 next_index;
  91   u8 cached;
  92 } nat44_in2out_reass_trace_t;
  93
  94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  95 {
  96   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  97   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  98   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  99
 100   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 101               t->sw_if_index, t->next_index,
 102               t->cached ? "cached" : "translated");
 103
 104   return s;
 105 }
 106
 107 vlib_node_registration_t snat_in2out_node;
 108 vlib_node_registration_t snat_in2out_slowpath_node;
 109 vlib_node_registration_t snat_in2out_fast_node;
 110 vlib_node_registration_t snat_in2out_worker_handoff_node;
 111 vlib_node_registration_t snat_det_in2out_node;
 112 vlib_node_registration_t snat_in2out_output_node;
 113 vlib_node_registration_t snat_in2out_output_slowpath_node;
 114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 115 vlib_node_registration_t snat_hairpin_dst_node;
 116 vlib_node_registration_t snat_hairpin_src_node;
 117 vlib_node_registration_t nat44_hairpinning_node;
 118 vlib_node_registration_t nat44_in2out_reass_node;
 119 vlib_node_registration_t nat44_ed_in2out_node;
 120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
 121 vlib_node_registration_t nat44_ed_in2out_output_node;
 122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
 123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
 124 vlib_node_registration_t nat44_ed_hairpin_src_node;
 125 vlib_node_registration_t nat44_ed_hairpinning_node;
 126
 127 #define foreach_snat_in2out_error                       \
 128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 129 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 130 _(OUT_OF_PORTS, "Out of ports")                         \
 131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 132 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 133 _(NO_TRANSLATION, "No translation")                     \
 134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 135 _(DROP_FRAGMENT, "Drop fragment")                       \
 136 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 138 _(FQ_CONGESTED, "Handoff frame queue congested")
 139
 140 typedef enum {
 141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 142   foreach_snat_in2out_error
 143 #undef _
 144   SNAT_IN2OUT_N_ERROR,
 145 } snat_in2out_error_t;
 146
 147 static char * snat_in2out_error_strings[] = {
 148 #define _(sym,string) string,
 149   foreach_snat_in2out_error
 150 #undef _
 151 };
 152
 153 typedef enum {
 154   SNAT_IN2OUT_NEXT_LOOKUP,
 155   SNAT_IN2OUT_NEXT_DROP,
 156   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 157   SNAT_IN2OUT_NEXT_SLOW_PATH,
 158   SNAT_IN2OUT_NEXT_REASS,
 159   SNAT_IN2OUT_N_NEXT,
 160 } snat_in2out_next_t;
 161
 162 typedef enum {
 163   SNAT_HAIRPIN_SRC_NEXT_DROP,
 164   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 165   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 166   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 167   SNAT_HAIRPIN_SRC_N_NEXT,
 168 } snat_hairpin_next_t;
 169
 170 /**
 171  * @brief Check if packet should be translated
 172  *
 173  * Packets aimed at outside interface and external address with active session
 174  * should be translated.
 175  *
 176  * @param sm            NAT main
 177  * @param rt            NAT runtime data
 178  * @param sw_if_index0  index of the inside interface
 179  * @param ip0           IPv4 header
 180  * @param proto0        NAT protocol
 181  * @param rx_fib_index0 RX FIB index
 182  *
 183  * @returns 0 if packet should be translated otherwise 1
 184  */
 185 static inline int
 186 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 187                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 188                          u32 rx_fib_index0)
 189 {
 190   if (sm->out2in_dpo)
 191     return 0;
 192
 193   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 194   nat_outside_fib_t *outside_fib;
 195   fib_prefix_t pfx = {
 196     .fp_proto = FIB_PROTOCOL_IP4,
 197     .fp_len = 32,
 198     .fp_addr = {
 199         .ip4.as_u32 = ip0->dst_address.as_u32,
 200     },
 201   };
 202
 203   /* Don't NAT packet aimed at the intfc address */
 204   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 205                                       ip0->dst_address.as_u32)))
 206     return 1;
 207
 208   fei = fib_table_lookup (rx_fib_index0, &pfx);
 209   if (FIB_NODE_INDEX_INVALID != fei)
 210     {
 211       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 212       if (sw_if_index == ~0)
 213         {
 214           vec_foreach (outside_fib, sm->outside_fibs)
 215             {
 216               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 217               if (FIB_NODE_INDEX_INVALID != fei)
 218                 {
 219                   sw_if_index = fib_entry_get_resolving_interface (fei);
 220                   if (sw_if_index != ~0)
 221                     break;
 222                 }
 223             }
 224         }
 225       if (sw_if_index == ~0)
 226         return 1;
 227
 228       snat_interface_t *i;
 229       pool_foreach (i, sm->interfaces,
 230       ({
 231         /* NAT packet aimed at outside interface */
 232         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 233           return 0;
 234       }));
 235     }
 236
 237   return 1;
 238 }
 239
 240 static inline int
 241 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 242                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 243                     u32 rx_fib_index0, u32 thread_index)
 244 {
 245   udp_header_t * udp0 = ip4_next_header (ip0);
 246   snat_session_key_t key0, sm0;
 247   clib_bihash_kv_8_8_t kv0, value0;
 248
 249   key0.addr = ip0->dst_address;
 250   key0.port = udp0->dst_port;
 251   key0.protocol = proto0;
 252   key0.fib_index = sm->outside_fib_index;
 253   kv0.key = key0.as_u64;
 254
 255   /* NAT packet aimed at external address if */
 256   /* has active sessions */
 257   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 258                               &value0))
 259     {
 260       /* or is static mappings */
 261       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 262         return 0;
 263     }
 264   else
 265     return 0;
 266
 267   if (sm->forwarding_enabled)
 268     return 1;
 269
 270   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 271                                  rx_fib_index0);
 272 }
 273
 274 static inline int
 275 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 276                                   u32 proto0, u16 src_port, u16 dst_port,
 277                                   u32 thread_index, u32 sw_if_index)
 278 {
 279   snat_session_key_t key0;
 280   clib_bihash_kv_8_8_t kv0, value0;
 281   snat_interface_t *i;
 282
 283   /* src NAT check */
 284   key0.addr = ip0->src_address;
 285   key0.port = src_port;
 286   key0.protocol = proto0;
 287   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 288   kv0.key = key0.as_u64;
 289
 290   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 291                                &value0))
 292     return 1;
 293
 294   /* dst NAT check */
 295   key0.addr = ip0->dst_address;
 296   key0.port = dst_port;
 297   key0.protocol = proto0;
 298   kv0.key = key0.as_u64;
 299   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 300                                &value0))
 301   {
 302     /* hairpinning */
 303     pool_foreach (i, sm->output_feature_interfaces,
 304     ({
 305       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 306         return 0;
 307     }));
 308     return 1;
 309   }
 310
 311   return 0;
 312 }
 313
 314 int
 315 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void * arg)
 316 {
 317   snat_main_t *sm = &snat_main;
 318   nat44_is_idle_session_ctx_t *ctx = arg;
 319   snat_session_t *s;
 320   u64 sess_timeout_time;
 321   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 322                                                        ctx->thread_index);
 323   clib_bihash_kv_8_8_t s_kv;
 324
 325   s = pool_elt_at_index (tsm->sessions, kv->value);
 326   sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
 327   if (ctx->now >= sess_timeout_time)
 328     {
 329       s_kv.key = s->out2in.as_u64;
 330       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 331         nat_log_warn ("out2in key del failed");
 332
 333       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 334                                           s->out2in.addr.as_u32,
 335                                           s->in2out.protocol,
 336                                           s->in2out.port,
 337                                           s->out2in.port,
 338                                           s->in2out.fib_index);
 339
 340       if (!snat_is_session_static (s))
 341         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 342                                             &s->out2in);
 343
 344       nat44_delete_session (sm, s, ctx->thread_index);
 345       return 1;
 346     }
 347
 348   return 0;
 349 }
 350
 351 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 352                       ip4_header_t * ip0,
 353                       u32 rx_fib_index0,
 354                       snat_session_key_t * key0,
 355                       snat_session_t ** sessionp,
 356                       vlib_node_runtime_t * node,
 357                       u32 next0,
 358                       u32 thread_index,
 359                       f64 now)
 360 {
 361   snat_user_t *u;
 362   snat_session_t *s;
 363   clib_bihash_kv_8_8_t kv0;
 364   snat_session_key_t key1;
 365   u32 address_index = ~0;
 366   udp_header_t * udp0 = ip4_next_header (ip0);
 367   u8 is_sm = 0;
 368   nat_outside_fib_t *outside_fib;
 369   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 370   fib_prefix_t pfx = {
 371     .fp_proto = FIB_PROTOCOL_IP4,
 372     .fp_len = 32,
 373     .fp_addr = {
 374         .ip4.as_u32 = ip0->dst_address.as_u32,
 375     },
 376   };
 377   nat44_is_idle_session_ctx_t ctx0;
 378
 379   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 380     {
 381       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 382       nat_ipfix_logging_max_sessions(sm->max_translations);
 383       nat_log_notice ("maximum sessions exceeded");
 384       return SNAT_IN2OUT_NEXT_DROP;
 385     }
 386
 387   key1.protocol = key0->protocol;
 388
 389   /* First try to match static mapping by local address and port */
 390   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 391     {
 392       /* Try to create dynamic translation */
 393       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 394                                                thread_index, &key1,
 395                                                &address_index,
 396                                                sm->port_per_thread,
 397                                                sm->per_thread_data[thread_index].snat_thread_index))
 398         {
 399           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 400           return SNAT_IN2OUT_NEXT_DROP;
 401         }
 402     }
 403   else
 404     is_sm = 1;
 405
 406   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 407                               thread_index);
 408   if (!u)
 409     {
 410       nat_log_warn ("create NAT user failed");
 411       return SNAT_IN2OUT_NEXT_DROP;
 412     }
 413
 414   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 415   if (!s)
 416     {
 417       nat44_delete_user_with_no_session (sm, u, thread_index);
 418       nat_log_warn ("create NAT session failed");
 419       return SNAT_IN2OUT_NEXT_DROP;
 420     }
 421
 422   if (is_sm)
 423     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 424   user_session_increment (sm, u, is_sm);
 425   s->outside_address_index = address_index;
 426   s->in2out = *key0;
 427   s->out2in = key1;
 428   s->out2in.protocol = key0->protocol;
 429   s->out2in.fib_index = sm->outside_fib_index;
 430   switch (vec_len (sm->outside_fibs))
 431     {
 432     case 0:
 433       s->out2in.fib_index = sm->outside_fib_index;
 434       break;
 435     case 1:
 436       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 437       break;
 438     default:
 439       vec_foreach (outside_fib, sm->outside_fibs)
 440         {
 441           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 442           if (FIB_NODE_INDEX_INVALID != fei)
 443             {
 444               if (fib_entry_get_resolving_interface (fei) != ~0)
 445                 {
 446                   s->out2in.fib_index = outside_fib->fib_index;
 447                   break;
 448                 }
 449             }
 450         }
 451       break;
 452     }
 453   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 454   s->ext_host_port = udp0->dst_port;
 455   *sessionp = s;
 456
 457   /* Add to translation hashes */
 458   ctx0.now = now;
 459   ctx0.thread_index = thread_index;
 460   kv0.key = s->in2out.as_u64;
 461   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 462   if (clib_bihash_add_or_overwrite_stale_8_8 (
 463         &sm->per_thread_data[thread_index].in2out, &kv0,
 464         nat44_i2o_is_idle_session_cb, &ctx0))
 465       nat_log_notice ("in2out key add failed");
 466
 467   kv0.key = s->out2in.as_u64;
 468   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 469
 470   if (clib_bihash_add_or_overwrite_stale_8_8 (
 471         &sm->per_thread_data[thread_index].out2in, &kv0,
 472         nat44_o2i_is_idle_session_cb, &ctx0))
 473       nat_log_notice ("out2in key add failed");
 474
 475   /* log NAT event */
 476   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 477                                       s->out2in.addr.as_u32,
 478                                       s->in2out.protocol,
 479                                       s->in2out.port,
 480                                       s->out2in.port,
 481                                       s->in2out.fib_index);
 482   return next0;
 483 }
 484
 485 static_always_inline
 486 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 487                                  snat_session_key_t *p_key0)
 488 {
 489   icmp46_header_t *icmp0;
 490   snat_session_key_t key0;
 491   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 492   ip4_header_t *inner_ip0 = 0;
 493   void *l4_header = 0;
 494   icmp46_header_t *inner_icmp0;
 495
 496   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 497   echo0 = (icmp_echo_header_t *)(icmp0+1);
 498
 499   if (!icmp_is_error_message (icmp0))
 500     {
 501       key0.protocol = SNAT_PROTOCOL_ICMP;
 502       key0.addr = ip0->src_address;
 503       key0.port = echo0->identifier;
 504     }
 505   else
 506     {
 507       inner_ip0 = (ip4_header_t *)(echo0+1);
 508       l4_header = ip4_next_header (inner_ip0);
 509       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 510       key0.addr = inner_ip0->dst_address;
 511       switch (key0.protocol)
 512         {
 513         case SNAT_PROTOCOL_ICMP:
 514           inner_icmp0 = (icmp46_header_t*)l4_header;
 515           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 516           key0.port = inner_echo0->identifier;
 517           break;
 518         case SNAT_PROTOCOL_UDP:
 519         case SNAT_PROTOCOL_TCP:
 520           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 521           break;
 522         default:
 523           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 524         }
 525     }
 526   *p_key0 = key0;
 527   return -1; /* success */
 528 }
 529
 530 /**
 531  * Get address and port values to be used for ICMP packet translation
 532  * and create session if needed
 533  *
 534  * @param[in,out] sm             NAT main
 535  * @param[in,out] node           NAT node runtime
 536  * @param[in] thread_index       thread index
 537  * @param[in,out] b0             buffer containing packet to be translated
 538  * @param[out] p_proto           protocol used for matching
 539  * @param[out] p_value           address and port after NAT translation
 540  * @param[out] p_dont_translate  if packet should not be translated
 541  * @param d                      optional parameter
 542  * @param e                      optional parameter
 543  */
 544 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 545                            u32 thread_index, vlib_buffer_t *b0,
 546                            ip4_header_t *ip0, u8 *p_proto,
 547                            snat_session_key_t *p_value,
 548                            u8 *p_dont_translate, void *d, void *e)
 549 {
 550   icmp46_header_t *icmp0;
 551   u32 sw_if_index0;
 552   u32 rx_fib_index0;
 553   snat_session_key_t key0;
 554   snat_session_t *s0 = 0;
 555   u8 dont_translate = 0;
 556   clib_bihash_kv_8_8_t kv0, value0;
 557   u32 next0 = ~0;
 558   int err;
 559
 560   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 561   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 562   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 563
 564   err = icmp_get_key (ip0, &key0);
 565   if (err != -1)
 566     {
 567       b0->error = node->errors[err];
 568       next0 = SNAT_IN2OUT_NEXT_DROP;
 569       goto out;
 570     }
 571   key0.fib_index = rx_fib_index0;
 572
 573   kv0.key = key0.as_u64;
 574
 575   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 576                               &value0))
 577     {
 578       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 579         {
 580           if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
 581               key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
 582             {
 583               dont_translate = 1;
 584               goto out;
 585             }
 586         }
 587       else
 588         {
 589           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 590               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 591             {
 592               dont_translate = 1;
 593               goto out;
 594             }
 595         }
 596
 597       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 598         {
 599           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 600           next0 = SNAT_IN2OUT_NEXT_DROP;
 601           goto out;
 602         }
 603
 604       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 605                          thread_index, vlib_time_now (sm->vlib_main));
 606
 607       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 608         goto out;
 609     }
 610   else
 611     {
 612       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 613                         icmp0->type != ICMP4_echo_reply &&
 614                         !icmp_is_error_message (icmp0)))
 615         {
 616           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 617           next0 = SNAT_IN2OUT_NEXT_DROP;
 618           goto out;
 619         }
 620
 621       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 622                               value0.value);
 623     }
 624
 625 out:
 626   *p_proto = key0.protocol;
 627   if (s0)
 628     *p_value = s0->out2in;
 629   *p_dont_translate = dont_translate;
 630   if (d)
 631     *(snat_session_t**)d = s0;
 632   return next0;
 633 }
 634
 635 /**
 636  * Get address and port values to be used for ICMP packet translation
 637  *
 638  * @param[in] sm                 NAT main
 639  * @param[in,out] node           NAT node runtime
 640  * @param[in] thread_index       thread index
 641  * @param[in,out] b0             buffer containing packet to be translated
 642  * @param[out] p_proto           protocol used for matching
 643  * @param[out] p_value           address and port after NAT translation
 644  * @param[out] p_dont_translate  if packet should not be translated
 645  * @param d                      optional parameter
 646  * @param e                      optional parameter
 647  */
 648 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 649                            u32 thread_index, vlib_buffer_t *b0,
 650                            ip4_header_t *ip0, u8 *p_proto,
 651                            snat_session_key_t *p_value,
 652                            u8 *p_dont_translate, void *d, void *e)
 653 {
 654   icmp46_header_t *icmp0;
 655   u32 sw_if_index0;
 656   u32 rx_fib_index0;
 657   snat_session_key_t key0;
 658   snat_session_key_t sm0;
 659   u8 dont_translate = 0;
 660   u8 is_addr_only;
 661   u32 next0 = ~0;
 662   int err;
 663
 664   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 665   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 666   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 667
 668   err = icmp_get_key (ip0, &key0);
 669   if (err != -1)
 670     {
 671       b0->error = node->errors[err];
 672       next0 = SNAT_IN2OUT_NEXT_DROP;
 673       goto out2;
 674     }
 675   key0.fib_index = rx_fib_index0;
 676
 677   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 678     {
 679       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 680           IP_PROTOCOL_ICMP, rx_fib_index0)))
 681         {
 682           dont_translate = 1;
 683           goto out;
 684         }
 685
 686       if (icmp_is_error_message (icmp0))
 687         {
 688           next0 = SNAT_IN2OUT_NEXT_DROP;
 689           goto out;
 690         }
 691
 692       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 693       next0 = SNAT_IN2OUT_NEXT_DROP;
 694       goto out;
 695     }
 696
 697   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 698                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 699                     !icmp_is_error_message (icmp0)))
 700     {
 701       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 702       next0 = SNAT_IN2OUT_NEXT_DROP;
 703       goto out;
 704     }
 705
 706 out:
 707   *p_value = sm0;
 708 out2:
 709   *p_proto = key0.protocol;
 710   *p_dont_translate = dont_translate;
 711   return next0;
 712 }
 713
 714 static inline u32 icmp_in2out (snat_main_t *sm,
 715                                vlib_buffer_t * b0,
 716                                ip4_header_t * ip0,
 717                                icmp46_header_t * icmp0,
 718                                u32 sw_if_index0,
 719                                u32 rx_fib_index0,
 720                                vlib_node_runtime_t * node,
 721                                u32 next0,
 722                                u32 thread_index,
 723                                void *d,
 724                                void *e)
 725 {
 726   snat_session_key_t sm0;
 727   u8 protocol;
 728   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 729   ip4_header_t *inner_ip0;
 730   void *l4_header = 0;
 731   icmp46_header_t *inner_icmp0;
 732   u8 dont_translate;
 733   u32 new_addr0, old_addr0;
 734   u16 old_id0, new_id0;
 735   ip_csum_t sum0;
 736   u16 checksum0;
 737   u32 next0_tmp;
 738
 739   echo0 = (icmp_echo_header_t *)(icmp0+1);
 740
 741   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 742                                        &protocol, &sm0, &dont_translate, d, e);
 743   if (next0_tmp != ~0)
 744     next0 = next0_tmp;
 745   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 746     goto out;
 747
 748   sum0 = ip_incremental_checksum (0, icmp0,
 749                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 750   checksum0 = ~ip_csum_fold (sum0);
 751   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 752     {
 753       next0 = SNAT_IN2OUT_NEXT_DROP;
 754       goto out;
 755     }
 756
 757   old_addr0 = ip0->src_address.as_u32;
 758   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 759   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 760     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 761
 762   sum0 = ip0->checksum;
 763   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 764                          src_address /* changed member */);
 765   ip0->checksum = ip_csum_fold (sum0);
 766
 767   if (icmp0->checksum == 0)
 768     icmp0->checksum = 0xffff;
 769
 770   if (!icmp_is_error_message (icmp0))
 771     {
 772       new_id0 = sm0.port;
 773       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 774         {
 775           old_id0 = echo0->identifier;
 776           new_id0 = sm0.port;
 777           echo0->identifier = new_id0;
 778
 779           sum0 = icmp0->checksum;
 780           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 781                                  identifier);
 782           icmp0->checksum = ip_csum_fold (sum0);
 783         }
 784     }
 785   else
 786     {
 787       inner_ip0 = (ip4_header_t *)(echo0+1);
 788       l4_header = ip4_next_header (inner_ip0);
 789
 790       if (!ip4_header_checksum_is_valid (inner_ip0))
 791         {
 792           next0 = SNAT_IN2OUT_NEXT_DROP;
 793           goto out;
 794         }
 795
 796       old_addr0 = inner_ip0->dst_address.as_u32;
 797       inner_ip0->dst_address = sm0.addr;
 798       new_addr0 = inner_ip0->dst_address.as_u32;
 799
 800       sum0 = icmp0->checksum;
 801       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 802                              dst_address /* changed member */);
 803       icmp0->checksum = ip_csum_fold (sum0);
 804
 805       switch (protocol)
 806         {
 807           case SNAT_PROTOCOL_ICMP:
 808             inner_icmp0 = (icmp46_header_t*)l4_header;
 809             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 810
 811             old_id0 = inner_echo0->identifier;
 812             new_id0 = sm0.port;
 813             inner_echo0->identifier = new_id0;
 814
 815             sum0 = icmp0->checksum;
 816             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 817                                    identifier);
 818             icmp0->checksum = ip_csum_fold (sum0);
 819             break;
 820           case SNAT_PROTOCOL_UDP:
 821           case SNAT_PROTOCOL_TCP:
 822             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 823             new_id0 = sm0.port;
 824             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 825
 826             sum0 = icmp0->checksum;
 827             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 828                                    dst_port);
 829             icmp0->checksum = ip_csum_fold (sum0);
 830             break;
 831           default:
 832             ASSERT(0);
 833         }
 834     }
 835
 836 out:
 837   return next0;
 838 }
 839
 840 /**
 841  * @brief Hairpinning
 842  *
 843  * Hairpinning allows two endpoints on the internal side of the NAT to
 844  * communicate even if they only use each other's external IP addresses
 845  * and ports.
 846  *
 847  * @param sm     NAT main.
 848  * @param b0     Vlib buffer.
 849  * @param ip0    IP header.
 850  * @param udp0   UDP header.
 851  * @param tcp0   TCP header.
 852  * @param proto0 NAT protocol.
 853  */
 854 static inline int
 855 snat_hairpinning (snat_main_t *sm,
 856                   vlib_buffer_t * b0,
 857                   ip4_header_t * ip0,
 858                   udp_header_t * udp0,
 859                   tcp_header_t * tcp0,
 860                   u32 proto0,
 861                   int is_ed)
 862 {
 863   snat_session_key_t key0, sm0;
 864   snat_session_t * s0;
 865   clib_bihash_kv_8_8_t kv0, value0;
 866   ip_csum_t sum0;
 867   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 868   u16 new_dst_port0, old_dst_port0;
 869   int rv;
 870
 871   key0.addr = ip0->dst_address;
 872   key0.port = udp0->dst_port;
 873   key0.protocol = proto0;
 874   key0.fib_index = sm->outside_fib_index;
 875   kv0.key = key0.as_u64;
 876
 877   /* Check if destination is static mappings */
 878   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 879     {
 880       new_dst_addr0 = sm0.addr.as_u32;
 881       new_dst_port0 = sm0.port;
 882       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 883     }
 884   /* or active session */
 885   else
 886     {
 887       if (sm->num_workers > 1)
 888         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 889       else
 890         ti = sm->num_workers;
 891
 892       if (is_ed)
 893         {
 894           clib_bihash_kv_16_8_t ed_kv, ed_value;
 895           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 896                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 897                       udp0->src_port);
 898           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 899                                         &ed_kv, &ed_value);
 900           si = ed_value.value;
 901         }
 902       else
 903         {
 904           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 905                                        &value0);
 906           si = value0.value;
 907         }
 908       if (rv)
 909         return 0;
 910
 911       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 912       new_dst_addr0 = s0->in2out.addr.as_u32;
 913       new_dst_port0 = s0->in2out.port;
 914       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 915     }
 916
 917   /* Destination is behind the same NAT, use internal address and port */
 918   if (new_dst_addr0)
 919     {
 920       old_dst_addr0 = ip0->dst_address.as_u32;
 921       ip0->dst_address.as_u32 = new_dst_addr0;
 922       sum0 = ip0->checksum;
 923       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 924                              ip4_header_t, dst_address);
 925       ip0->checksum = ip_csum_fold (sum0);
 926
 927       old_dst_port0 = tcp0->dst;
 928       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 929         {
 930           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 931             {
 932               tcp0->dst = new_dst_port0;
 933               sum0 = tcp0->checksum;
 934               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 935                                      ip4_header_t, dst_address);
 936               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 937                                      ip4_header_t /* cheat */, length);
 938               tcp0->checksum = ip_csum_fold(sum0);
 939             }
 940           else
 941             {
 942               udp0->dst_port = new_dst_port0;
 943               udp0->checksum = 0;
 944             }
 945         }
 946       else
 947         {
 948           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 949             {
 950               sum0 = tcp0->checksum;
 951               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 952                                      ip4_header_t, dst_address);
 953               tcp0->checksum = ip_csum_fold(sum0);
 954             }
 955         }
 956       return 1;
 957     }
 958   return 0;
 959 }
 960
 961 static inline void
 962 snat_icmp_hairpinning (snat_main_t *sm,
 963                        vlib_buffer_t * b0,
 964                        ip4_header_t * ip0,
 965                        icmp46_header_t * icmp0,
 966                        int is_ed)
 967 {
 968   snat_session_key_t key0, sm0;
 969   clib_bihash_kv_8_8_t kv0, value0;
 970   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 971   ip_csum_t sum0;
 972   snat_session_t *s0;
 973   int rv;
 974
 975   if (!icmp_is_error_message (icmp0))
 976     {
 977       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 978       u16 icmp_id0 = echo0->identifier;
 979       key0.addr = ip0->dst_address;
 980       key0.port = icmp_id0;
 981       key0.protocol = SNAT_PROTOCOL_ICMP;
 982       key0.fib_index = sm->outside_fib_index;
 983       kv0.key = key0.as_u64;
 984
 985       if (sm->num_workers > 1)
 986         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 987       else
 988         ti = sm->num_workers;
 989
 990       /* Check if destination is in active sessions */
 991       if (is_ed)
 992         {
 993           clib_bihash_kv_16_8_t ed_kv, ed_value;
 994           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 995                       IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
 996           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 997                                         &ed_kv, &ed_value);
 998           si = ed_value.value;
 999         }
1000       else
1001         {
1002           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1003                                        &value0);
1004           si = value0.value;
1005         }
1006       if (rv)
1007         {
1008           /* or static mappings */
1009           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1010             {
1011               new_dst_addr0 = sm0.addr.as_u32;
1012               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1013             }
1014         }
1015       else
1016         {
1017           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1018           new_dst_addr0 = s0->in2out.addr.as_u32;
1019           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1020           echo0->identifier = s0->in2out.port;
1021           sum0 = icmp0->checksum;
1022           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1023                                  icmp_echo_header_t, identifier);
1024           icmp0->checksum = ip_csum_fold (sum0);
1025         }
1026
1027       /* Destination is behind the same NAT, use internal address and port */
1028       if (new_dst_addr0)
1029         {
1030           old_dst_addr0 = ip0->dst_address.as_u32;
1031           ip0->dst_address.as_u32 = new_dst_addr0;
1032           sum0 = ip0->checksum;
1033           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1034                                  ip4_header_t, dst_address);
1035           ip0->checksum = ip_csum_fold (sum0);
1036         }
1037     }
1038
1039 }
1040
1041 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1042                                          vlib_buffer_t * b0,
1043                                          ip4_header_t * ip0,
1044                                          icmp46_header_t * icmp0,
1045                                          u32 sw_if_index0,
1046                                          u32 rx_fib_index0,
1047                                          vlib_node_runtime_t * node,
1048                                          u32 next0,
1049                                          f64 now,
1050                                          u32 thread_index,
1051                                          snat_session_t ** p_s0)
1052 {
1053   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1054                       next0, thread_index, p_s0, 0);
1055   snat_session_t * s0 = *p_s0;
1056   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1057     {
1058       /* Hairpinning */
1059       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1060         snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1061       /* Accounting */
1062       nat44_session_update_counters (s0, now,
1063                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
1064       /* Per-user LRU list maintenance */
1065       nat44_session_update_lru (sm, s0, thread_index);
1066     }
1067   return next0;
1068 }
1069
1070 static inline void
1071 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1072                                   vlib_buffer_t * b,
1073                                   ip4_header_t * ip)
1074 {
1075   clib_bihash_kv_8_8_t kv, value;
1076   snat_static_mapping_t *m;
1077   u32 old_addr, new_addr;
1078   ip_csum_t sum;
1079
1080   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1081   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1082     return;
1083
1084   m = pool_elt_at_index (sm->static_mappings, value.value);
1085
1086   old_addr = ip->dst_address.as_u32;
1087   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1088   sum = ip->checksum;
1089   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1090   ip->checksum = ip_csum_fold (sum);
1091
1092   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1093     vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1094 }
1095
1096 static int
1097 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1098                              vlib_buffer_t * b,
1099                              ip4_header_t * ip,
1100                              u32 rx_fib_index)
1101 {
1102   clib_bihash_kv_8_8_t kv, value;
1103   snat_static_mapping_t *m;
1104   snat_session_key_t m_key;
1105   u32 old_addr, new_addr;
1106   ip_csum_t sum;
1107
1108   m_key.addr = ip->src_address;
1109   m_key.port = 0;
1110   m_key.protocol = 0;
1111   m_key.fib_index = rx_fib_index;
1112   kv.key = m_key.as_u64;
1113   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1114     return 1;
1115
1116   m = pool_elt_at_index (sm->static_mappings, value.value);
1117
1118   old_addr = ip->src_address.as_u32;
1119   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1120   sum = ip->checksum;
1121   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1122   ip->checksum = ip_csum_fold (sum);
1123
1124
1125   /* Hairpinning */
1126   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1127     {
1128       vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1129       nat_hairpinning_sm_unknown_proto (sm, b, ip);
1130     }
1131
1132   return 0;
1133 }
1134
1135 static inline uword
1136 snat_in2out_node_fn_inline (vlib_main_t * vm,
1137                             vlib_node_runtime_t * node,
1138                             vlib_frame_t * frame, int is_slow_path,
1139                             int is_output_feature)
1140 {
1141   u32 n_left_from, * from, * to_next;
1142   snat_in2out_next_t next_index;
1143   u32 pkts_processed = 0;
1144   snat_main_t * sm = &snat_main;
1145   f64 now = vlib_time_now (vm);
1146   u32 stats_node_index;
1147   u32 thread_index = vm->thread_index;
1148
1149   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1150     snat_in2out_node.index;
1151
1152   from = vlib_frame_vector_args (frame);
1153   n_left_from = frame->n_vectors;
1154   next_index = node->cached_next_index;
1155
1156   while (n_left_from > 0)
1157     {
1158       u32 n_left_to_next;
1159
1160       vlib_get_next_frame (vm, node, next_index,
1161                            to_next, n_left_to_next);
1162
1163       while (n_left_from >= 4 && n_left_to_next >= 2)
1164         {
1165           u32 bi0, bi1;
1166           vlib_buffer_t * b0, * b1;
1167           u32 next0, next1;
1168           u32 sw_if_index0, sw_if_index1;
1169           ip4_header_t * ip0, * ip1;
1170           ip_csum_t sum0, sum1;
1171           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1172           u16 old_port0, new_port0, old_port1, new_port1;
1173           udp_header_t * udp0, * udp1;
1174           tcp_header_t * tcp0, * tcp1;
1175           icmp46_header_t * icmp0, * icmp1;
1176           snat_session_key_t key0, key1;
1177           u32 rx_fib_index0, rx_fib_index1;
1178           u32 proto0, proto1;
1179           snat_session_t * s0 = 0, * s1 = 0;
1180           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1181           u32 iph_offset0 = 0, iph_offset1 = 0;
1182
1183           /* Prefetch next iteration. */
1184           {
1185             vlib_buffer_t * p2, * p3;
1186
1187             p2 = vlib_get_buffer (vm, from[2]);
1188             p3 = vlib_get_buffer (vm, from[3]);
1189
1190             vlib_prefetch_buffer_header (p2, LOAD);
1191             vlib_prefetch_buffer_header (p3, LOAD);
1192
1193             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1194             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1195           }
1196
1197           /* speculatively enqueue b0 and b1 to the current next frame */
1198           to_next[0] = bi0 = from[0];
1199           to_next[1] = bi1 = from[1];
1200           from += 2;
1201           to_next += 2;
1202           n_left_from -= 2;
1203           n_left_to_next -= 2;
1204
1205           b0 = vlib_get_buffer (vm, bi0);
1206           b1 = vlib_get_buffer (vm, bi1);
1207
1208           if (is_output_feature)
1209             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1210
1211           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1212                  iph_offset0);
1213
1214           udp0 = ip4_next_header (ip0);
1215           tcp0 = (tcp_header_t *) udp0;
1216           icmp0 = (icmp46_header_t *) udp0;
1217
1218           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1219           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1220                                    sw_if_index0);
1221
1222           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1223
1224           if (PREDICT_FALSE(ip0->ttl == 1))
1225             {
1226               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1227               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1228                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1229                                            0);
1230               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1231               goto trace00;
1232             }
1233
1234           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1235
1236           /* Next configured feature, probably ip4-lookup */
1237           if (is_slow_path)
1238             {
1239               if (PREDICT_FALSE (proto0 == ~0))
1240                 {
1241                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1242                     {
1243                       next0 = SNAT_IN2OUT_NEXT_DROP;
1244                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1245                     }
1246                   goto trace00;
1247                 }
1248
1249               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1250                 {
1251                   next0 = icmp_in2out_slow_path
1252                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1253                      node, next0, now, thread_index, &s0);
1254                   goto trace00;
1255                 }
1256             }
1257           else
1258             {
1259               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1260                 {
1261                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1262                   goto trace00;
1263                 }
1264
1265               if (ip4_is_fragment (ip0))
1266                 {
1267                   next0 = SNAT_IN2OUT_NEXT_REASS;
1268                   goto trace00;
1269                 }
1270             }
1271
1272           key0.addr = ip0->src_address;
1273           key0.port = udp0->src_port;
1274           key0.protocol = proto0;
1275           key0.fib_index = rx_fib_index0;
1276
1277           kv0.key = key0.as_u64;
1278
1279           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1280               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1281             {
1282               if (is_slow_path)
1283                 {
1284                   if (is_output_feature)
1285                     {
1286                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1287                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1288                         goto trace00;
1289                     }
1290                   else
1291                     {
1292                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1293                           ip0, proto0, rx_fib_index0, thread_index)))
1294                         goto trace00;
1295                     }
1296
1297                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1298                                      &s0, node, next0, thread_index, now);
1299                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1300                     goto trace00;
1301                 }
1302               else
1303                 {
1304                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1305                   goto trace00;
1306                 }
1307             }
1308           else
1309             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1310                                     value0.value);
1311
1312           b0->flags |= VNET_BUFFER_F_IS_NATED;
1313
1314           old_addr0 = ip0->src_address.as_u32;
1315           ip0->src_address = s0->out2in.addr;
1316           new_addr0 = ip0->src_address.as_u32;
1317           if (!is_output_feature)
1318             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1319
1320           sum0 = ip0->checksum;
1321           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1322                                  ip4_header_t,
1323                                  src_address /* changed member */);
1324           ip0->checksum = ip_csum_fold (sum0);
1325
1326           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1327             {
1328               old_port0 = tcp0->src_port;
1329               tcp0->src_port = s0->out2in.port;
1330               new_port0 = tcp0->src_port;
1331
1332               sum0 = tcp0->checksum;
1333               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1334                                      ip4_header_t,
1335                                      dst_address /* changed member */);
1336               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1337                                      ip4_header_t /* cheat */,
1338                                      length /* changed member */);
1339               tcp0->checksum = ip_csum_fold(sum0);
1340             }
1341           else
1342             {
1343               old_port0 = udp0->src_port;
1344               udp0->src_port = s0->out2in.port;
1345               udp0->checksum = 0;
1346             }
1347
1348           /* Accounting */
1349           nat44_session_update_counters (s0, now,
1350                                          vlib_buffer_length_in_chain (vm, b0));
1351           /* Per-user LRU list maintenance */
1352           nat44_session_update_lru (sm, s0, thread_index);
1353         trace00:
1354
1355           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1356                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1357             {
1358               snat_in2out_trace_t *t =
1359                  vlib_add_trace (vm, node, b0, sizeof (*t));
1360               t->is_slow_path = is_slow_path;
1361               t->sw_if_index = sw_if_index0;
1362               t->next_index = next0;
1363                   t->session_index = ~0;
1364               if (s0)
1365                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1366             }
1367
1368           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1369
1370           if (is_output_feature)
1371             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1372
1373           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1374                  iph_offset1);
1375
1376           udp1 = ip4_next_header (ip1);
1377           tcp1 = (tcp_header_t *) udp1;
1378           icmp1 = (icmp46_header_t *) udp1;
1379
1380           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1381           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1382                                    sw_if_index1);
1383
1384           if (PREDICT_FALSE(ip1->ttl == 1))
1385             {
1386               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1387               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1388                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1389                                            0);
1390               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1391               goto trace01;
1392             }
1393
1394           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1395
1396           /* Next configured feature, probably ip4-lookup */
1397           if (is_slow_path)
1398             {
1399               if (PREDICT_FALSE (proto1 == ~0))
1400                 {
1401                   if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1402                     {
1403                       next1 = SNAT_IN2OUT_NEXT_DROP;
1404                       b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1405                     }
1406                   goto trace01;
1407                 }
1408
1409               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1410                 {
1411                   next1 = icmp_in2out_slow_path
1412                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1413                      next1, now, thread_index, &s1);
1414                   goto trace01;
1415                 }
1416             }
1417           else
1418             {
1419               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1420                 {
1421                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1422                   goto trace01;
1423                 }
1424
1425               if (ip4_is_fragment (ip1))
1426                 {
1427                   next1 = SNAT_IN2OUT_NEXT_REASS;
1428                   goto trace01;
1429                 }
1430             }
1431
1432           key1.addr = ip1->src_address;
1433           key1.port = udp1->src_port;
1434           key1.protocol = proto1;
1435           key1.fib_index = rx_fib_index1;
1436
1437           kv1.key = key1.as_u64;
1438
1439             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1440                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1441             {
1442               if (is_slow_path)
1443                 {
1444                   if (is_output_feature)
1445                     {
1446                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1447                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1448                         goto trace01;
1449                     }
1450                   else
1451                     {
1452                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1453                           ip1, proto1, rx_fib_index1, thread_index)))
1454                         goto trace01;
1455                     }
1456
1457                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1458                                      &s1, node, next1, thread_index, now);
1459                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1460                     goto trace01;
1461                 }
1462               else
1463                 {
1464                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1465                   goto trace01;
1466                 }
1467             }
1468           else
1469             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1470                                     value1.value);
1471
1472           b1->flags |= VNET_BUFFER_F_IS_NATED;
1473
1474           old_addr1 = ip1->src_address.as_u32;
1475           ip1->src_address = s1->out2in.addr;
1476           new_addr1 = ip1->src_address.as_u32;
1477           if (!is_output_feature)
1478             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1479
1480           sum1 = ip1->checksum;
1481           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1482                                  ip4_header_t,
1483                                  src_address /* changed member */);
1484           ip1->checksum = ip_csum_fold (sum1);
1485
1486           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1487             {
1488               old_port1 = tcp1->src_port;
1489               tcp1->src_port = s1->out2in.port;
1490               new_port1 = tcp1->src_port;
1491
1492               sum1 = tcp1->checksum;
1493               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1494                                      ip4_header_t,
1495                                      dst_address /* changed member */);
1496               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1497                                      ip4_header_t /* cheat */,
1498                                      length /* changed member */);
1499               tcp1->checksum = ip_csum_fold(sum1);
1500             }
1501           else
1502             {
1503               old_port1 = udp1->src_port;
1504               udp1->src_port = s1->out2in.port;
1505               udp1->checksum = 0;
1506             }
1507
1508           /* Accounting */
1509           nat44_session_update_counters (s1, now,
1510                                          vlib_buffer_length_in_chain (vm, b1));
1511           /* Per-user LRU list maintenance */
1512           nat44_session_update_lru (sm, s1, thread_index);
1513         trace01:
1514
1515           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1516                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1517             {
1518               snat_in2out_trace_t *t =
1519                  vlib_add_trace (vm, node, b1, sizeof (*t));
1520               t->sw_if_index = sw_if_index1;
1521               t->next_index = next1;
1522               t->session_index = ~0;
1523               if (s1)
1524                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1525             }
1526
1527           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1528
1529           /* verify speculative enqueues, maybe switch current next frame */
1530           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1531                                            to_next, n_left_to_next,
1532                                            bi0, bi1, next0, next1);
1533         }
1534
1535       while (n_left_from > 0 && n_left_to_next > 0)
1536         {
1537           u32 bi0;
1538           vlib_buffer_t * b0;
1539           u32 next0;
1540           u32 sw_if_index0;
1541           ip4_header_t * ip0;
1542           ip_csum_t sum0;
1543           u32 new_addr0, old_addr0;
1544           u16 old_port0, new_port0;
1545           udp_header_t * udp0;
1546           tcp_header_t * tcp0;
1547           icmp46_header_t * icmp0;
1548           snat_session_key_t key0;
1549           u32 rx_fib_index0;
1550           u32 proto0;
1551           snat_session_t * s0 = 0;
1552           clib_bihash_kv_8_8_t kv0, value0;
1553           u32 iph_offset0 = 0;
1554
1555           /* speculatively enqueue b0 to the current next frame */
1556           bi0 = from[0];
1557           to_next[0] = bi0;
1558           from += 1;
1559           to_next += 1;
1560           n_left_from -= 1;
1561           n_left_to_next -= 1;
1562
1563           b0 = vlib_get_buffer (vm, bi0);
1564           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1565
1566           if (is_output_feature)
1567             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1568
1569           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1570                  iph_offset0);
1571
1572           udp0 = ip4_next_header (ip0);
1573           tcp0 = (tcp_header_t *) udp0;
1574           icmp0 = (icmp46_header_t *) udp0;
1575
1576           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1577           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1578                                    sw_if_index0);
1579
1580           if (PREDICT_FALSE(ip0->ttl == 1))
1581             {
1582               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1583               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1584                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1585                                            0);
1586               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1587               goto trace0;
1588             }
1589
1590           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1591
1592           /* Next configured feature, probably ip4-lookup */
1593           if (is_slow_path)
1594             {
1595               if (PREDICT_FALSE (proto0 == ~0))
1596                 {
1597                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1598                     {
1599                       next0 = SNAT_IN2OUT_NEXT_DROP;
1600                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1601                     }
1602                   goto trace0;
1603                 }
1604
1605               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1606                 {
1607                   next0 = icmp_in2out_slow_path
1608                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1609                      next0, now, thread_index, &s0);
1610                   goto trace0;
1611                 }
1612             }
1613           else
1614             {
1615               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1616                 {
1617                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1618                   goto trace0;
1619                 }
1620
1621               if (ip4_is_fragment (ip0))
1622                 {
1623                   next0 = SNAT_IN2OUT_NEXT_REASS;
1624                   goto trace0;
1625                 }
1626             }
1627
1628           key0.addr = ip0->src_address;
1629           key0.port = udp0->src_port;
1630           key0.protocol = proto0;
1631           key0.fib_index = rx_fib_index0;
1632
1633           kv0.key = key0.as_u64;
1634
1635           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1636                                       &kv0, &value0))
1637             {
1638               if (is_slow_path)
1639                 {
1640                   if (is_output_feature)
1641                     {
1642                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1643                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1644                         goto trace0;
1645                     }
1646                   else
1647                     {
1648                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1649                           ip0, proto0, rx_fib_index0, thread_index)))
1650                         goto trace0;
1651                     }
1652
1653                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1654                                      &s0, node, next0, thread_index, now);
1655
1656                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1657                     goto trace0;
1658                 }
1659               else
1660                 {
1661                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1662                   goto trace0;
1663                 }
1664             }
1665           else
1666           s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1667                                   value0.value);
1668
1669           b0->flags |= VNET_BUFFER_F_IS_NATED;
1670
1671           old_addr0 = ip0->src_address.as_u32;
1672           ip0->src_address = s0->out2in.addr;
1673           new_addr0 = ip0->src_address.as_u32;
1674           if (!is_output_feature)
1675             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1676
1677           sum0 = ip0->checksum;
1678           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1679                                  ip4_header_t,
1680                                  src_address /* changed member */);
1681           ip0->checksum = ip_csum_fold (sum0);
1682
1683           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1684             {
1685               old_port0 = tcp0->src_port;
1686               tcp0->src_port = s0->out2in.port;
1687               new_port0 = tcp0->src_port;
1688
1689               sum0 = tcp0->checksum;
1690               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1691                                      ip4_header_t,
1692                                      dst_address /* changed member */);
1693               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1694                                      ip4_header_t /* cheat */,
1695                                      length /* changed member */);
1696               tcp0->checksum = ip_csum_fold(sum0);
1697             }
1698           else
1699             {
1700               old_port0 = udp0->src_port;
1701               udp0->src_port = s0->out2in.port;
1702               udp0->checksum = 0;
1703             }
1704
1705           /* Accounting */
1706           nat44_session_update_counters (s0, now,
1707                                          vlib_buffer_length_in_chain (vm, b0));
1708           /* Per-user LRU list maintenance */
1709           nat44_session_update_lru (sm, s0, thread_index);
1710
1711         trace0:
1712           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1713                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1714             {
1715               snat_in2out_trace_t *t =
1716                  vlib_add_trace (vm, node, b0, sizeof (*t));
1717               t->is_slow_path = is_slow_path;
1718               t->sw_if_index = sw_if_index0;
1719               t->next_index = next0;
1720                   t->session_index = ~0;
1721               if (s0)
1722                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1723             }
1724
1725           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1726
1727           /* verify speculative enqueue, maybe switch current next frame */
1728           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1729                                            to_next, n_left_to_next,
1730                                            bi0, next0);
1731         }
1732
1733       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1734     }
1735
1736   vlib_node_increment_counter (vm, stats_node_index,
1737                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1738                                pkts_processed);
1739   return frame->n_vectors;
1740 }
1741
1742 static uword
1743 snat_in2out_fast_path_fn (vlib_main_t * vm,
1744                           vlib_node_runtime_t * node,
1745                           vlib_frame_t * frame)
1746 {
1747   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1748 }
1749
1750 VLIB_REGISTER_NODE (snat_in2out_node) = {
1751   .function = snat_in2out_fast_path_fn,
1752   .name = "nat44-in2out",
1753   .vector_size = sizeof (u32),
1754   .format_trace = format_snat_in2out_trace,
1755   .type = VLIB_NODE_TYPE_INTERNAL,
1756
1757   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1758   .error_strings = snat_in2out_error_strings,
1759
1760   .runtime_data_bytes = sizeof (snat_runtime_t),
1761
1762   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1763
1764   /* edit / add dispositions here */
1765   .next_nodes = {
1766     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1767     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1768     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1769     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1770     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1771   },
1772 };
1773
1774 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1775
1776 static uword
1777 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1778                                  vlib_node_runtime_t * node,
1779                                  vlib_frame_t * frame)
1780 {
1781   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1782 }
1783
1784 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1785   .function = snat_in2out_output_fast_path_fn,
1786   .name = "nat44-in2out-output",
1787   .vector_size = sizeof (u32),
1788   .format_trace = format_snat_in2out_trace,
1789   .type = VLIB_NODE_TYPE_INTERNAL,
1790
1791   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1792   .error_strings = snat_in2out_error_strings,
1793
1794   .runtime_data_bytes = sizeof (snat_runtime_t),
1795
1796   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1797
1798   /* edit / add dispositions here */
1799   .next_nodes = {
1800     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1801     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1802     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1803     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1804     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1805   },
1806 };
1807
1808 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1809                               snat_in2out_output_fast_path_fn);
1810
1811 static uword
1812 snat_in2out_slow_path_fn (vlib_main_t * vm,
1813                           vlib_node_runtime_t * node,
1814                           vlib_frame_t * frame)
1815 {
1816   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1817 }
1818
1819 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1820   .function = snat_in2out_slow_path_fn,
1821   .name = "nat44-in2out-slowpath",
1822   .vector_size = sizeof (u32),
1823   .format_trace = format_snat_in2out_trace,
1824   .type = VLIB_NODE_TYPE_INTERNAL,
1825
1826   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1827   .error_strings = snat_in2out_error_strings,
1828
1829   .runtime_data_bytes = sizeof (snat_runtime_t),
1830
1831   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1832
1833   /* edit / add dispositions here */
1834   .next_nodes = {
1835     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1836     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1837     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1838     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1839     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1840   },
1841 };
1842
1843 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1844                               snat_in2out_slow_path_fn);
1845
1846 static uword
1847 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1848                                  vlib_node_runtime_t * node,
1849                                  vlib_frame_t * frame)
1850 {
1851   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1852 }
1853
1854 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1855   .function = snat_in2out_output_slow_path_fn,
1856   .name = "nat44-in2out-output-slowpath",
1857   .vector_size = sizeof (u32),
1858   .format_trace = format_snat_in2out_trace,
1859   .type = VLIB_NODE_TYPE_INTERNAL,
1860
1861   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1862   .error_strings = snat_in2out_error_strings,
1863
1864   .runtime_data_bytes = sizeof (snat_runtime_t),
1865
1866   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1867
1868   /* edit / add dispositions here */
1869   .next_nodes = {
1870     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1871     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1872     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1873     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1874     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1875   },
1876 };
1877
1878 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1879                               snat_in2out_output_slow_path_fn);
1880
1881 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1882
1883 static inline uword
1884 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1885                              vlib_node_runtime_t * node,
1886                              vlib_frame_t * frame,
1887                              int is_ed)
1888 {
1889   u32 n_left_from, * from, * to_next, stats_node_index;
1890   snat_in2out_next_t next_index;
1891   u32 pkts_processed = 0;
1892   snat_main_t * sm = &snat_main;
1893   vnet_feature_main_t *fm = &feature_main;
1894   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1895   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1896
1897   stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1898     nat44_hairpinning_node.index;
1899   from = vlib_frame_vector_args (frame);
1900   n_left_from = frame->n_vectors;
1901   next_index = node->cached_next_index;
1902
1903   while (n_left_from > 0)
1904     {
1905       u32 n_left_to_next;
1906
1907       vlib_get_next_frame (vm, node, next_index,
1908                            to_next, n_left_to_next);
1909
1910       while (n_left_from > 0 && n_left_to_next > 0)
1911         {
1912           u32 bi0;
1913           vlib_buffer_t * b0;
1914           u32 next0;
1915           ip4_header_t * ip0;
1916           u32 proto0;
1917           udp_header_t * udp0;
1918           tcp_header_t * tcp0;
1919
1920           /* speculatively enqueue b0 to the current next frame */
1921           bi0 = from[0];
1922           to_next[0] = bi0;
1923           from += 1;
1924           to_next += 1;
1925           n_left_from -= 1;
1926           n_left_to_next -= 1;
1927
1928           b0 = vlib_get_buffer (vm, bi0);
1929           ip0 = vlib_buffer_get_current (b0);
1930           udp0 = ip4_next_header (ip0);
1931           tcp0 = (tcp_header_t *) udp0;
1932
1933           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1934
1935           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
1936                                 &next0, 0);
1937
1938           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1939             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1940
1941           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1942
1943           /* verify speculative enqueue, maybe switch current next frame */
1944           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1945                                            to_next, n_left_to_next,
1946                                            bi0, next0);
1947          }
1948
1949       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1950     }
1951
1952   vlib_node_increment_counter (vm, stats_node_index,
1953                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1954                                pkts_processed);
1955   return frame->n_vectors;
1956 }
1957
1958 static uword
1959 nat44_hairpinning_fn (vlib_main_t * vm,
1960                       vlib_node_runtime_t * node,
1961                       vlib_frame_t * frame)
1962 {
1963   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1964 }
1965
1966 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
1967   .function = nat44_hairpinning_fn,
1968   .name = "nat44-hairpinning",
1969   .vector_size = sizeof (u32),
1970   .type = VLIB_NODE_TYPE_INTERNAL,
1971   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1972   .error_strings = snat_in2out_error_strings,
1973   .n_next_nodes = 2,
1974   .next_nodes = {
1975     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1976     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1977   },
1978 };
1979
1980 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
1981                               nat44_hairpinning_fn);
1982
1983 static uword
1984 nat44_ed_hairpinning_fn (vlib_main_t * vm,
1985                          vlib_node_runtime_t * node,
1986                          vlib_frame_t * frame)
1987 {
1988   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1989 }
1990
1991 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
1992   .function = nat44_ed_hairpinning_fn,
1993   .name = "nat44-ed-hairpinning",
1994   .vector_size = sizeof (u32),
1995   .type = VLIB_NODE_TYPE_INTERNAL,
1996   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1997   .error_strings = snat_in2out_error_strings,
1998   .n_next_nodes = 2,
1999   .next_nodes = {
2000     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2001     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2002   },
2003 };
2004
2005 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
2006                               nat44_ed_hairpinning_fn);
2007
2008 static inline void
2009 nat44_reass_hairpinning (snat_main_t *sm,
2010                          vlib_buffer_t * b0,
2011                          ip4_header_t * ip0,
2012                          u16 sport,
2013                          u16 dport,
2014                          u32 proto0)
2015 {
2016   snat_session_key_t key0, sm0;
2017   snat_session_t * s0;
2018   clib_bihash_kv_8_8_t kv0, value0;
2019   ip_csum_t sum0;
2020   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2021   u16 new_dst_port0, old_dst_port0;
2022   udp_header_t * udp0;
2023   tcp_header_t * tcp0;
2024
2025   key0.addr = ip0->dst_address;
2026   key0.port = dport;
2027   key0.protocol = proto0;
2028   key0.fib_index = sm->outside_fib_index;
2029   kv0.key = key0.as_u64;
2030
2031   udp0 = ip4_next_header (ip0);
2032
2033   /* Check if destination is static mappings */
2034   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2035     {
2036       new_dst_addr0 = sm0.addr.as_u32;
2037       new_dst_port0 = sm0.port;
2038       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2039     }
2040   /* or active sessions */
2041   else
2042     {
2043       if (sm->num_workers > 1)
2044         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2045       else
2046         ti = sm->num_workers;
2047
2048       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2049         {
2050           si = value0.value;
2051           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2052           new_dst_addr0 = s0->in2out.addr.as_u32;
2053           new_dst_port0 = s0->in2out.port;
2054           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2055         }
2056     }
2057
2058   /* Destination is behind the same NAT, use internal address and port */
2059   if (new_dst_addr0)
2060     {
2061       old_dst_addr0 = ip0->dst_address.as_u32;
2062       ip0->dst_address.as_u32 = new_dst_addr0;
2063       sum0 = ip0->checksum;
2064       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2065                              ip4_header_t, dst_address);
2066       ip0->checksum = ip_csum_fold (sum0);
2067
2068       old_dst_port0 = dport;
2069       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2070                        ip4_is_first_fragment (ip0)))
2071         {
2072           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2073             {
2074               tcp0 = ip4_next_header (ip0);
2075               tcp0->dst = new_dst_port0;
2076               sum0 = tcp0->checksum;
2077               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2078                                      ip4_header_t, dst_address);
2079               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2080                                      ip4_header_t /* cheat */, length);
2081               tcp0->checksum = ip_csum_fold(sum0);
2082             }
2083           else
2084             {
2085               udp0->dst_port = new_dst_port0;
2086               udp0->checksum = 0;
2087             }
2088         }
2089       else
2090         {
2091           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2092             {
2093               tcp0 = ip4_next_header (ip0);
2094               sum0 = tcp0->checksum;
2095               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2096                                      ip4_header_t, dst_address);
2097               tcp0->checksum = ip_csum_fold(sum0);
2098             }
2099         }
2100     }
2101 }
2102
2103 static uword
2104 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2105                             vlib_node_runtime_t * node,
2106                             vlib_frame_t * frame)
2107 {
2108   u32 n_left_from, *from, *to_next;
2109   snat_in2out_next_t next_index;
2110   u32 pkts_processed = 0;
2111   snat_main_t *sm = &snat_main;
2112   f64 now = vlib_time_now (vm);
2113   u32 thread_index = vm->thread_index;
2114   snat_main_per_thread_data_t *per_thread_data =
2115     &sm->per_thread_data[thread_index];
2116   u32 *fragments_to_drop = 0;
2117   u32 *fragments_to_loopback = 0;
2118
2119   from = vlib_frame_vector_args (frame);
2120   n_left_from = frame->n_vectors;
2121   next_index = node->cached_next_index;
2122
2123   while (n_left_from > 0)
2124     {
2125       u32 n_left_to_next;
2126
2127       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2128
2129       while (n_left_from > 0 && n_left_to_next > 0)
2130        {
2131           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2132           vlib_buffer_t *b0;
2133           u32 next0;
2134           u8 cached0 = 0;
2135           ip4_header_t *ip0;
2136           nat_reass_ip4_t *reass0;
2137           udp_header_t * udp0;
2138           tcp_header_t * tcp0;
2139           snat_session_key_t key0;
2140           clib_bihash_kv_8_8_t kv0, value0;
2141           snat_session_t * s0 = 0;
2142           u16 old_port0, new_port0;
2143           ip_csum_t sum0;
2144
2145           /* speculatively enqueue b0 to the current next frame */
2146           bi0 = from[0];
2147           to_next[0] = bi0;
2148           from += 1;
2149           to_next += 1;
2150           n_left_from -= 1;
2151           n_left_to_next -= 1;
2152
2153           b0 = vlib_get_buffer (vm, bi0);
2154           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2155
2156           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2157           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2158                                                                sw_if_index0);
2159
2160           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2161             {
2162               next0 = SNAT_IN2OUT_NEXT_DROP;
2163               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2164               goto trace0;
2165             }
2166
2167           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2168           udp0 = ip4_next_header (ip0);
2169           tcp0 = (tcp_header_t *) udp0;
2170           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2171
2172           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2173                                                  ip0->dst_address,
2174                                                  ip0->fragment_id,
2175                                                  ip0->protocol,
2176                                                  1,
2177                                                  &fragments_to_drop);
2178
2179           if (PREDICT_FALSE (!reass0))
2180             {
2181               next0 = SNAT_IN2OUT_NEXT_DROP;
2182               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2183               nat_log_notice ("maximum reassemblies exceeded");
2184               goto trace0;
2185             }
2186
2187           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2188             {
2189               key0.addr = ip0->src_address;
2190               key0.port = udp0->src_port;
2191               key0.protocol = proto0;
2192               key0.fib_index = rx_fib_index0;
2193               kv0.key = key0.as_u64;
2194
2195               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2196                 {
2197                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2198                       ip0, proto0, rx_fib_index0, thread_index)))
2199                     goto trace0;
2200
2201                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2202                                      &s0, node, next0, thread_index, now);
2203
2204                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2205                     goto trace0;
2206
2207                   reass0->sess_index = s0 - per_thread_data->sessions;
2208                 }
2209               else
2210                 {
2211                   s0 = pool_elt_at_index (per_thread_data->sessions,
2212                                           value0.value);
2213                   reass0->sess_index = value0.value;
2214                 }
2215               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2216             }
2217           else
2218             {
2219               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2220                 {
2221                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2222                     {
2223                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2224                       nat_log_notice ("maximum fragments per reassembly exceeded");
2225                       next0 = SNAT_IN2OUT_NEXT_DROP;
2226                       goto trace0;
2227                     }
2228                   cached0 = 1;
2229                   goto trace0;
2230                 }
2231               s0 = pool_elt_at_index (per_thread_data->sessions,
2232                                       reass0->sess_index);
2233             }
2234
2235           old_addr0 = ip0->src_address.as_u32;
2236           ip0->src_address = s0->out2in.addr;
2237           new_addr0 = ip0->src_address.as_u32;
2238           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2239
2240           sum0 = ip0->checksum;
2241           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2242                                  ip4_header_t,
2243                                  src_address /* changed member */);
2244           ip0->checksum = ip_csum_fold (sum0);
2245
2246           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2247             {
2248               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2249                 {
2250                   old_port0 = tcp0->src_port;
2251                   tcp0->src_port = s0->out2in.port;
2252                   new_port0 = tcp0->src_port;
2253
2254                   sum0 = tcp0->checksum;
2255                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2256                                          ip4_header_t,
2257                                          dst_address /* changed member */);
2258                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2259                                          ip4_header_t /* cheat */,
2260                                          length /* changed member */);
2261                   tcp0->checksum = ip_csum_fold(sum0);
2262                 }
2263               else
2264                 {
2265                   old_port0 = udp0->src_port;
2266                   udp0->src_port = s0->out2in.port;
2267                   udp0->checksum = 0;
2268                 }
2269             }
2270
2271           /* Hairpinning */
2272           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2273                                    s0->ext_host_port, proto0);
2274
2275           /* Accounting */
2276           nat44_session_update_counters (s0, now,
2277                                          vlib_buffer_length_in_chain (vm, b0));
2278           /* Per-user LRU list maintenance */
2279           nat44_session_update_lru (sm, s0, thread_index);
2280
2281         trace0:
2282           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2283                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2284             {
2285               nat44_in2out_reass_trace_t *t =
2286                  vlib_add_trace (vm, node, b0, sizeof (*t));
2287               t->cached = cached0;
2288               t->sw_if_index = sw_if_index0;
2289               t->next_index = next0;
2290             }
2291
2292           if (cached0)
2293             {
2294               n_left_to_next++;
2295               to_next--;
2296             }
2297           else
2298             {
2299               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2300
2301               /* verify speculative enqueue, maybe switch current next frame */
2302               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2303                                                to_next, n_left_to_next,
2304                                                bi0, next0);
2305             }
2306
2307           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2308             {
2309               from = vlib_frame_vector_args (frame);
2310               u32 len = vec_len (fragments_to_loopback);
2311               if (len <= VLIB_FRAME_SIZE)
2312                 {
2313                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2314                   n_left_from = len;
2315                   vec_reset_length (fragments_to_loopback);
2316                 }
2317               else
2318                 {
2319                   clib_memcpy (from,
2320                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2321                                sizeof (u32) * VLIB_FRAME_SIZE);
2322                   n_left_from = VLIB_FRAME_SIZE;
2323                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2324                 }
2325             }
2326        }
2327
2328       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2329     }
2330
2331   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2332                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2333                                pkts_processed);
2334
2335   nat_send_all_to_node (vm, fragments_to_drop, node,
2336                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2337                         SNAT_IN2OUT_NEXT_DROP);
2338
2339   vec_free (fragments_to_drop);
2340   vec_free (fragments_to_loopback);
2341   return frame->n_vectors;
2342 }
2343
2344 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2345   .function = nat44_in2out_reass_node_fn,
2346   .name = "nat44-in2out-reass",
2347   .vector_size = sizeof (u32),
2348   .format_trace = format_nat44_in2out_reass_trace,
2349   .type = VLIB_NODE_TYPE_INTERNAL,
2350
2351   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2352   .error_strings = snat_in2out_error_strings,
2353
2354   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2355   .next_nodes = {
2356     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2357     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2358     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2359     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2360     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2361   },
2362 };
2363
2364 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2365                               nat44_in2out_reass_node_fn);
2366
2367 /*******************************/
2368 /*** endpoint-dependent mode ***/
2369 /*******************************/
2370
2371 static_always_inline int
2372 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2373 {
2374   icmp46_header_t *icmp0;
2375   nat_ed_ses_key_t key0;
2376   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2377   ip4_header_t *inner_ip0 = 0;
2378   void *l4_header = 0;
2379   icmp46_header_t *inner_icmp0;
2380
2381   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2382   echo0 = (icmp_echo_header_t *)(icmp0+1);
2383
2384   if (!icmp_is_error_message (icmp0))
2385     {
2386       key0.proto = IP_PROTOCOL_ICMP;
2387       key0.l_addr = ip0->src_address;
2388       key0.r_addr = ip0->dst_address;
2389       key0.l_port = echo0->identifier;
2390       key0.r_port = 0;
2391     }
2392   else
2393     {
2394       inner_ip0 = (ip4_header_t *)(echo0+1);
2395       l4_header = ip4_next_header (inner_ip0);
2396       key0.proto = inner_ip0->protocol;
2397       key0.r_addr = inner_ip0->src_address;
2398       key0.l_addr = inner_ip0->dst_address;
2399       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2400         {
2401         case SNAT_PROTOCOL_ICMP:
2402           inner_icmp0 = (icmp46_header_t*)l4_header;
2403           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2404           key0.r_port = 0;
2405           key0.l_port = inner_echo0->identifier;
2406           break;
2407         case SNAT_PROTOCOL_UDP:
2408         case SNAT_PROTOCOL_TCP:
2409           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2410           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2411           break;
2412         default:
2413           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2414         }
2415     }
2416   *p_key0 = key0;
2417   return 0;
2418 }
2419
2420 int
2421 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void * arg)
2422 {
2423   snat_main_t *sm = &snat_main;
2424   nat44_is_idle_session_ctx_t *ctx = arg;
2425   snat_session_t *s;
2426   u64 sess_timeout_time;
2427   nat_ed_ses_key_t ed_key;
2428   clib_bihash_kv_16_8_t ed_kv;
2429   int i;
2430   snat_address_t *a;
2431   snat_session_key_t key;
2432   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
2433                                                        ctx->thread_index);
2434
2435   s = pool_elt_at_index (tsm->sessions, kv->value);
2436   sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
2437   if (ctx->now >= sess_timeout_time)
2438     {
2439       if (is_fwd_bypass_session (s))
2440         goto delete;
2441
2442       ed_key.l_addr = s->out2in.addr;
2443       ed_key.r_addr = s->ext_host_addr;
2444       ed_key.fib_index = s->out2in.fib_index;
2445       if (snat_is_unk_proto_session (s))
2446         {
2447           ed_key.proto = s->in2out.port;
2448           ed_key.r_port = 0;
2449           ed_key.l_port = 0;
2450         }
2451       else
2452         {
2453           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
2454           ed_key.l_port = s->out2in.port;
2455           ed_key.r_port = s->ext_host_port;
2456         }
2457       ed_kv.key[0] = ed_key.as_u64[0];
2458       ed_kv.key[1] = ed_key.as_u64[1];
2459       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
2460         nat_log_warn ("out2in_ed key del failed");
2461
2462       if (snat_is_unk_proto_session (s))
2463         goto delete;
2464
2465       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
2466                                           s->out2in.addr.as_u32,
2467                                           s->in2out.protocol,
2468                                           s->in2out.port,
2469                                           s->out2in.port,
2470                                           s->in2out.fib_index);
2471
2472       if (is_twice_nat_session (s))
2473         {
2474           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
2475             {
2476               key.protocol = s->in2out.protocol;
2477               key.port = s->ext_host_nat_port;
2478               a = sm->twice_nat_addresses + i;
2479               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
2480                 {
2481                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
2482                                                       ctx->thread_index, &key);
2483                   break;
2484                 }
2485             }
2486         }
2487
2488       if (snat_is_session_static (s))
2489         goto delete;
2490
2491       if (s->outside_address_index != ~0)
2492         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
2493                                             &s->out2in);
2494     delete:
2495       nat44_delete_session (sm, s, ctx->thread_index);
2496       return 1;
2497     }
2498
2499   return 0;
2500 }
2501
2502 static inline u32
2503 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
2504                           ip4_header_t * ip0, icmp46_header_t * icmp0,
2505                           u32 sw_if_index0, u32 rx_fib_index0,
2506                           vlib_node_runtime_t * node, u32 next0, f64 now,
2507                           u32 thread_index, snat_session_t ** p_s0)
2508 {
2509   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2510                       next0, thread_index, p_s0, 0);
2511   snat_session_t * s0 = *p_s0;
2512   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
2513     {
2514       /* Hairpinning */
2515       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
2516         snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
2517       /* Accounting */
2518       nat44_session_update_counters (s0, now,
2519                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
2520     }
2521   return next0;
2522 }
2523
2524 static u32
2525 slow_path_ed (snat_main_t *sm,
2526               vlib_buffer_t *b,
2527               u32 rx_fib_index,
2528               clib_bihash_kv_16_8_t *kv,
2529               snat_session_t ** sessionp,
2530               vlib_node_runtime_t * node,
2531               u32 next,
2532               u32 thread_index,
2533               f64 now)
2534 {
2535   snat_session_t *s;
2536   snat_user_t *u;
2537   snat_session_key_t key0, key1;
2538   u8 lb = 0, is_sm = 0;
2539   u32 address_index = ~0;
2540   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2541   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2542   u32 proto = ip_proto_to_snat_proto (key->proto);
2543   nat_outside_fib_t *outside_fib;
2544   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2545   fib_prefix_t pfx = {
2546     .fp_proto = FIB_PROTOCOL_IP4,
2547     .fp_len = 32,
2548     .fp_addr = {
2549         .ip4.as_u32 = key->r_addr.as_u32,
2550     },
2551   };
2552   nat44_is_idle_session_ctx_t ctx;
2553
2554   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2555     {
2556       b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2557       nat_ipfix_logging_max_sessions(sm->max_translations);
2558       nat_log_notice ("maximum sessions exceeded");
2559       return SNAT_IN2OUT_NEXT_DROP;
2560     }
2561
2562   key0.addr = key->l_addr;
2563   key0.port = key->l_port;
2564   key1.protocol = key0.protocol = proto;
2565   key0.fib_index = rx_fib_index;
2566   key1.fib_index = sm->outside_fib_index;
2567   /* First try to match static mapping by local address and port */
2568   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb))
2569     {
2570       /* Try to create dynamic translation */
2571       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2572                                                thread_index, &key1,
2573                                                &address_index,
2574                                                sm->port_per_thread,
2575                                                tsm->snat_thread_index))
2576         {
2577           nat_log_notice ("addresses exhausted");
2578           b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2579           return SNAT_IN2OUT_NEXT_DROP;
2580         }
2581     }
2582   else
2583     is_sm = 1;
2584
2585   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2586   if (!u)
2587     {
2588       nat_log_warn ("create NAT user failed");
2589       if (!is_sm)
2590         snat_free_outside_address_and_port (sm->addresses,
2591                                             thread_index, &key1);
2592       return SNAT_IN2OUT_NEXT_DROP;
2593     }
2594
2595   s = nat_ed_session_alloc (sm, u, thread_index);
2596   if (!s)
2597     {
2598       nat44_delete_user_with_no_session (sm, u, thread_index);
2599       nat_log_warn ("create NAT session failed");
2600       if (!is_sm)
2601         snat_free_outside_address_and_port (sm->addresses,
2602                                             thread_index, &key1);
2603       return SNAT_IN2OUT_NEXT_DROP;
2604     }
2605
2606   user_session_increment (sm, u, is_sm);
2607   if (is_sm)
2608     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2609   if (lb)
2610     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2611   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2612   s->outside_address_index = address_index;
2613   s->ext_host_addr = key->r_addr;
2614   s->ext_host_port = key->r_port;
2615   s->in2out = key0;
2616   s->out2in = key1;
2617   s->out2in.protocol = key0.protocol;
2618
2619   switch (vec_len (sm->outside_fibs))
2620     {
2621     case 0:
2622       s->out2in.fib_index = sm->outside_fib_index;
2623       break;
2624     case 1:
2625       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2626       break;
2627     default:
2628       vec_foreach (outside_fib, sm->outside_fibs)
2629         {
2630           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2631           if (FIB_NODE_INDEX_INVALID != fei)
2632             {
2633               if (fib_entry_get_resolving_interface (fei) != ~0)
2634                 {
2635                   s->out2in.fib_index = outside_fib->fib_index;
2636                   break;
2637                 }
2638             }
2639         }
2640       break;
2641     }
2642
2643   /* Add to lookup tables */
2644   kv->value = s - tsm->sessions;
2645   ctx.now = now;
2646   ctx.thread_index = thread_index;
2647   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
2648                                                nat44_i2o_ed_is_idle_session_cb,
2649                                                &ctx))
2650     nat_log_notice ("in2out-ed key add failed");
2651
2652   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2653               key1.port, key->r_port);
2654   kv->value = s - tsm->sessions;
2655   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
2656                                                nat44_o2i_ed_is_idle_session_cb,
2657                                                &ctx))
2658     nat_log_notice ("out2in-ed key add failed");
2659
2660   *sessionp = s;
2661
2662   /* log NAT event */
2663   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2664                                       s->out2in.addr.as_u32,
2665                                       s->in2out.protocol,
2666                                       s->in2out.port,
2667                                       s->out2in.port,
2668                                       s->in2out.fib_index);
2669   return next;
2670 }
2671
2672 static_always_inline int
2673 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2674                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
2675                         u32 rx_fib_index, u32 thread_index)
2676 {
2677   udp_header_t *udp = ip4_next_header (ip);
2678   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2679   clib_bihash_kv_16_8_t kv, value;
2680   snat_session_key_t key0, key1;
2681
2682   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2683               sm->outside_fib_index, udp->dst_port, udp->src_port);
2684
2685   /* NAT packet aimed at external address if */
2686   /* has active sessions */
2687   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2688     {
2689       key0.addr = ip->dst_address;
2690       key0.port = udp->dst_port;
2691       key0.protocol = proto;
2692       key0.fib_index = sm->outside_fib_index;
2693       /* or is static mappings */
2694       if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0))
2695         return 0;
2696     }
2697   else
2698     return 0;
2699
2700   if (sm->forwarding_enabled)
2701     return 1;
2702
2703   return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2704 }
2705
2706 static_always_inline int
2707 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2708                                       u32 thread_index, f64 now,
2709                                       vlib_main_t * vm, vlib_buffer_t * b)
2710 {
2711   nat_ed_ses_key_t key;
2712   clib_bihash_kv_16_8_t kv, value;
2713   udp_header_t *udp;
2714   snat_session_t *s = 0;
2715   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2716
2717   if (!sm->forwarding_enabled)
2718     return 0;
2719
2720   if (ip->protocol == IP_PROTOCOL_ICMP)
2721     {
2722       key.as_u64[0] = key.as_u64[1] = 0;
2723       if (icmp_get_ed_key (ip, &key))
2724         return 0;
2725       key.fib_index = 0;
2726       kv.key[0] = key.as_u64[0];
2727       kv.key[1] = key.as_u64[1];
2728     }
2729   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2730     {
2731       udp = ip4_next_header(ip);
2732       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2733                   udp->src_port, udp->dst_port);
2734     }
2735   else
2736     {
2737       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2738                   0);
2739     }
2740
2741   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2742     {
2743       s = pool_elt_at_index (tsm->sessions, value.value);
2744       if (is_fwd_bypass_session (s))
2745         {
2746           if (ip->protocol == IP_PROTOCOL_TCP)
2747             {
2748               tcp_header_t *tcp = ip4_next_header(ip);
2749               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2750                 return 1;
2751             }
2752           /* Accounting */
2753           nat44_session_update_counters (s, now,
2754                                          vlib_buffer_length_in_chain (vm, b));
2755           return 1;
2756         }
2757       else
2758         return 0;
2759     }
2760
2761   return 0;
2762 }
2763
2764 static_always_inline int
2765 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2766                                        u8 proto, u16 src_port, u16 dst_port,
2767                                        u32 thread_index, u32 sw_if_index)
2768 {
2769   clib_bihash_kv_16_8_t kv, value;
2770   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2771   snat_interface_t *i;
2772   snat_session_t *s;
2773   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2774
2775   /* src NAT check */
2776   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, fib_index,
2777               src_port, dst_port);
2778   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2779     return 1;
2780
2781   /* dst NAT check */
2782   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, fib_index,
2783               dst_port, src_port);
2784   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2785   {
2786     s = pool_elt_at_index (tsm->sessions, value.value);
2787     if (is_fwd_bypass_session (s))
2788       return 0;
2789
2790     /* hairpinning */
2791     pool_foreach (i, sm->output_feature_interfaces,
2792     ({
2793       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
2794         return 0;
2795     }));
2796     return 1;
2797   }
2798
2799   return 0;
2800 }
2801
2802 u32
2803 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2804                      u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2805                      u8 *p_proto, snat_session_key_t *p_value,
2806                      u8 *p_dont_translate, void *d, void *e)
2807 {
2808   icmp46_header_t *icmp;
2809   u32 sw_if_index;
2810   u32 rx_fib_index;
2811   nat_ed_ses_key_t key;
2812   snat_session_t *s = 0;
2813   u8 dont_translate = 0;
2814   clib_bihash_kv_16_8_t kv, value;
2815   u32 next = ~0;
2816   int err;
2817   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2818
2819   icmp = (icmp46_header_t *) ip4_next_header (ip);
2820   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2821   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2822
2823   key.as_u64[0] = key.as_u64[1] = 0;
2824   err = icmp_get_ed_key (ip, &key);
2825   if (err != 0)
2826     {
2827       b->error = node->errors[err];
2828       next = SNAT_IN2OUT_NEXT_DROP;
2829       goto out;
2830     }
2831   key.fib_index = rx_fib_index;
2832
2833   kv.key[0] = key.as_u64[0];
2834   kv.key[1] = key.as_u64[1];
2835
2836   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2837     {
2838       if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2839         {
2840           if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2841               key.proto, key.l_port, key.r_port, thread_index, sw_if_index)))
2842             {
2843               dont_translate = 1;
2844               goto out;
2845             }
2846         }
2847       else
2848         {
2849           if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2850               ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2851             {
2852               dont_translate = 1;
2853               goto out;
2854             }
2855         }
2856
2857       if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2858         {
2859           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2860           next = SNAT_IN2OUT_NEXT_DROP;
2861           goto out;
2862         }
2863
2864       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2865                            thread_index, vlib_time_now (sm->vlib_main));
2866
2867       if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2868         goto out;
2869     }
2870   else
2871     {
2872       if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2873                         icmp->type != ICMP4_echo_reply &&
2874                         !icmp_is_error_message (icmp)))
2875         {
2876           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2877           next = SNAT_IN2OUT_NEXT_DROP;
2878           goto out;
2879         }
2880
2881       s = pool_elt_at_index (tsm->sessions, value.value);
2882     }
2883
2884   *p_proto = ip_proto_to_snat_proto (key.proto);
2885 out:
2886   if (s)
2887     *p_value = s->out2in;
2888   *p_dont_translate = dont_translate;
2889   if (d)
2890     *(snat_session_t**)d = s;
2891   return next;
2892 }
2893
2894 static inline void
2895 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
2896                                     vlib_buffer_t * b,
2897                                     ip4_header_t * ip)
2898 {
2899   u32 old_addr, new_addr = 0, ti = 0;
2900   clib_bihash_kv_8_8_t kv, value;
2901   clib_bihash_kv_16_8_t s_kv, s_value;
2902   snat_static_mapping_t *m;
2903   ip_csum_t sum;
2904   snat_session_t *s;
2905   snat_main_per_thread_data_t *tsm;
2906
2907   if (sm->num_workers > 1)
2908     ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2909   else
2910     ti = sm->num_workers;
2911   tsm = &sm->per_thread_data[ti];
2912
2913   old_addr = ip->dst_address.as_u32;
2914   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2915               sm->outside_fib_index, 0, 0);
2916   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2917     {
2918       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2919       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2920         return;
2921
2922       m = pool_elt_at_index (sm->static_mappings, value.value);
2923       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2924         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2925       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2926     }
2927   else
2928     {
2929       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2930       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2931         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2932       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2933     }
2934   sum = ip->checksum;
2935   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2936   ip->checksum = ip_csum_fold (sum);
2937 }
2938
2939 static snat_session_t *
2940 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
2941                                vlib_buffer_t * b,
2942                                ip4_header_t * ip,
2943                                u32 rx_fib_index,
2944                                u32 thread_index,
2945                                f64 now,
2946                                vlib_main_t * vm,
2947                                vlib_node_runtime_t * node)
2948 {
2949   clib_bihash_kv_8_8_t kv, value;
2950   clib_bihash_kv_16_8_t s_kv, s_value;
2951   snat_static_mapping_t *m;
2952   u32 old_addr, new_addr = 0;
2953   ip_csum_t sum;
2954   snat_user_t *u;
2955   dlist_elt_t *head, *elt;
2956   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2957   u32 elt_index, head_index, ses_index;
2958   snat_session_t * s;
2959   u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2960   int i;
2961   u8 is_sm = 0;
2962   nat_outside_fib_t *outside_fib;
2963   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2964   fib_prefix_t pfx = {
2965     .fp_proto = FIB_PROTOCOL_IP4,
2966     .fp_len = 32,
2967     .fp_addr = {
2968         .ip4.as_u32 = ip->dst_address.as_u32,
2969     },
2970   };
2971
2972   switch (vec_len (sm->outside_fibs))
2973     {
2974     case 0:
2975       outside_fib_index = sm->outside_fib_index;
2976       break;
2977     case 1:
2978       outside_fib_index = sm->outside_fibs[0].fib_index;
2979       break;
2980     default:
2981       vec_foreach (outside_fib, sm->outside_fibs)
2982         {
2983           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2984           if (FIB_NODE_INDEX_INVALID != fei)
2985             {
2986               if (fib_entry_get_resolving_interface (fei) != ~0)
2987                 {
2988                   outside_fib_index = outside_fib->fib_index;
2989                   break;
2990                 }
2991             }
2992         }
2993       break;
2994     }
2995   old_addr = ip->src_address.as_u32;
2996
2997   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
2998               rx_fib_index, 0, 0);
2999
3000   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
3001     {
3002       s = pool_elt_at_index (tsm->sessions, s_value.value);
3003       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
3004     }
3005   else
3006     {
3007       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
3008         {
3009           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
3010           nat_ipfix_logging_max_sessions(sm->max_translations);
3011           nat_log_notice ("maximum sessions exceeded");
3012           return 0;
3013         }
3014
3015       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
3016                                   thread_index);
3017       if (!u)
3018         {
3019           nat_log_warn ("create NAT user failed");
3020           return 0;
3021         }
3022
3023       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
3024
3025       /* Try to find static mapping first */
3026       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
3027         {
3028           m = pool_elt_at_index (sm->static_mappings, value.value);
3029           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
3030           is_sm = 1;
3031           goto create_ses;
3032         }
3033       /* Fallback to 3-tuple key */
3034       else
3035         {
3036           /* Choose same out address as for TCP/UDP session to same destination */
3037           head_index = u->sessions_per_user_list_head_index;
3038           head = pool_elt_at_index (tsm->list_pool, head_index);
3039           elt_index = head->next;
3040           if (PREDICT_FALSE (elt_index == ~0))
3041             ses_index = ~0;
3042           else
3043             {
3044               elt = pool_elt_at_index (tsm->list_pool, elt_index);
3045               ses_index = elt->value;
3046             }
3047
3048           while (ses_index != ~0)
3049             {
3050               s =  pool_elt_at_index (tsm->sessions, ses_index);
3051               elt_index = elt->next;
3052               elt = pool_elt_at_index (tsm->list_pool, elt_index);
3053               ses_index = elt->value;
3054
3055               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
3056                 {
3057                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
3058                   address_index = s->outside_address_index;
3059
3060                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
3061                               ip->protocol, outside_fib_index, 0, 0);
3062                   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3063                     goto create_ses;
3064
3065                   break;
3066                 }
3067             }
3068
3069           for (i = 0; i < vec_len (sm->addresses); i++)
3070             {
3071               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
3072                           ip->protocol, outside_fib_index, 0, 0);
3073               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3074                 {
3075                   new_addr = ip->src_address.as_u32 =
3076                     sm->addresses[i].addr.as_u32;
3077                   address_index = i;
3078                   goto create_ses;
3079                 }
3080             }
3081           return 0;
3082         }
3083
3084 create_ses:
3085       s = nat_ed_session_alloc (sm, u, thread_index);
3086       if (!s)
3087         {
3088           nat44_delete_user_with_no_session (sm, u, thread_index);
3089           nat_log_warn ("create NAT session failed");
3090           return 0;
3091         }
3092
3093       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
3094       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
3095       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
3096       s->outside_address_index = address_index;
3097       s->out2in.addr.as_u32 = new_addr;
3098       s->out2in.fib_index = outside_fib_index;
3099       s->in2out.addr.as_u32 = old_addr;
3100       s->in2out.fib_index = rx_fib_index;
3101       s->in2out.port = s->out2in.port = ip->protocol;
3102       if (is_sm)
3103         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
3104       user_session_increment (sm, u, is_sm);
3105
3106       /* Add to lookup tables */
3107       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
3108                   rx_fib_index, 0, 0);
3109       s_kv.value = s - tsm->sessions;
3110       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
3111         nat_log_notice ("in2out key add failed");
3112
3113       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
3114                   outside_fib_index, 0, 0);
3115       s_kv.value = s - tsm->sessions;
3116       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
3117         nat_log_notice ("out2in key add failed");
3118   }
3119
3120   /* Update IP checksum */
3121   sum = ip->checksum;
3122   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
3123   ip->checksum = ip_csum_fold (sum);
3124
3125   /* Accounting */
3126   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
3127
3128   /* Hairpinning */
3129   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3130     nat44_ed_hairpinning_unknown_proto(sm, b, ip);
3131
3132   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3133     vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
3134
3135   return s;
3136 }
3137
3138 static inline uword
3139 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
3140                                 vlib_node_runtime_t * node,
3141                                 vlib_frame_t * frame, int is_slow_path,
3142                                 int is_output_feature)
3143 {
3144   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
3145   snat_in2out_next_t next_index;
3146   snat_main_t *sm = &snat_main;
3147   f64 now = vlib_time_now (vm);
3148   u32 thread_index = vm->thread_index;
3149   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3150
3151   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
3152     nat44_ed_in2out_node.index;
3153
3154   from = vlib_frame_vector_args (frame);
3155   n_left_from = frame->n_vectors;
3156   next_index = node->cached_next_index;
3157
3158   while (n_left_from > 0)
3159     {
3160       u32 n_left_to_next;
3161
3162       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3163
3164       while (n_left_from >= 4 && n_left_to_next >= 2)
3165         {
3166           u32 bi0, bi1;
3167           vlib_buffer_t *b0, *b1;
3168           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3169               new_addr0, old_addr0;
3170           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3171               new_addr1, old_addr1;
3172           u16 old_port0, new_port0, old_port1, new_port1;
3173           ip4_header_t *ip0, *ip1;
3174           udp_header_t *udp0, *udp1;
3175           tcp_header_t *tcp0, *tcp1;
3176           icmp46_header_t *icmp0, *icmp1;
3177           snat_session_t *s0 = 0, *s1 = 0;
3178           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3179           ip_csum_t sum0, sum1;
3180
3181           /* Prefetch next iteration. */
3182           {
3183             vlib_buffer_t * p2, * p3;
3184
3185             p2 = vlib_get_buffer (vm, from[2]);
3186             p3 = vlib_get_buffer (vm, from[3]);
3187
3188             vlib_prefetch_buffer_header (p2, LOAD);
3189             vlib_prefetch_buffer_header (p3, LOAD);
3190
3191             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3192             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3193           }
3194
3195           /* speculatively enqueue b0 and b1 to the current next frame */
3196           to_next[0] = bi0 = from[0];
3197           to_next[1] = bi1 = from[1];
3198           from += 2;
3199           to_next += 2;
3200           n_left_from -= 2;
3201           n_left_to_next -= 2;
3202
3203           b0 = vlib_get_buffer (vm, bi0);
3204           b1 = vlib_get_buffer (vm, bi1);
3205
3206           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3207
3208           if (is_output_feature)
3209             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3210
3211           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3212                  iph_offset0);
3213
3214           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3215           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3216                                                                sw_if_index0);
3217
3218           if (PREDICT_FALSE(ip0->ttl == 1))
3219             {
3220               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3221               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3222                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3223                                            0);
3224               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3225               goto trace00;
3226             }
3227
3228           udp0 = ip4_next_header (ip0);
3229           tcp0 = (tcp_header_t *) udp0;
3230           icmp0 = (icmp46_header_t *) udp0;
3231           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3232
3233           if (is_slow_path)
3234             {
3235               if (PREDICT_FALSE (proto0 == ~0))
3236                 {
3237                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3238                                                       rx_fib_index0,
3239                                                       thread_index, now, vm,
3240                                                       node);
3241                   if (!s0)
3242                     next0 = SNAT_IN2OUT_NEXT_DROP;
3243                   goto trace00;
3244                 }
3245
3246               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3247                 {
3248                   next0 = icmp_in2out_ed_slow_path
3249                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3250                      next0, now, thread_index, &s0);
3251                   goto trace00;
3252                 }
3253             }
3254           else
3255             {
3256                if (is_output_feature)
3257                 {
3258                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3259                       sm, ip0, thread_index, now, vm, b0)))
3260                     goto trace00;
3261                 }
3262
3263               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3264                 {
3265                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3266                   goto trace00;
3267                 }
3268
3269               if (ip4_is_fragment (ip0))
3270                 {
3271                   b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3272                   next0 = SNAT_IN2OUT_NEXT_DROP;
3273                   goto trace00;
3274                 }
3275             }
3276
3277           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3278                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3279
3280           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3281             {
3282               if (is_slow_path)
3283                 {
3284                   if (is_output_feature)
3285                     {
3286                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3287                           sm, ip0, ip0->protocol, udp0->src_port,
3288                           udp0->dst_port, thread_index, sw_if_index0)))
3289                         goto trace00;
3290                     }
3291                   else
3292                     {
3293                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3294                           sw_if_index0, ip0, proto0, rx_fib_index0,
3295                           thread_index)))
3296                         goto trace00;
3297                     }
3298
3299                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3300                                         next0, thread_index, now);
3301
3302                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3303                     goto trace00;
3304                 }
3305               else
3306                 {
3307                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3308                   goto trace00;
3309                 }
3310             }
3311           else
3312             {
3313               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3314             }
3315
3316           b0->flags |= VNET_BUFFER_F_IS_NATED;
3317
3318           if (!is_output_feature)
3319             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3320
3321           old_addr0 = ip0->src_address.as_u32;
3322           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3323           sum0 = ip0->checksum;
3324           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3325                                  src_address);
3326           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3327             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3328                                    s0->ext_host_addr.as_u32, ip4_header_t,
3329                                    dst_address);
3330           ip0->checksum = ip_csum_fold (sum0);
3331
3332           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3333             {
3334               old_port0 = tcp0->src_port;
3335               new_port0 = tcp0->src_port = s0->out2in.port;
3336
3337               sum0 = tcp0->checksum;
3338               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3339                                      dst_address);
3340               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3341                                      length);
3342               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3343                 {
3344                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3345                                          s0->ext_host_addr.as_u32,
3346                                          ip4_header_t, dst_address);
3347                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3348                                          s0->ext_host_port, ip4_header_t,
3349                                          length);
3350                   tcp0->dst_port = s0->ext_host_port;
3351                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3352                 }
3353               tcp0->checksum = ip_csum_fold(sum0);
3354               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3355                 goto trace00;
3356             }
3357           else
3358             {
3359               udp0->src_port = s0->out2in.port;
3360               udp0->checksum = 0;
3361               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3362                 {
3363                   udp0->dst_port = s0->ext_host_port;
3364                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3365                 }
3366             }
3367
3368           /* Accounting */
3369           nat44_session_update_counters (s0, now,
3370                                          vlib_buffer_length_in_chain (vm, b0));
3371
3372         trace00:
3373           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3374                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3375             {
3376               snat_in2out_trace_t *t =
3377                 vlib_add_trace (vm, node, b0, sizeof (*t));
3378               t->is_slow_path = is_slow_path;
3379               t->sw_if_index = sw_if_index0;
3380               t->next_index = next0;
3381               t->session_index = ~0;
3382               if (s0)
3383                 t->session_index = s0 - tsm->sessions;
3384             }
3385
3386           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3387
3388
3389           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3390
3391           if (is_output_feature)
3392             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3393
3394           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3395                  iph_offset1);
3396
3397           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3398           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3399                                                                sw_if_index1);
3400
3401           if (PREDICT_FALSE(ip1->ttl == 1))
3402             {
3403               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3404               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3405                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3406                                            0);
3407               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3408               goto trace01;
3409             }
3410
3411           udp1 = ip4_next_header (ip1);
3412           tcp1 = (tcp_header_t *) udp1;
3413           icmp1 = (icmp46_header_t *) udp1;
3414           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3415
3416           if (is_slow_path)
3417             {
3418               if (PREDICT_FALSE (proto1 == ~0))
3419                 {
3420                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3421                                                       rx_fib_index1,
3422                                                       thread_index, now, vm,
3423                                                       node);
3424                   if (!s1)
3425                     next1 = SNAT_IN2OUT_NEXT_DROP;
3426                   goto trace01;
3427                 }
3428
3429               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3430                 {
3431                   next1 = icmp_in2out_ed_slow_path
3432                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3433                      next1, now, thread_index, &s1);
3434                   goto trace01;
3435                 }
3436             }
3437           else
3438             {
3439                if (is_output_feature)
3440                 {
3441                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3442                       sm, ip1, thread_index, now, vm, b1)))
3443                     goto trace01;
3444                 }
3445
3446               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3447                 {
3448                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3449                   goto trace01;
3450                 }
3451
3452               if (ip4_is_fragment (ip1))
3453                 {
3454                   b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3455                   next1 = SNAT_IN2OUT_NEXT_DROP;
3456                   goto trace01;
3457                 }
3458             }
3459
3460           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3461                       rx_fib_index1, udp1->src_port, udp1->dst_port);
3462
3463           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3464             {
3465               if (is_slow_path)
3466                 {
3467                   if (is_output_feature)
3468                     {
3469                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3470                           sm, ip1, ip1->protocol, udp1->src_port,
3471                           udp1->dst_port, thread_index, sw_if_index1)))
3472                         goto trace01;
3473                     }
3474                   else
3475                     {
3476                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3477                           sw_if_index1, ip1, proto1, rx_fib_index1,
3478                           thread_index)))
3479                         goto trace01;
3480                     }
3481
3482                   next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3483                                         next1, thread_index, now);
3484
3485                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3486                     goto trace01;
3487                 }
3488               else
3489                 {
3490                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3491                   goto trace01;
3492                 }
3493             }
3494           else
3495             {
3496               s1 = pool_elt_at_index (tsm->sessions, value1.value);
3497             }
3498
3499           b1->flags |= VNET_BUFFER_F_IS_NATED;
3500
3501           if (!is_output_feature)
3502             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3503
3504           old_addr1 = ip1->src_address.as_u32;
3505           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3506           sum1 = ip1->checksum;
3507           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3508                                  src_address);
3509           if (PREDICT_FALSE (is_twice_nat_session (s1)))
3510             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3511                                    s1->ext_host_addr.as_u32, ip4_header_t,
3512                                    dst_address);
3513           ip1->checksum = ip_csum_fold (sum1);
3514
3515           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3516             {
3517               old_port1 = tcp1->src_port;
3518               new_port1 = tcp1->src_port = s1->out2in.port;
3519
3520               sum1 = tcp1->checksum;
3521               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3522                                      dst_address);
3523               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3524                                      length);
3525               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3526                 {
3527                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3528                                          s1->ext_host_addr.as_u32,
3529                                          ip4_header_t, dst_address);
3530                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
3531                                          s1->ext_host_port, ip4_header_t,
3532                                          length);
3533                   tcp1->dst_port = s1->ext_host_port;
3534                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3535                 }
3536               tcp1->checksum = ip_csum_fold(sum1);
3537               if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3538                 goto trace01;
3539             }
3540           else
3541             {
3542               udp1->src_port = s1->out2in.port;
3543               udp1->checksum = 0;
3544               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3545                 {
3546                   udp1->dst_port = s1->ext_host_port;
3547                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3548                 }
3549             }
3550
3551           /* Accounting */
3552           nat44_session_update_counters (s1, now,
3553                                          vlib_buffer_length_in_chain (vm, b1));
3554
3555         trace01:
3556           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3557                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3558             {
3559               snat_in2out_trace_t *t =
3560                 vlib_add_trace (vm, node, b1, sizeof (*t));
3561               t->is_slow_path = is_slow_path;
3562               t->sw_if_index = sw_if_index1;
3563               t->next_index = next1;
3564               t->session_index = ~0;
3565               if (s1)
3566                 t->session_index = s1 - tsm->sessions;
3567             }
3568
3569           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3570
3571           /* verify speculative enqueues, maybe switch current next frame */
3572           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3573                                            to_next, n_left_to_next,
3574                                            bi0, bi1, next0, next1);
3575         }
3576
3577       while (n_left_from > 0 && n_left_to_next > 0)
3578         {
3579           u32 bi0;
3580           vlib_buffer_t *b0;
3581           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3582               new_addr0, old_addr0;
3583           u16 old_port0, new_port0;
3584           ip4_header_t *ip0;
3585           udp_header_t *udp0;
3586           tcp_header_t *tcp0;
3587           icmp46_header_t * icmp0;
3588           snat_session_t *s0 = 0;
3589           clib_bihash_kv_16_8_t kv0, value0;
3590           ip_csum_t sum0;
3591
3592           /* speculatively enqueue b0 to the current next frame */
3593           bi0 = from[0];
3594           to_next[0] = bi0;
3595           from += 1;
3596           to_next += 1;
3597           n_left_from -= 1;
3598           n_left_to_next -= 1;
3599
3600           b0 = vlib_get_buffer (vm, bi0);
3601           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3602
3603           if (is_output_feature)
3604             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3605
3606           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3607                  iph_offset0);
3608
3609           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3610           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3611                                                                sw_if_index0);
3612
3613           if (PREDICT_FALSE(ip0->ttl == 1))
3614             {
3615               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3616               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3617                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3618                                            0);
3619               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3620               goto trace0;
3621             }
3622
3623           udp0 = ip4_next_header (ip0);
3624           tcp0 = (tcp_header_t *) udp0;
3625           icmp0 = (icmp46_header_t *) udp0;
3626           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3627
3628           if (is_slow_path)
3629             {
3630               if (PREDICT_FALSE (proto0 == ~0))
3631                 {
3632                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3633                                                       rx_fib_index0,
3634                                                       thread_index, now, vm,
3635                                                       node);
3636                   if (!s0)
3637                     next0 = SNAT_IN2OUT_NEXT_DROP;
3638                   goto trace0;
3639                 }
3640
3641               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3642                 {
3643                   next0 = icmp_in2out_ed_slow_path
3644                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3645                      next0, now, thread_index, &s0);
3646                   goto trace0;
3647                 }
3648             }
3649           else
3650             {
3651                if (is_output_feature)
3652                 {
3653                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3654                       sm, ip0, thread_index, now, vm, b0)))
3655                     goto trace0;
3656                 }
3657
3658               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3659                 {
3660                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3661                   goto trace0;
3662                 }
3663
3664               if (ip4_is_fragment (ip0))
3665                 {
3666                   b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3667                   next0 = SNAT_IN2OUT_NEXT_DROP;
3668                   goto trace0;
3669                 }
3670             }
3671
3672           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3673                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3674
3675           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3676             {
3677               if (is_slow_path)
3678                 {
3679                   if (is_output_feature)
3680                     {
3681                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3682                           sm, ip0, ip0->protocol, udp0->src_port,
3683                           udp0->dst_port, thread_index, sw_if_index0)))
3684                         goto trace0;
3685                     }
3686                   else
3687                     {
3688                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3689                           sw_if_index0, ip0, proto0, rx_fib_index0,
3690                           thread_index)))
3691                         goto trace0;
3692                     }
3693
3694                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3695                                         next0, thread_index, now);
3696
3697                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3698                     goto trace0;
3699                 }
3700               else
3701                 {
3702                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3703                   goto trace0;
3704                 }
3705             }
3706           else
3707             {
3708               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3709             }
3710
3711           b0->flags |= VNET_BUFFER_F_IS_NATED;
3712
3713           if (!is_output_feature)
3714             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3715
3716           old_addr0 = ip0->src_address.as_u32;
3717           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3718           sum0 = ip0->checksum;
3719           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3720                                  src_address);
3721           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3722             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3723                                    s0->ext_host_addr.as_u32, ip4_header_t,
3724                                    dst_address);
3725           ip0->checksum = ip_csum_fold (sum0);
3726
3727           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3728             {
3729               old_port0 = tcp0->src_port;
3730               new_port0 = tcp0->src_port = s0->out2in.port;
3731
3732               sum0 = tcp0->checksum;
3733               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3734                                      dst_address);
3735               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3736                                      length);
3737               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3738                 {
3739                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3740                                          s0->ext_host_addr.as_u32,
3741                                          ip4_header_t, dst_address);
3742                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3743                                          s0->ext_host_port, ip4_header_t,
3744                                          length);
3745                   tcp0->dst_port = s0->ext_host_port;
3746                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3747                 }
3748               tcp0->checksum = ip_csum_fold(sum0);
3749               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3750                 goto trace0;
3751             }
3752           else
3753             {
3754               udp0->src_port = s0->out2in.port;
3755               udp0->checksum = 0;
3756               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3757                 {
3758                   udp0->dst_port = s0->ext_host_port;
3759                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3760                 }
3761             }
3762
3763           /* Accounting */
3764           nat44_session_update_counters (s0, now,
3765                                          vlib_buffer_length_in_chain (vm, b0));
3766
3767         trace0:
3768           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3769                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3770             {
3771               snat_in2out_trace_t *t =
3772                 vlib_add_trace (vm, node, b0, sizeof (*t));
3773               t->is_slow_path = is_slow_path;
3774               t->sw_if_index = sw_if_index0;
3775               t->next_index = next0;
3776               t->session_index = ~0;
3777               if (s0)
3778                 t->session_index = s0 - tsm->sessions;
3779             }
3780
3781           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3782
3783           /* verify speculative enqueue, maybe switch current next frame */
3784           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3785                                            to_next, n_left_to_next,
3786                                            bi0, next0);
3787         }
3788
3789       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3790     }
3791
3792   vlib_node_increment_counter (vm, stats_node_index,
3793                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3794                                pkts_processed);
3795   return frame->n_vectors;
3796 }
3797
3798 static uword
3799 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3800                               vlib_node_runtime_t * node,
3801                               vlib_frame_t * frame)
3802 {
3803   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3804 }
3805
3806 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3807   .function = nat44_ed_in2out_fast_path_fn,
3808   .name = "nat44-ed-in2out",
3809   .vector_size = sizeof (u32),
3810   .format_trace = format_snat_in2out_trace,
3811   .type = VLIB_NODE_TYPE_INTERNAL,
3812
3813   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3814   .error_strings = snat_in2out_error_strings,
3815
3816   .runtime_data_bytes = sizeof (snat_runtime_t),
3817
3818   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3819
3820   /* edit / add dispositions here */
3821   .next_nodes = {
3822     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3823     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3824     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3825     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3826     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3827   },
3828 };
3829
3830 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3831
3832 static uword
3833 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3834                                      vlib_node_runtime_t * node,
3835                                      vlib_frame_t * frame)
3836 {
3837   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3838 }
3839
3840 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3841   .function = nat44_ed_in2out_output_fast_path_fn,
3842   .name = "nat44-ed-in2out-output",
3843   .vector_size = sizeof (u32),
3844   .format_trace = format_snat_in2out_trace,
3845   .type = VLIB_NODE_TYPE_INTERNAL,
3846
3847   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3848   .error_strings = snat_in2out_error_strings,
3849
3850   .runtime_data_bytes = sizeof (snat_runtime_t),
3851
3852   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3853
3854   /* edit / add dispositions here */
3855   .next_nodes = {
3856     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3857     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3858     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3859     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3860     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3861   },
3862 };
3863
3864 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
3865                               nat44_ed_in2out_output_fast_path_fn);
3866
3867 static uword
3868 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
3869                               vlib_node_runtime_t * node,
3870                               vlib_frame_t * frame)
3871 {
3872   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3873 }
3874
3875 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
3876   .function = nat44_ed_in2out_slow_path_fn,
3877   .name = "nat44-ed-in2out-slowpath",
3878   .vector_size = sizeof (u32),
3879   .format_trace = format_snat_in2out_trace,
3880   .type = VLIB_NODE_TYPE_INTERNAL,
3881
3882   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3883   .error_strings = snat_in2out_error_strings,
3884
3885   .runtime_data_bytes = sizeof (snat_runtime_t),
3886
3887   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3888
3889   /* edit / add dispositions here */
3890   .next_nodes = {
3891     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3892     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3893     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3894     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3895     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3896   },
3897 };
3898
3899 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
3900                               nat44_ed_in2out_slow_path_fn);
3901
3902 static uword
3903 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
3904                                      vlib_node_runtime_t * node,
3905                                      vlib_frame_t * frame)
3906 {
3907   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3908 }
3909
3910 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
3911   .function = nat44_ed_in2out_output_slow_path_fn,
3912   .name = "nat44-ed-in2out-output-slowpath",
3913   .vector_size = sizeof (u32),
3914   .format_trace = format_snat_in2out_trace,
3915   .type = VLIB_NODE_TYPE_INTERNAL,
3916
3917   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3918   .error_strings = snat_in2out_error_strings,
3919
3920   .runtime_data_bytes = sizeof (snat_runtime_t),
3921
3922   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3923
3924   /* edit / add dispositions here */
3925   .next_nodes = {
3926     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3927     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3928     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3929     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3930     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3931   },
3932 };
3933
3934 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
3935                               nat44_ed_in2out_output_slow_path_fn);
3936
3937 /**************************/
3938 /*** deterministic mode ***/
3939 /**************************/
3940 static uword
3941 snat_det_in2out_node_fn (vlib_main_t * vm,
3942                          vlib_node_runtime_t * node,
3943                          vlib_frame_t * frame)
3944 {
3945   u32 n_left_from, * from, * to_next;
3946   snat_in2out_next_t next_index;
3947   u32 pkts_processed = 0;
3948   snat_main_t * sm = &snat_main;
3949   u32 now = (u32) vlib_time_now (vm);
3950   u32 thread_index = vm->thread_index;
3951
3952   from = vlib_frame_vector_args (frame);
3953   n_left_from = frame->n_vectors;
3954   next_index = node->cached_next_index;
3955
3956   while (n_left_from > 0)
3957     {
3958       u32 n_left_to_next;
3959
3960       vlib_get_next_frame (vm, node, next_index,
3961                            to_next, n_left_to_next);
3962
3963       while (n_left_from >= 4 && n_left_to_next >= 2)
3964         {
3965           u32 bi0, bi1;
3966           vlib_buffer_t * b0, * b1;
3967           u32 next0, next1;
3968           u32 sw_if_index0, sw_if_index1;
3969           ip4_header_t * ip0, * ip1;
3970           ip_csum_t sum0, sum1;
3971           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3972           u16 old_port0, new_port0, lo_port0, i0;
3973           u16 old_port1, new_port1, lo_port1, i1;
3974           udp_header_t * udp0, * udp1;
3975           tcp_header_t * tcp0, * tcp1;
3976           u32 proto0, proto1;
3977           snat_det_out_key_t key0, key1;
3978           snat_det_map_t * dm0, * dm1;
3979           snat_det_session_t * ses0 = 0, * ses1 = 0;
3980           u32 rx_fib_index0, rx_fib_index1;
3981           icmp46_header_t * icmp0, * icmp1;
3982
3983           /* Prefetch next iteration. */
3984           {
3985             vlib_buffer_t * p2, * p3;
3986
3987             p2 = vlib_get_buffer (vm, from[2]);
3988             p3 = vlib_get_buffer (vm, from[3]);
3989
3990             vlib_prefetch_buffer_header (p2, LOAD);
3991             vlib_prefetch_buffer_header (p3, LOAD);
3992
3993             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3994             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3995           }
3996
3997           /* speculatively enqueue b0 and b1 to the current next frame */
3998           to_next[0] = bi0 = from[0];
3999           to_next[1] = bi1 = from[1];
4000           from += 2;
4001           to_next += 2;
4002           n_left_from -= 2;
4003           n_left_to_next -= 2;
4004
4005           b0 = vlib_get_buffer (vm, bi0);
4006           b1 = vlib_get_buffer (vm, bi1);
4007
4008           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4009           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
4010
4011           ip0 = vlib_buffer_get_current (b0);
4012           udp0 = ip4_next_header (ip0);
4013           tcp0 = (tcp_header_t *) udp0;
4014
4015           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4016
4017           if (PREDICT_FALSE(ip0->ttl == 1))
4018             {
4019               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4020               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4021                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4022                                            0);
4023               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4024               goto trace0;
4025             }
4026
4027           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4028
4029           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4030             {
4031               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4032               icmp0 = (icmp46_header_t *) udp0;
4033
4034               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4035                                   rx_fib_index0, node, next0, thread_index,
4036                                   &ses0, &dm0);
4037               goto trace0;
4038             }
4039
4040           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4041           if (PREDICT_FALSE(!dm0))
4042             {
4043               nat_log_info ("no match for internal host %U",
4044                             format_ip4_address, &ip0->src_address);
4045               next0 = SNAT_IN2OUT_NEXT_DROP;
4046               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4047               goto trace0;
4048             }
4049
4050           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4051
4052           key0.ext_host_addr = ip0->dst_address;
4053           key0.ext_host_port = tcp0->dst;
4054
4055           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4056           if (PREDICT_FALSE(!ses0))
4057             {
4058               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4059                 {
4060                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
4061                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4062
4063                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4064                     continue;
4065
4066                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4067                   break;
4068                 }
4069               if (PREDICT_FALSE(!ses0))
4070                 {
4071                   /* too many sessions for user, send ICMP error packet */
4072
4073                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4074                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4075                                                ICMP4_destination_unreachable_destination_unreachable_host,
4076                                                0);
4077                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4078                   goto trace0;
4079                 }
4080             }
4081
4082           new_port0 = ses0->out.out_port;
4083
4084           old_addr0.as_u32 = ip0->src_address.as_u32;
4085           ip0->src_address.as_u32 = new_addr0.as_u32;
4086           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4087
4088           sum0 = ip0->checksum;
4089           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4090                                  ip4_header_t,
4091                                  src_address /* changed member */);
4092           ip0->checksum = ip_csum_fold (sum0);
4093
4094           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4095             {
4096               if (tcp0->flags & TCP_FLAG_SYN)
4097                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4098               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4099                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4100               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4101                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4102               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4103                 snat_det_ses_close(dm0, ses0);
4104               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4105                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4106               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4107                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4108
4109               old_port0 = tcp0->src;
4110               tcp0->src = new_port0;
4111
4112               sum0 = tcp0->checksum;
4113               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4114                                      ip4_header_t,
4115                                      dst_address /* changed member */);
4116               sum0 = ip_csum_update (sum0, old_port0, new_port0,
4117                                      ip4_header_t /* cheat */,
4118                                      length /* changed member */);
4119               tcp0->checksum = ip_csum_fold(sum0);
4120             }
4121           else
4122             {
4123               ses0->state = SNAT_SESSION_UDP_ACTIVE;
4124               old_port0 = udp0->src_port;
4125               udp0->src_port = new_port0;
4126               udp0->checksum = 0;
4127             }
4128
4129           switch(ses0->state)
4130             {
4131             case SNAT_SESSION_UDP_ACTIVE:
4132                 ses0->expire = now + sm->udp_timeout;
4133                 break;
4134             case SNAT_SESSION_TCP_SYN_SENT:
4135             case SNAT_SESSION_TCP_FIN_WAIT:
4136             case SNAT_SESSION_TCP_CLOSE_WAIT:
4137             case SNAT_SESSION_TCP_LAST_ACK:
4138                 ses0->expire = now + sm->tcp_transitory_timeout;
4139                 break;
4140             case SNAT_SESSION_TCP_ESTABLISHED:
4141                 ses0->expire = now + sm->tcp_established_timeout;
4142                 break;
4143             }
4144
4145         trace0:
4146           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4147                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4148             {
4149               snat_in2out_trace_t *t =
4150                  vlib_add_trace (vm, node, b0, sizeof (*t));
4151               t->is_slow_path = 0;
4152               t->sw_if_index = sw_if_index0;
4153               t->next_index = next0;
4154               t->session_index = ~0;
4155               if (ses0)
4156                 t->session_index = ses0 - dm0->sessions;
4157             }
4158
4159           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4160
4161           ip1 = vlib_buffer_get_current (b1);
4162           udp1 = ip4_next_header (ip1);
4163           tcp1 = (tcp_header_t *) udp1;
4164
4165           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4166
4167           if (PREDICT_FALSE(ip1->ttl == 1))
4168             {
4169               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4170               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4171                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4172                                            0);
4173               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4174               goto trace1;
4175             }
4176
4177           proto1 = ip_proto_to_snat_proto (ip1->protocol);
4178
4179           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4180             {
4181               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4182               icmp1 = (icmp46_header_t *) udp1;
4183
4184               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4185                                   rx_fib_index1, node, next1, thread_index,
4186                                   &ses1, &dm1);
4187               goto trace1;
4188             }
4189
4190           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4191           if (PREDICT_FALSE(!dm1))
4192             {
4193               nat_log_info ("no match for internal host %U",
4194                             format_ip4_address, &ip0->src_address);
4195               next1 = SNAT_IN2OUT_NEXT_DROP;
4196               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4197               goto trace1;
4198             }
4199
4200           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4201
4202           key1.ext_host_addr = ip1->dst_address;
4203           key1.ext_host_port = tcp1->dst;
4204
4205           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4206           if (PREDICT_FALSE(!ses1))
4207             {
4208               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4209                 {
4210                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
4211                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4212
4213                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4214                     continue;
4215
4216                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4217                   break;
4218                 }
4219               if (PREDICT_FALSE(!ses1))
4220                 {
4221                   /* too many sessions for user, send ICMP error packet */
4222
4223                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4224                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4225                                                ICMP4_destination_unreachable_destination_unreachable_host,
4226                                                0);
4227                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4228                   goto trace1;
4229                 }
4230             }
4231
4232           new_port1 = ses1->out.out_port;
4233
4234           old_addr1.as_u32 = ip1->src_address.as_u32;
4235           ip1->src_address.as_u32 = new_addr1.as_u32;
4236           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4237
4238           sum1 = ip1->checksum;
4239           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4240                                  ip4_header_t,
4241                                  src_address /* changed member */);
4242           ip1->checksum = ip_csum_fold (sum1);
4243
4244           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4245             {
4246               if (tcp1->flags & TCP_FLAG_SYN)
4247                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4248               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4249                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4250               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4251                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4252               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4253                 snat_det_ses_close(dm1, ses1);
4254               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4255                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4256               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4257                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4258
4259               old_port1 = tcp1->src;
4260               tcp1->src = new_port1;
4261
4262               sum1 = tcp1->checksum;
4263               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4264                                      ip4_header_t,
4265                                      dst_address /* changed member */);
4266               sum1 = ip_csum_update (sum1, old_port1, new_port1,
4267                                      ip4_header_t /* cheat */,
4268                                      length /* changed member */);
4269               tcp1->checksum = ip_csum_fold(sum1);
4270             }
4271           else
4272             {
4273               ses1->state = SNAT_SESSION_UDP_ACTIVE;
4274               old_port1 = udp1->src_port;
4275               udp1->src_port = new_port1;
4276               udp1->checksum = 0;
4277             }
4278
4279           switch(ses1->state)
4280             {
4281             case SNAT_SESSION_UDP_ACTIVE:
4282                 ses1->expire = now + sm->udp_timeout;
4283                 break;
4284             case SNAT_SESSION_TCP_SYN_SENT:
4285             case SNAT_SESSION_TCP_FIN_WAIT:
4286             case SNAT_SESSION_TCP_CLOSE_WAIT:
4287             case SNAT_SESSION_TCP_LAST_ACK:
4288                 ses1->expire = now + sm->tcp_transitory_timeout;
4289                 break;
4290             case SNAT_SESSION_TCP_ESTABLISHED:
4291                 ses1->expire = now + sm->tcp_established_timeout;
4292                 break;
4293             }
4294
4295         trace1:
4296           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4297                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4298             {
4299               snat_in2out_trace_t *t =
4300                  vlib_add_trace (vm, node, b1, sizeof (*t));
4301               t->is_slow_path = 0;
4302               t->sw_if_index = sw_if_index1;
4303               t->next_index = next1;
4304               t->session_index = ~0;
4305               if (ses1)
4306                 t->session_index = ses1 - dm1->sessions;
4307             }
4308
4309           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4310
4311           /* verify speculative enqueues, maybe switch current next frame */
4312           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4313                                            to_next, n_left_to_next,
4314                                            bi0, bi1, next0, next1);
4315          }
4316
4317       while (n_left_from > 0 && n_left_to_next > 0)
4318         {
4319           u32 bi0;
4320           vlib_buffer_t * b0;
4321           u32 next0;
4322           u32 sw_if_index0;
4323           ip4_header_t * ip0;
4324           ip_csum_t sum0;
4325           ip4_address_t new_addr0, old_addr0;
4326           u16 old_port0, new_port0, lo_port0, i0;
4327           udp_header_t * udp0;
4328           tcp_header_t * tcp0;
4329           u32 proto0;
4330           snat_det_out_key_t key0;
4331           snat_det_map_t * dm0;
4332           snat_det_session_t * ses0 = 0;
4333           u32 rx_fib_index0;
4334           icmp46_header_t * icmp0;
4335
4336           /* speculatively enqueue b0 to the current next frame */
4337           bi0 = from[0];
4338           to_next[0] = bi0;
4339           from += 1;
4340           to_next += 1;
4341           n_left_from -= 1;
4342           n_left_to_next -= 1;
4343
4344           b0 = vlib_get_buffer (vm, bi0);
4345           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4346
4347           ip0 = vlib_buffer_get_current (b0);
4348           udp0 = ip4_next_header (ip0);
4349           tcp0 = (tcp_header_t *) udp0;
4350
4351           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4352
4353           if (PREDICT_FALSE(ip0->ttl == 1))
4354             {
4355               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4356               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4357                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4358                                            0);
4359               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4360               goto trace00;
4361             }
4362
4363           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4364
4365           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4366             {
4367               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4368               icmp0 = (icmp46_header_t *) udp0;
4369
4370               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4371                                   rx_fib_index0, node, next0, thread_index,
4372                                   &ses0, &dm0);
4373               goto trace00;
4374             }
4375
4376           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4377           if (PREDICT_FALSE(!dm0))
4378             {
4379               nat_log_info ("no match for internal host %U",
4380                             format_ip4_address, &ip0->src_address);
4381               next0 = SNAT_IN2OUT_NEXT_DROP;
4382               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4383               goto trace00;
4384             }
4385
4386           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4387
4388           key0.ext_host_addr = ip0->dst_address;
4389           key0.ext_host_port = tcp0->dst;
4390
4391           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4392           if (PREDICT_FALSE(!ses0))
4393             {
4394               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4395                 {
4396                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
4397                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4398
4399                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4400                     continue;
4401
4402                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4403                   break;
4404                 }
4405               if (PREDICT_FALSE(!ses0))
4406                 {
4407                   /* too many sessions for user, send ICMP error packet */
4408
4409                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4410                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4411                                                ICMP4_destination_unreachable_destination_unreachable_host,
4412                                                0);
4413                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4414                   goto trace00;
4415                 }
4416             }
4417
4418           new_port0 = ses0->out.out_port;
4419
4420           old_addr0.as_u32 = ip0->src_address.as_u32;
4421           ip0->src_address.as_u32 = new_addr0.as_u32;
4422           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4423
4424           sum0 = ip0->checksum;
4425           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4426                                  ip4_header_t,
4427                                  src_address /* changed member */);
4428           ip0->checksum = ip_csum_fold (sum0);
4429
4430           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4431             {
4432               if (tcp0->flags & TCP_FLAG_SYN)
4433                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4434               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4435                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4436               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4437                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4438               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4439                 snat_det_ses_close(dm0, ses0);
4440               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4441                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4442               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4443                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4444
4445               old_port0 = tcp0->src;
4446               tcp0->src = new_port0;
4447
4448               sum0 = tcp0->checksum;
4449               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4450                                      ip4_header_t,
4451                                      dst_address /* changed member */);
4452               sum0 = ip_csum_update (sum0, old_port0, new_port0,
4453                                      ip4_header_t /* cheat */,
4454                                      length /* changed member */);
4455               tcp0->checksum = ip_csum_fold(sum0);
4456             }
4457           else
4458             {
4459               ses0->state = SNAT_SESSION_UDP_ACTIVE;
4460               old_port0 = udp0->src_port;
4461               udp0->src_port = new_port0;
4462               udp0->checksum = 0;
4463             }
4464
4465           switch(ses0->state)
4466             {
4467             case SNAT_SESSION_UDP_ACTIVE:
4468                 ses0->expire = now + sm->udp_timeout;
4469                 break;
4470             case SNAT_SESSION_TCP_SYN_SENT:
4471             case SNAT_SESSION_TCP_FIN_WAIT:
4472             case SNAT_SESSION_TCP_CLOSE_WAIT:
4473             case SNAT_SESSION_TCP_LAST_ACK:
4474                 ses0->expire = now + sm->tcp_transitory_timeout;
4475                 break;
4476             case SNAT_SESSION_TCP_ESTABLISHED:
4477                 ses0->expire = now + sm->tcp_established_timeout;
4478                 break;
4479             }
4480
4481         trace00:
4482           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4483                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4484             {
4485               snat_in2out_trace_t *t =
4486                  vlib_add_trace (vm, node, b0, sizeof (*t));
4487               t->is_slow_path = 0;
4488               t->sw_if_index = sw_if_index0;
4489               t->next_index = next0;
4490               t->session_index = ~0;
4491               if (ses0)
4492                 t->session_index = ses0 - dm0->sessions;
4493             }
4494
4495           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4496
4497           /* verify speculative enqueue, maybe switch current next frame */
4498           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4499                                            to_next, n_left_to_next,
4500                                            bi0, next0);
4501         }
4502
4503       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4504     }
4505
4506   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
4507                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4508                                pkts_processed);
4509   return frame->n_vectors;
4510 }
4511
4512 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
4513   .function = snat_det_in2out_node_fn,
4514   .name = "nat44-det-in2out",
4515   .vector_size = sizeof (u32),
4516   .format_trace = format_snat_in2out_trace,
4517   .type = VLIB_NODE_TYPE_INTERNAL,
4518
4519   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4520   .error_strings = snat_in2out_error_strings,
4521
4522   .runtime_data_bytes = sizeof (snat_runtime_t),
4523
4524   .n_next_nodes = 3,
4525
4526   /* edit / add dispositions here */
4527   .next_nodes = {
4528     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4529     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4530     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4531   },
4532 };
4533
4534 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
4535
4536 /**
4537  * Get address and port values to be used for ICMP packet translation
4538  * and create session if needed
4539  *
4540  * @param[in,out] sm             NAT main
4541  * @param[in,out] node           NAT node runtime
4542  * @param[in] thread_index       thread index
4543  * @param[in,out] b0             buffer containing packet to be translated
4544  * @param[out] p_proto           protocol used for matching
4545  * @param[out] p_value           address and port after NAT translation
4546  * @param[out] p_dont_translate  if packet should not be translated
4547  * @param d                      optional parameter
4548  * @param e                      optional parameter
4549  */
4550 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
4551                           u32 thread_index, vlib_buffer_t *b0,
4552                           ip4_header_t *ip0, u8 *p_proto,
4553                           snat_session_key_t *p_value,
4554                           u8 *p_dont_translate, void *d, void *e)
4555 {
4556   icmp46_header_t *icmp0;
4557   u32 sw_if_index0;
4558   u32 rx_fib_index0;
4559   u8 protocol;
4560   snat_det_out_key_t key0;
4561   u8 dont_translate = 0;
4562   u32 next0 = ~0;
4563   icmp_echo_header_t *echo0, *inner_echo0 = 0;
4564   ip4_header_t *inner_ip0;
4565   void *l4_header = 0;
4566   icmp46_header_t *inner_icmp0;
4567   snat_det_map_t * dm0 = 0;
4568   ip4_address_t new_addr0;
4569   u16 lo_port0, i0;
4570   snat_det_session_t * ses0 = 0;
4571   ip4_address_t in_addr;
4572   u16 in_port;
4573
4574   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4575   echo0 = (icmp_echo_header_t *)(icmp0+1);
4576   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4577   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4578
4579   if (!icmp_is_error_message (icmp0))
4580     {
4581       protocol = SNAT_PROTOCOL_ICMP;
4582       in_addr = ip0->src_address;
4583       in_port = echo0->identifier;
4584     }
4585   else
4586     {
4587       inner_ip0 = (ip4_header_t *)(echo0+1);
4588       l4_header = ip4_next_header (inner_ip0);
4589       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4590       in_addr = inner_ip0->dst_address;
4591       switch (protocol)
4592         {
4593         case SNAT_PROTOCOL_ICMP:
4594           inner_icmp0 = (icmp46_header_t*)l4_header;
4595           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4596           in_port = inner_echo0->identifier;
4597           break;
4598         case SNAT_PROTOCOL_UDP:
4599         case SNAT_PROTOCOL_TCP:
4600           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4601           break;
4602         default:
4603           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4604           next0 = SNAT_IN2OUT_NEXT_DROP;
4605           goto out;
4606         }
4607     }
4608
4609   dm0 = snat_det_map_by_user(sm, &in_addr);
4610   if (PREDICT_FALSE(!dm0))
4611     {
4612       nat_log_info ("no match for internal host %U",
4613                     format_ip4_address, &in_addr);
4614       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4615           IP_PROTOCOL_ICMP, rx_fib_index0)))
4616         {
4617           dont_translate = 1;
4618           goto out;
4619         }
4620       next0 = SNAT_IN2OUT_NEXT_DROP;
4621       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4622       goto out;
4623     }
4624
4625   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4626
4627   key0.ext_host_addr = ip0->dst_address;
4628   key0.ext_host_port = 0;
4629
4630   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4631   if (PREDICT_FALSE(!ses0))
4632     {
4633       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4634           IP_PROTOCOL_ICMP, rx_fib_index0)))
4635         {
4636           dont_translate = 1;
4637           goto out;
4638         }
4639       if (icmp0->type != ICMP4_echo_request)
4640         {
4641           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4642           next0 = SNAT_IN2OUT_NEXT_DROP;
4643           goto out;
4644         }
4645       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4646         {
4647           key0.out_port = clib_host_to_net_u16 (lo_port0 +
4648             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4649
4650           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4651             continue;
4652
4653           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4654           break;
4655         }
4656       if (PREDICT_FALSE(!ses0))
4657         {
4658           next0 = SNAT_IN2OUT_NEXT_DROP;
4659           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4660           goto out;
4661         }
4662     }
4663
4664   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4665                     !icmp_is_error_message (icmp0)))
4666     {
4667       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4668       next0 = SNAT_IN2OUT_NEXT_DROP;
4669       goto out;
4670     }
4671
4672   u32 now = (u32) vlib_time_now (sm->vlib_main);
4673
4674   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4675   ses0->expire = now + sm->icmp_timeout;
4676
4677 out:
4678   *p_proto = protocol;
4679   if (ses0)
4680     {
4681       p_value->addr = new_addr0;
4682       p_value->fib_index = sm->outside_fib_index;
4683       p_value->port = ses0->out.out_port;
4684     }
4685   *p_dont_translate = dont_translate;
4686   if (d)
4687     *(snat_det_session_t**)d = ses0;
4688   if (e)
4689     *(snat_det_map_t**)e = dm0;
4690   return next0;
4691 }
4692
4693 /**********************/
4694 /*** worker handoff ***/
4695 /**********************/
4696 static inline uword
4697 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
4698                                       vlib_node_runtime_t * node,
4699                                       vlib_frame_t * frame,
4700                                       u8 is_output)
4701 {
4702   snat_main_t *sm = &snat_main;
4703   vlib_thread_main_t *tm = vlib_get_thread_main ();
4704   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4705   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4706   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4707     = 0;
4708   vlib_frame_queue_elt_t *hf = 0;
4709   vlib_frame_queue_t *fq;
4710   vlib_frame_t *f = 0;
4711   int i;
4712   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4713   u32 next_worker_index = 0;
4714   u32 current_worker_index = ~0;
4715   u32 thread_index = vm->thread_index;
4716   u32 fq_index;
4717   u32 to_node_index;
4718   vlib_frame_t *d = 0;
4719
4720   ASSERT (vec_len (sm->workers));
4721
4722   if (is_output)
4723     {
4724       fq_index = sm->fq_in2out_output_index;
4725       to_node_index = sm->in2out_output_node_index;
4726     }
4727   else
4728     {
4729       fq_index = sm->fq_in2out_index;
4730       to_node_index = sm->in2out_node_index;
4731     }
4732
4733   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4734     {
4735       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4736
4737       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4738                                tm->n_vlib_mains - 1,
4739                                (vlib_frame_queue_t *) (~0));
4740     }
4741
4742   from = vlib_frame_vector_args (frame);
4743   n_left_from = frame->n_vectors;
4744
4745   while (n_left_from > 0)
4746     {
4747       u32 bi0;
4748       vlib_buffer_t *b0;
4749       u32 sw_if_index0;
4750       u32 rx_fib_index0;
4751       ip4_header_t * ip0;
4752       u8 do_handoff;
4753
4754       bi0 = from[0];
4755       from += 1;
4756       n_left_from -= 1;
4757
4758       b0 = vlib_get_buffer (vm, bi0);
4759
4760       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4761       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4762
4763       ip0 = vlib_buffer_get_current (b0);
4764
4765       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4766
4767       if (PREDICT_FALSE (next_worker_index != thread_index))
4768         {
4769           do_handoff = 1;
4770
4771           if (next_worker_index != current_worker_index)
4772             {
4773               fq = is_vlib_frame_queue_congested (
4774                 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4775                 congested_handoff_queue_by_worker_index);
4776
4777               if (fq)
4778                 {
4779                   /* if this is 1st frame */
4780                   if (!d)
4781                     {
4782                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
4783                       to_next_drop = vlib_frame_vector_args (d);
4784                     }
4785
4786                   to_next_drop[0] = bi0;
4787                   to_next_drop += 1;
4788                   d->n_vectors++;
4789                   b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4790                   goto trace0;
4791                 }
4792
4793               if (hf)
4794                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4795
4796               hf = vlib_get_worker_handoff_queue_elt (fq_index,
4797                                                       next_worker_index,
4798                                                       handoff_queue_elt_by_worker_index);
4799
4800               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4801               to_next_worker = &hf->buffer_index[hf->n_vectors];
4802               current_worker_index = next_worker_index;
4803             }
4804
4805           /* enqueue to correct worker thread */
4806           to_next_worker[0] = bi0;
4807           to_next_worker++;
4808           n_left_to_next_worker--;
4809
4810           if (n_left_to_next_worker == 0)
4811             {
4812               hf->n_vectors = VLIB_FRAME_SIZE;
4813               vlib_put_frame_queue_elt (hf);
4814               current_worker_index = ~0;
4815               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4816               hf = 0;
4817             }
4818         }
4819       else
4820         {
4821           do_handoff = 0;
4822           /* if this is 1st frame */
4823           if (!f)
4824             {
4825               f = vlib_get_frame_to_node (vm, to_node_index);
4826               to_next = vlib_frame_vector_args (f);
4827             }
4828
4829           to_next[0] = bi0;
4830           to_next += 1;
4831           f->n_vectors++;
4832         }
4833
4834 trace0:
4835       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
4836                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4837         {
4838           snat_in2out_worker_handoff_trace_t *t =
4839             vlib_add_trace (vm, node, b0, sizeof (*t));
4840           t->next_worker_index = next_worker_index;
4841           t->do_handoff = do_handoff;
4842         }
4843     }
4844
4845   if (f)
4846     vlib_put_frame_to_node (vm, to_node_index, f);
4847
4848   if (d)
4849     vlib_put_frame_to_node (vm, sm->error_node_index, d);
4850
4851   if (hf)
4852     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4853
4854   /* Ship frames to the worker nodes */
4855   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4856     {
4857       if (handoff_queue_elt_by_worker_index[i])
4858         {
4859           hf = handoff_queue_elt_by_worker_index[i];
4860           /*
4861            * It works better to let the handoff node
4862            * rate-adapt, always ship the handoff queue element.
4863            */
4864           if (1 || hf->n_vectors == hf->last_n_vectors)
4865             {
4866               vlib_put_frame_queue_elt (hf);
4867               handoff_queue_elt_by_worker_index[i] = 0;
4868             }
4869           else
4870             hf->last_n_vectors = hf->n_vectors;
4871         }
4872       congested_handoff_queue_by_worker_index[i] =
4873         (vlib_frame_queue_t *) (~0);
4874     }
4875   hf = 0;
4876   current_worker_index = ~0;
4877   return frame->n_vectors;
4878 }
4879
4880 static uword
4881 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
4882                                vlib_node_runtime_t * node,
4883                                vlib_frame_t * frame)
4884 {
4885   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4886 }
4887
4888 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
4889   .function = snat_in2out_worker_handoff_fn,
4890   .name = "nat44-in2out-worker-handoff",
4891   .vector_size = sizeof (u32),
4892   .format_trace = format_snat_in2out_worker_handoff_trace,
4893   .type = VLIB_NODE_TYPE_INTERNAL,
4894
4895   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4896   .error_strings = snat_in2out_error_strings,
4897
4898   .n_next_nodes = 1,
4899
4900   .next_nodes = {
4901     [0] = "error-drop",
4902   },
4903 };
4904
4905 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
4906                               snat_in2out_worker_handoff_fn);
4907
4908 static uword
4909 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
4910                                       vlib_node_runtime_t * node,
4911                                       vlib_frame_t * frame)
4912 {
4913   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4914 }
4915
4916 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
4917   .function = snat_in2out_output_worker_handoff_fn,
4918   .name = "nat44-in2out-output-worker-handoff",
4919   .vector_size = sizeof (u32),
4920   .format_trace = format_snat_in2out_worker_handoff_trace,
4921   .type = VLIB_NODE_TYPE_INTERNAL,
4922
4923   .n_next_nodes = 1,
4924
4925   .next_nodes = {
4926     [0] = "error-drop",
4927   },
4928 };
4929
4930 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
4931                               snat_in2out_output_worker_handoff_fn);
4932
4933 static_always_inline int
4934 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
4935 {
4936   snat_address_t * ap;
4937   clib_bihash_kv_8_8_t kv, value;
4938   snat_session_key_t m_key;
4939
4940   vec_foreach (ap, sm->addresses)
4941     {
4942       if (ap->addr.as_u32 == dst_addr->as_u32)
4943         return 1;
4944     }
4945
4946   m_key.addr.as_u32 = dst_addr->as_u32;
4947   m_key.fib_index = 0;
4948   m_key.port = 0;
4949   m_key.protocol = 0;
4950   kv.key = m_key.as_u64;
4951   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4952     return 1;
4953
4954   return 0;
4955 }
4956
4957 static inline uword
4958 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
4959                             vlib_node_runtime_t * node,
4960                             vlib_frame_t * frame,
4961                             int is_ed)
4962 {
4963   u32 n_left_from, * from, * to_next, stats_node_index;
4964   snat_in2out_next_t next_index;
4965   u32 pkts_processed = 0;
4966   snat_main_t * sm = &snat_main;
4967
4968   stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4969     snat_hairpin_dst_node.index;
4970
4971   from = vlib_frame_vector_args (frame);
4972   n_left_from = frame->n_vectors;
4973   next_index = node->cached_next_index;
4974
4975   while (n_left_from > 0)
4976     {
4977       u32 n_left_to_next;
4978
4979       vlib_get_next_frame (vm, node, next_index,
4980                            to_next, n_left_to_next);
4981
4982       while (n_left_from > 0 && n_left_to_next > 0)
4983         {
4984           u32 bi0;
4985           vlib_buffer_t * b0;
4986           u32 next0;
4987           ip4_header_t * ip0;
4988           u32 proto0;
4989
4990           /* speculatively enqueue b0 to the current next frame */
4991           bi0 = from[0];
4992           to_next[0] = bi0;
4993           from += 1;
4994           to_next += 1;
4995           n_left_from -= 1;
4996           n_left_to_next -= 1;
4997
4998           b0 = vlib_get_buffer (vm, bi0);
4999           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5000           ip0 = vlib_buffer_get_current (b0);
5001
5002           proto0 = ip_proto_to_snat_proto (ip0->protocol);
5003
5004           vnet_buffer (b0)->snat.flags = 0;
5005           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
5006             {
5007               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
5008                 {
5009                   udp_header_t * udp0 = ip4_next_header (ip0);
5010                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
5011
5012                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
5013                 }
5014               else if (proto0 == SNAT_PROTOCOL_ICMP)
5015                 {
5016                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
5017
5018                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
5019                 }
5020               else
5021                 {
5022                   if (is_ed)
5023                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
5024                   else
5025                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
5026                 }
5027
5028               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
5029             }
5030
5031           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5032
5033           /* verify speculative enqueue, maybe switch current next frame */
5034           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5035                                            to_next, n_left_to_next,
5036                                            bi0, next0);
5037          }
5038
5039       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5040     }
5041
5042   vlib_node_increment_counter (vm, stats_node_index,
5043                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5044                                pkts_processed);
5045   return frame->n_vectors;
5046 }
5047
5048 static uword
5049 snat_hairpin_dst_fn (vlib_main_t * vm,
5050                      vlib_node_runtime_t * node,
5051                      vlib_frame_t * frame)
5052 {
5053   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
5054 }
5055
5056 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
5057   .function = snat_hairpin_dst_fn,
5058   .name = "nat44-hairpin-dst",
5059   .vector_size = sizeof (u32),
5060   .type = VLIB_NODE_TYPE_INTERNAL,
5061   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5062   .error_strings = snat_in2out_error_strings,
5063   .n_next_nodes = 2,
5064   .next_nodes = {
5065     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5066     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5067   },
5068 };
5069
5070 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
5071                               snat_hairpin_dst_fn);
5072
5073 static uword
5074 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
5075                          vlib_node_runtime_t * node,
5076                          vlib_frame_t * frame)
5077 {
5078   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
5079 }
5080
5081 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
5082   .function = nat44_ed_hairpin_dst_fn,
5083   .name = "nat44-ed-hairpin-dst",
5084   .vector_size = sizeof (u32),
5085   .type = VLIB_NODE_TYPE_INTERNAL,
5086   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5087   .error_strings = snat_in2out_error_strings,
5088   .n_next_nodes = 2,
5089   .next_nodes = {
5090     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5091     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5092   },
5093 };
5094
5095 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
5096                               nat44_ed_hairpin_dst_fn);
5097
5098 static inline uword
5099 snat_hairpin_src_fn_inline (vlib_main_t * vm,
5100                             vlib_node_runtime_t * node,
5101                             vlib_frame_t * frame,
5102                             int is_ed)
5103 {
5104   u32 n_left_from, * from, * to_next, stats_node_index;
5105   snat_in2out_next_t next_index;
5106   u32 pkts_processed = 0;
5107   snat_main_t *sm = &snat_main;
5108
5109   stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
5110     snat_hairpin_src_node.index;
5111
5112   from = vlib_frame_vector_args (frame);
5113   n_left_from = frame->n_vectors;
5114   next_index = node->cached_next_index;
5115
5116   while (n_left_from > 0)
5117     {
5118       u32 n_left_to_next;
5119
5120       vlib_get_next_frame (vm, node, next_index,
5121                            to_next, n_left_to_next);
5122
5123       while (n_left_from > 0 && n_left_to_next > 0)
5124         {
5125           u32 bi0;
5126           vlib_buffer_t * b0;
5127           u32 next0;
5128           snat_interface_t *i;
5129           u32 sw_if_index0;
5130
5131           /* speculatively enqueue b0 to the current next frame */
5132           bi0 = from[0];
5133           to_next[0] = bi0;
5134           from += 1;
5135           to_next += 1;
5136           n_left_from -= 1;
5137           n_left_to_next -= 1;
5138
5139           b0 = vlib_get_buffer (vm, bi0);
5140           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5141           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
5142
5143           pool_foreach (i, sm->output_feature_interfaces,
5144           ({
5145             /* Only packets from NAT inside interface */
5146             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
5147               {
5148                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
5149                                     SNAT_FLAG_HAIRPINNING))
5150                   {
5151                     if (PREDICT_TRUE (sm->num_workers > 1))
5152                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5153                     else
5154                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5155                   }
5156                 break;
5157               }
5158           }));
5159
5160           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5161
5162           /* verify speculative enqueue, maybe switch current next frame */
5163           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5164                                            to_next, n_left_to_next,
5165                                            bi0, next0);
5166          }
5167
5168       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5169     }
5170
5171   vlib_node_increment_counter (vm, stats_node_index,
5172                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5173                                pkts_processed);
5174   return frame->n_vectors;
5175 }
5176
5177 static uword
5178 snat_hairpin_src_fn (vlib_main_t * vm,
5179                      vlib_node_runtime_t * node,
5180                      vlib_frame_t * frame)
5181 {
5182   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5183 }
5184
5185 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5186   .function = snat_hairpin_src_fn,
5187   .name = "nat44-hairpin-src",
5188   .vector_size = sizeof (u32),
5189   .type = VLIB_NODE_TYPE_INTERNAL,
5190   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5191   .error_strings = snat_in2out_error_strings,
5192   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5193   .next_nodes = {
5194      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5195      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5196      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5197      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5198   },
5199 };
5200
5201 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5202                               snat_hairpin_src_fn);
5203
5204 static uword
5205 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5206                          vlib_node_runtime_t * node,
5207                          vlib_frame_t * frame)
5208 {
5209   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5210 }
5211
5212 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5213   .function = nat44_ed_hairpin_src_fn,
5214   .name = "nat44-ed-hairpin-src",
5215   .vector_size = sizeof (u32),
5216   .type = VLIB_NODE_TYPE_INTERNAL,
5217   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5218   .error_strings = snat_in2out_error_strings,
5219   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5220   .next_nodes = {
5221      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5222      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5223      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5224      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5225   },
5226 };
5227
5228 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5229                               nat44_ed_hairpin_src_fn);
5230
5231 static uword
5232 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5233                                 vlib_node_runtime_t * node,
5234                                 vlib_frame_t * frame)
5235 {
5236   u32 n_left_from, * from, * to_next;
5237   snat_in2out_next_t next_index;
5238   u32 pkts_processed = 0;
5239   snat_main_t * sm = &snat_main;
5240   u32 stats_node_index;
5241
5242   stats_node_index = snat_in2out_fast_node.index;
5243
5244   from = vlib_frame_vector_args (frame);
5245   n_left_from = frame->n_vectors;
5246   next_index = node->cached_next_index;
5247
5248   while (n_left_from > 0)
5249     {
5250       u32 n_left_to_next;
5251
5252       vlib_get_next_frame (vm, node, next_index,
5253                            to_next, n_left_to_next);
5254
5255       while (n_left_from > 0 && n_left_to_next > 0)
5256         {
5257           u32 bi0;
5258           vlib_buffer_t * b0;
5259           u32 next0;
5260           u32 sw_if_index0;
5261           ip4_header_t * ip0;
5262           ip_csum_t sum0;
5263           u32 new_addr0, old_addr0;
5264           u16 old_port0, new_port0;
5265           udp_header_t * udp0;
5266           tcp_header_t * tcp0;
5267           icmp46_header_t * icmp0;
5268           snat_session_key_t key0, sm0;
5269           u32 proto0;
5270           u32 rx_fib_index0;
5271
5272           /* speculatively enqueue b0 to the current next frame */
5273           bi0 = from[0];
5274           to_next[0] = bi0;
5275           from += 1;
5276           to_next += 1;
5277           n_left_from -= 1;
5278           n_left_to_next -= 1;
5279
5280           b0 = vlib_get_buffer (vm, bi0);
5281           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5282
5283           ip0 = vlib_buffer_get_current (b0);
5284           udp0 = ip4_next_header (ip0);
5285           tcp0 = (tcp_header_t *) udp0;
5286           icmp0 = (icmp46_header_t *) udp0;
5287
5288           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5289           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5290
5291           if (PREDICT_FALSE(ip0->ttl == 1))
5292             {
5293               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5294               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5295                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
5296                                            0);
5297               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5298               goto trace0;
5299             }
5300
5301           proto0 = ip_proto_to_snat_proto (ip0->protocol);
5302
5303           if (PREDICT_FALSE (proto0 == ~0))
5304               goto trace0;
5305
5306           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5307             {
5308               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5309                                   rx_fib_index0, node, next0, ~0, 0, 0);
5310               goto trace0;
5311             }
5312
5313           key0.addr = ip0->src_address;
5314           key0.protocol = proto0;
5315           key0.port = udp0->src_port;
5316           key0.fib_index = rx_fib_index0;
5317
5318           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
5319             {
5320               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5321               next0= SNAT_IN2OUT_NEXT_DROP;
5322               goto trace0;
5323             }
5324
5325           new_addr0 = sm0.addr.as_u32;
5326           new_port0 = sm0.port;
5327           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5328           old_addr0 = ip0->src_address.as_u32;
5329           ip0->src_address.as_u32 = new_addr0;
5330
5331           sum0 = ip0->checksum;
5332           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5333                                  ip4_header_t,
5334                                  src_address /* changed member */);
5335           ip0->checksum = ip_csum_fold (sum0);
5336
5337           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5338             {
5339               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5340                 {
5341                   old_port0 = tcp0->src_port;
5342                   tcp0->src_port = new_port0;
5343
5344                   sum0 = tcp0->checksum;
5345                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5346                                          ip4_header_t,
5347                                          dst_address /* changed member */);
5348                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
5349                                          ip4_header_t /* cheat */,
5350                                          length /* changed member */);
5351                   tcp0->checksum = ip_csum_fold(sum0);
5352                 }
5353               else
5354                 {
5355                   old_port0 = udp0->src_port;
5356                   udp0->src_port = new_port0;
5357                   udp0->checksum = 0;
5358                 }
5359             }
5360           else
5361             {
5362               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5363                 {
5364                   sum0 = tcp0->checksum;
5365                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5366                                          ip4_header_t,
5367                                          dst_address /* changed member */);
5368                   tcp0->checksum = ip_csum_fold(sum0);
5369                 }
5370             }
5371
5372           /* Hairpinning */
5373           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5374
5375         trace0:
5376           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5377                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5378             {
5379               snat_in2out_trace_t *t =
5380                  vlib_add_trace (vm, node, b0, sizeof (*t));
5381               t->sw_if_index = sw_if_index0;
5382               t->next_index = next0;
5383             }
5384
5385           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5386
5387           /* verify speculative enqueue, maybe switch current next frame */
5388           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5389                                            to_next, n_left_to_next,
5390                                            bi0, next0);
5391         }
5392
5393       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5394     }
5395
5396   vlib_node_increment_counter (vm, stats_node_index,
5397                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5398                                pkts_processed);
5399   return frame->n_vectors;
5400 }
5401
5402
5403 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5404   .function = snat_in2out_fast_static_map_fn,
5405   .name = "nat44-in2out-fast",
5406   .vector_size = sizeof (u32),
5407   .format_trace = format_snat_in2out_fast_trace,
5408   .type = VLIB_NODE_TYPE_INTERNAL,
5409
5410   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5411   .error_strings = snat_in2out_error_strings,
5412
5413   .runtime_data_bytes = sizeof (snat_runtime_t),
5414
5415   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5416
5417   /* edit / add dispositions here */
5418   .next_nodes = {
5419     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5420     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5421     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5422     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5423     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5424   },
5425 };
5426
5427 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);