src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28 #include <nat/nat_inlines.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38   u32 is_slow_path;
  39 } snat_in2out_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_in2out_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  52   char * tag;
  53
  54   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  55
  56   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  57               t->sw_if_index, t->next_index, t->session_index);
  58
  59   return s;
  60 }
  61
  62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  75 {
  76   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  77   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  78   snat_in2out_worker_handoff_trace_t * t =
  79     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  80   char * m;
  81
  82   m = t->do_handoff ? "next worker" : "same worker";
  83   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  84
  85   return s;
  86 }
  87
  88 typedef struct {
  89   u32 sw_if_index;
  90   u32 next_index;
  91   u8 cached;
  92 } nat44_in2out_reass_trace_t;
  93
  94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  95 {
  96   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  97   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  98   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  99
 100   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 101               t->sw_if_index, t->next_index,
 102               t->cached ? "cached" : "translated");
 103
 104   return s;
 105 }
 106
 107 vlib_node_registration_t snat_in2out_node;
 108 vlib_node_registration_t snat_in2out_slowpath_node;
 109 vlib_node_registration_t snat_in2out_fast_node;
 110 vlib_node_registration_t snat_in2out_worker_handoff_node;
 111 vlib_node_registration_t snat_det_in2out_node;
 112 vlib_node_registration_t snat_in2out_output_node;
 113 vlib_node_registration_t snat_in2out_output_slowpath_node;
 114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 115 vlib_node_registration_t snat_hairpin_dst_node;
 116 vlib_node_registration_t snat_hairpin_src_node;
 117 vlib_node_registration_t nat44_hairpinning_node;
 118 vlib_node_registration_t nat44_in2out_reass_node;
 119 vlib_node_registration_t nat44_ed_in2out_node;
 120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
 121 vlib_node_registration_t nat44_ed_in2out_output_node;
 122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
 123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
 124 vlib_node_registration_t nat44_ed_hairpin_src_node;
 125 vlib_node_registration_t nat44_ed_hairpinning_node;
 126 vlib_node_registration_t nat44_ed_in2out_reass_node;
 127
 128 #define foreach_snat_in2out_error                       \
 129 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 130 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 131 _(OUT_OF_PORTS, "Out of ports")                         \
 132 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 133 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 134 _(NO_TRANSLATION, "No translation")                     \
 135 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 136 _(DROP_FRAGMENT, "Drop fragment")                       \
 137 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 138 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 139 _(FQ_CONGESTED, "Handoff frame queue congested")
 140
 141 typedef enum {
 142 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 143   foreach_snat_in2out_error
 144 #undef _
 145   SNAT_IN2OUT_N_ERROR,
 146 } snat_in2out_error_t;
 147
 148 static char * snat_in2out_error_strings[] = {
 149 #define _(sym,string) string,
 150   foreach_snat_in2out_error
 151 #undef _
 152 };
 153
 154 typedef enum {
 155   SNAT_IN2OUT_NEXT_LOOKUP,
 156   SNAT_IN2OUT_NEXT_DROP,
 157   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 158   SNAT_IN2OUT_NEXT_SLOW_PATH,
 159   SNAT_IN2OUT_NEXT_REASS,
 160   SNAT_IN2OUT_N_NEXT,
 161 } snat_in2out_next_t;
 162
 163 typedef enum {
 164   SNAT_HAIRPIN_SRC_NEXT_DROP,
 165   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 166   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 167   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 168   SNAT_HAIRPIN_SRC_N_NEXT,
 169 } snat_hairpin_next_t;
 170
 171 /**
 172  * @brief Check if packet should be translated
 173  *
 174  * Packets aimed at outside interface and external address with active session
 175  * should be translated.
 176  *
 177  * @param sm            NAT main
 178  * @param rt            NAT runtime data
 179  * @param sw_if_index0  index of the inside interface
 180  * @param ip0           IPv4 header
 181  * @param proto0        NAT protocol
 182  * @param rx_fib_index0 RX FIB index
 183  *
 184  * @returns 0 if packet should be translated otherwise 1
 185  */
 186 static inline int
 187 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 188                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 189                          u32 rx_fib_index0)
 190 {
 191   if (sm->out2in_dpo)
 192     return 0;
 193
 194   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 195   nat_outside_fib_t *outside_fib;
 196   fib_prefix_t pfx = {
 197     .fp_proto = FIB_PROTOCOL_IP4,
 198     .fp_len = 32,
 199     .fp_addr = {
 200         .ip4.as_u32 = ip0->dst_address.as_u32,
 201     },
 202   };
 203
 204   /* Don't NAT packet aimed at the intfc address */
 205   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 206                                       ip0->dst_address.as_u32)))
 207     return 1;
 208
 209   fei = fib_table_lookup (rx_fib_index0, &pfx);
 210   if (FIB_NODE_INDEX_INVALID != fei)
 211     {
 212       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 213       if (sw_if_index == ~0)
 214         {
 215           vec_foreach (outside_fib, sm->outside_fibs)
 216             {
 217               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 218               if (FIB_NODE_INDEX_INVALID != fei)
 219                 {
 220                   sw_if_index = fib_entry_get_resolving_interface (fei);
 221                   if (sw_if_index != ~0)
 222                     break;
 223                 }
 224             }
 225         }
 226       if (sw_if_index == ~0)
 227         return 1;
 228
 229       snat_interface_t *i;
 230       pool_foreach (i, sm->interfaces,
 231       ({
 232         /* NAT packet aimed at outside interface */
 233         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 234           return 0;
 235       }));
 236     }
 237
 238   return 1;
 239 }
 240
 241 static inline int
 242 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 243                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 244                     u32 rx_fib_index0, u32 thread_index)
 245 {
 246   udp_header_t * udp0 = ip4_next_header (ip0);
 247   snat_session_key_t key0, sm0;
 248   clib_bihash_kv_8_8_t kv0, value0;
 249
 250   key0.addr = ip0->dst_address;
 251   key0.port = udp0->dst_port;
 252   key0.protocol = proto0;
 253   key0.fib_index = sm->outside_fib_index;
 254   kv0.key = key0.as_u64;
 255
 256   /* NAT packet aimed at external address if */
 257   /* has active sessions */
 258   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 259                               &value0))
 260     {
 261       /* or is static mappings */
 262       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
 263         return 0;
 264     }
 265   else
 266     return 0;
 267
 268   if (sm->forwarding_enabled)
 269     return 1;
 270
 271   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 272                                  rx_fib_index0);
 273 }
 274
 275 static inline int
 276 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 277                                   u32 proto0, u16 src_port, u16 dst_port,
 278                                   u32 thread_index, u32 sw_if_index)
 279 {
 280   snat_session_key_t key0;
 281   clib_bihash_kv_8_8_t kv0, value0;
 282   snat_interface_t *i;
 283
 284   /* src NAT check */
 285   key0.addr = ip0->src_address;
 286   key0.port = src_port;
 287   key0.protocol = proto0;
 288   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 289   kv0.key = key0.as_u64;
 290
 291   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 292                                &value0))
 293     return 1;
 294
 295   /* dst NAT check */
 296   key0.addr = ip0->dst_address;
 297   key0.port = dst_port;
 298   key0.protocol = proto0;
 299   kv0.key = key0.as_u64;
 300   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 301                                &value0))
 302   {
 303     /* hairpinning */
 304     pool_foreach (i, sm->output_feature_interfaces,
 305     ({
 306       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 307         return 0;
 308     }));
 309     return 1;
 310   }
 311
 312   return 0;
 313 }
 314
 315 int
 316 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void * arg)
 317 {
 318   snat_main_t *sm = &snat_main;
 319   nat44_is_idle_session_ctx_t *ctx = arg;
 320   snat_session_t *s;
 321   u64 sess_timeout_time;
 322   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 323                                                        ctx->thread_index);
 324   clib_bihash_kv_8_8_t s_kv;
 325
 326   s = pool_elt_at_index (tsm->sessions, kv->value);
 327   sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
 328   if (ctx->now >= sess_timeout_time)
 329     {
 330       s_kv.key = s->out2in.as_u64;
 331       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 332         nat_log_warn ("out2in key del failed");
 333
 334       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 335                                           s->out2in.addr.as_u32,
 336                                           s->in2out.protocol,
 337                                           s->in2out.port,
 338                                           s->out2in.port,
 339                                           s->in2out.fib_index);
 340
 341       if (!snat_is_session_static (s))
 342         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 343                                             &s->out2in);
 344
 345       nat44_delete_session (sm, s, ctx->thread_index);
 346       return 1;
 347     }
 348
 349   return 0;
 350 }
 351
 352 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 353                       ip4_header_t * ip0,
 354                       u32 rx_fib_index0,
 355                       snat_session_key_t * key0,
 356                       snat_session_t ** sessionp,
 357                       vlib_node_runtime_t * node,
 358                       u32 next0,
 359                       u32 thread_index,
 360                       f64 now)
 361 {
 362   snat_user_t *u;
 363   snat_session_t *s;
 364   clib_bihash_kv_8_8_t kv0;
 365   snat_session_key_t key1;
 366   u32 address_index = ~0;
 367   udp_header_t * udp0 = ip4_next_header (ip0);
 368   u8 is_sm = 0;
 369   nat_outside_fib_t *outside_fib;
 370   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 371   fib_prefix_t pfx = {
 372     .fp_proto = FIB_PROTOCOL_IP4,
 373     .fp_len = 32,
 374     .fp_addr = {
 375         .ip4.as_u32 = ip0->dst_address.as_u32,
 376     },
 377   };
 378   nat44_is_idle_session_ctx_t ctx0;
 379
 380   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 381     {
 382       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 383       nat_ipfix_logging_max_sessions(sm->max_translations);
 384       nat_log_notice ("maximum sessions exceeded");
 385       return SNAT_IN2OUT_NEXT_DROP;
 386     }
 387
 388   key1.protocol = key0->protocol;
 389
 390   /* First try to match static mapping by local address and port */
 391   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0, 0))
 392     {
 393       /* Try to create dynamic translation */
 394       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 395                                                thread_index, &key1,
 396                                                &address_index,
 397                                                sm->port_per_thread,
 398                                                sm->per_thread_data[thread_index].snat_thread_index))
 399         {
 400           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 401           return SNAT_IN2OUT_NEXT_DROP;
 402         }
 403     }
 404   else
 405     is_sm = 1;
 406
 407   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 408                               thread_index);
 409   if (!u)
 410     {
 411       nat_log_warn ("create NAT user failed");
 412       return SNAT_IN2OUT_NEXT_DROP;
 413     }
 414
 415   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 416   if (!s)
 417     {
 418       nat44_delete_user_with_no_session (sm, u, thread_index);
 419       nat_log_warn ("create NAT session failed");
 420       return SNAT_IN2OUT_NEXT_DROP;
 421     }
 422
 423   if (is_sm)
 424     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 425   user_session_increment (sm, u, is_sm);
 426   s->outside_address_index = address_index;
 427   s->in2out = *key0;
 428   s->out2in = key1;
 429   s->out2in.protocol = key0->protocol;
 430   s->out2in.fib_index = sm->outside_fib_index;
 431   switch (vec_len (sm->outside_fibs))
 432     {
 433     case 0:
 434       s->out2in.fib_index = sm->outside_fib_index;
 435       break;
 436     case 1:
 437       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 438       break;
 439     default:
 440       vec_foreach (outside_fib, sm->outside_fibs)
 441         {
 442           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 443           if (FIB_NODE_INDEX_INVALID != fei)
 444             {
 445               if (fib_entry_get_resolving_interface (fei) != ~0)
 446                 {
 447                   s->out2in.fib_index = outside_fib->fib_index;
 448                   break;
 449                 }
 450             }
 451         }
 452       break;
 453     }
 454   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 455   s->ext_host_port = udp0->dst_port;
 456   *sessionp = s;
 457
 458   /* Add to translation hashes */
 459   ctx0.now = now;
 460   ctx0.thread_index = thread_index;
 461   kv0.key = s->in2out.as_u64;
 462   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 463   if (clib_bihash_add_or_overwrite_stale_8_8 (
 464         &sm->per_thread_data[thread_index].in2out, &kv0,
 465         nat44_i2o_is_idle_session_cb, &ctx0))
 466       nat_log_notice ("in2out key add failed");
 467
 468   kv0.key = s->out2in.as_u64;
 469   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 470
 471   if (clib_bihash_add_or_overwrite_stale_8_8 (
 472         &sm->per_thread_data[thread_index].out2in, &kv0,
 473         nat44_o2i_is_idle_session_cb, &ctx0))
 474       nat_log_notice ("out2in key add failed");
 475
 476   /* log NAT event */
 477   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 478                                       s->out2in.addr.as_u32,
 479                                       s->in2out.protocol,
 480                                       s->in2out.port,
 481                                       s->out2in.port,
 482                                       s->in2out.fib_index);
 483   return next0;
 484 }
 485
 486 static_always_inline
 487 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 488                                  snat_session_key_t *p_key0)
 489 {
 490   icmp46_header_t *icmp0;
 491   snat_session_key_t key0;
 492   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 493   ip4_header_t *inner_ip0 = 0;
 494   void *l4_header = 0;
 495   icmp46_header_t *inner_icmp0;
 496
 497   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 498   echo0 = (icmp_echo_header_t *)(icmp0+1);
 499
 500   if (!icmp_is_error_message (icmp0))
 501     {
 502       key0.protocol = SNAT_PROTOCOL_ICMP;
 503       key0.addr = ip0->src_address;
 504       key0.port = echo0->identifier;
 505     }
 506   else
 507     {
 508       inner_ip0 = (ip4_header_t *)(echo0+1);
 509       l4_header = ip4_next_header (inner_ip0);
 510       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 511       key0.addr = inner_ip0->dst_address;
 512       switch (key0.protocol)
 513         {
 514         case SNAT_PROTOCOL_ICMP:
 515           inner_icmp0 = (icmp46_header_t*)l4_header;
 516           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 517           key0.port = inner_echo0->identifier;
 518           break;
 519         case SNAT_PROTOCOL_UDP:
 520         case SNAT_PROTOCOL_TCP:
 521           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 522           break;
 523         default:
 524           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 525         }
 526     }
 527   *p_key0 = key0;
 528   return -1; /* success */
 529 }
 530
 531 /**
 532  * Get address and port values to be used for ICMP packet translation
 533  * and create session if needed
 534  *
 535  * @param[in,out] sm             NAT main
 536  * @param[in,out] node           NAT node runtime
 537  * @param[in] thread_index       thread index
 538  * @param[in,out] b0             buffer containing packet to be translated
 539  * @param[out] p_proto           protocol used for matching
 540  * @param[out] p_value           address and port after NAT translation
 541  * @param[out] p_dont_translate  if packet should not be translated
 542  * @param d                      optional parameter
 543  * @param e                      optional parameter
 544  */
 545 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 546                            u32 thread_index, vlib_buffer_t *b0,
 547                            ip4_header_t *ip0, u8 *p_proto,
 548                            snat_session_key_t *p_value,
 549                            u8 *p_dont_translate, void *d, void *e)
 550 {
 551   icmp46_header_t *icmp0;
 552   u32 sw_if_index0;
 553   u32 rx_fib_index0;
 554   snat_session_key_t key0;
 555   snat_session_t *s0 = 0;
 556   u8 dont_translate = 0;
 557   clib_bihash_kv_8_8_t kv0, value0;
 558   u32 next0 = ~0;
 559   int err;
 560
 561   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 562   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 563   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 564
 565   err = icmp_get_key (ip0, &key0);
 566   if (err != -1)
 567     {
 568       b0->error = node->errors[err];
 569       next0 = SNAT_IN2OUT_NEXT_DROP;
 570       goto out;
 571     }
 572   key0.fib_index = rx_fib_index0;
 573
 574   kv0.key = key0.as_u64;
 575
 576   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 577                               &value0))
 578     {
 579       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 580         {
 581           if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
 582               key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
 583             {
 584               dont_translate = 1;
 585               goto out;
 586             }
 587         }
 588       else
 589         {
 590           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 591               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 592             {
 593               dont_translate = 1;
 594               goto out;
 595             }
 596         }
 597
 598       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 599         {
 600           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 601           next0 = SNAT_IN2OUT_NEXT_DROP;
 602           goto out;
 603         }
 604
 605       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 606                          thread_index, vlib_time_now (sm->vlib_main));
 607
 608       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 609         goto out;
 610     }
 611   else
 612     {
 613       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 614                         icmp0->type != ICMP4_echo_reply &&
 615                         !icmp_is_error_message (icmp0)))
 616         {
 617           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 618           next0 = SNAT_IN2OUT_NEXT_DROP;
 619           goto out;
 620         }
 621
 622       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 623                               value0.value);
 624     }
 625
 626 out:
 627   *p_proto = key0.protocol;
 628   if (s0)
 629     *p_value = s0->out2in;
 630   *p_dont_translate = dont_translate;
 631   if (d)
 632     *(snat_session_t**)d = s0;
 633   return next0;
 634 }
 635
 636 /**
 637  * Get address and port values to be used for ICMP packet translation
 638  *
 639  * @param[in] sm                 NAT main
 640  * @param[in,out] node           NAT node runtime
 641  * @param[in] thread_index       thread index
 642  * @param[in,out] b0             buffer containing packet to be translated
 643  * @param[out] p_proto           protocol used for matching
 644  * @param[out] p_value           address and port after NAT translation
 645  * @param[out] p_dont_translate  if packet should not be translated
 646  * @param d                      optional parameter
 647  * @param e                      optional parameter
 648  */
 649 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 650                            u32 thread_index, vlib_buffer_t *b0,
 651                            ip4_header_t *ip0, u8 *p_proto,
 652                            snat_session_key_t *p_value,
 653                            u8 *p_dont_translate, void *d, void *e)
 654 {
 655   icmp46_header_t *icmp0;
 656   u32 sw_if_index0;
 657   u32 rx_fib_index0;
 658   snat_session_key_t key0;
 659   snat_session_key_t sm0;
 660   u8 dont_translate = 0;
 661   u8 is_addr_only;
 662   u32 next0 = ~0;
 663   int err;
 664
 665   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 666   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 667   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 668
 669   err = icmp_get_key (ip0, &key0);
 670   if (err != -1)
 671     {
 672       b0->error = node->errors[err];
 673       next0 = SNAT_IN2OUT_NEXT_DROP;
 674       goto out2;
 675     }
 676   key0.fib_index = rx_fib_index0;
 677
 678   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0))
 679     {
 680       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 681           IP_PROTOCOL_ICMP, rx_fib_index0)))
 682         {
 683           dont_translate = 1;
 684           goto out;
 685         }
 686
 687       if (icmp_is_error_message (icmp0))
 688         {
 689           next0 = SNAT_IN2OUT_NEXT_DROP;
 690           goto out;
 691         }
 692
 693       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 694       next0 = SNAT_IN2OUT_NEXT_DROP;
 695       goto out;
 696     }
 697
 698   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 699                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 700                     !icmp_is_error_message (icmp0)))
 701     {
 702       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 703       next0 = SNAT_IN2OUT_NEXT_DROP;
 704       goto out;
 705     }
 706
 707 out:
 708   *p_value = sm0;
 709 out2:
 710   *p_proto = key0.protocol;
 711   *p_dont_translate = dont_translate;
 712   return next0;
 713 }
 714
 715 static inline u32
 716 snat_icmp_hairpinning (snat_main_t *sm,
 717                        vlib_buffer_t * b0,
 718                        ip4_header_t * ip0,
 719                        icmp46_header_t * icmp0,
 720                        int is_ed);
 721
 722 static inline u32 icmp_in2out (snat_main_t *sm,
 723                                vlib_buffer_t * b0,
 724                                ip4_header_t * ip0,
 725                                icmp46_header_t * icmp0,
 726                                u32 sw_if_index0,
 727                                u32 rx_fib_index0,
 728                                vlib_node_runtime_t * node,
 729                                u32 next0,
 730                                u32 thread_index,
 731                                void *d,
 732                                void *e)
 733 {
 734   snat_session_key_t sm0;
 735   u8 protocol;
 736   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 737   ip4_header_t *inner_ip0;
 738   void *l4_header = 0;
 739   icmp46_header_t *inner_icmp0;
 740   u8 dont_translate;
 741   u32 new_addr0, old_addr0;
 742   u16 old_id0, new_id0;
 743   u16 old_checksum0, new_checksum0;
 744   ip_csum_t sum0;
 745   u16 checksum0;
 746   u32 next0_tmp;
 747
 748   echo0 = (icmp_echo_header_t *)(icmp0+1);
 749
 750   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 751                                        &protocol, &sm0, &dont_translate, d, e);
 752   if (next0_tmp != ~0)
 753     next0 = next0_tmp;
 754   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 755     goto out;
 756
 757   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 758     {
 759       sum0 = ip_incremental_checksum (0, icmp0,
 760                                       ntohs(ip0->length) - ip4_header_bytes (ip0));
 761       checksum0 = ~ip_csum_fold (sum0);
 762       if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 763         {
 764           next0 = SNAT_IN2OUT_NEXT_DROP;
 765           goto out;
 766         }
 767     }
 768
 769   old_addr0 = ip0->src_address.as_u32;
 770   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 771
 772   sum0 = ip0->checksum;
 773   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 774                          src_address /* changed member */);
 775   ip0->checksum = ip_csum_fold (sum0);
 776
 777   if (icmp0->checksum == 0)
 778     icmp0->checksum = 0xffff;
 779
 780   if (!icmp_is_error_message (icmp0))
 781     {
 782       new_id0 = sm0.port;
 783       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 784         {
 785           old_id0 = echo0->identifier;
 786           new_id0 = sm0.port;
 787           echo0->identifier = new_id0;
 788
 789           sum0 = icmp0->checksum;
 790           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 791                                  identifier);
 792           icmp0->checksum = ip_csum_fold (sum0);
 793         }
 794     }
 795   else
 796     {
 797       inner_ip0 = (ip4_header_t *)(echo0+1);
 798       l4_header = ip4_next_header (inner_ip0);
 799
 800       if (!ip4_header_checksum_is_valid (inner_ip0))
 801         {
 802           next0 = SNAT_IN2OUT_NEXT_DROP;
 803           goto out;
 804         }
 805
 806       /* update inner destination IP address */
 807       old_addr0 = inner_ip0->dst_address.as_u32;
 808       inner_ip0->dst_address = sm0.addr;
 809       new_addr0 = inner_ip0->dst_address.as_u32;
 810       sum0 = icmp0->checksum;
 811       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 812                              dst_address /* changed member */);
 813       icmp0->checksum = ip_csum_fold (sum0);
 814
 815       /* update inner IP header checksum */
 816       old_checksum0 = inner_ip0->checksum;
 817       sum0 = inner_ip0->checksum;
 818       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 819                              dst_address /* changed member */);
 820       inner_ip0->checksum = ip_csum_fold (sum0);
 821       new_checksum0 = inner_ip0->checksum;
 822       sum0 = icmp0->checksum;
 823       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 824                              checksum);
 825       icmp0->checksum = ip_csum_fold (sum0);
 826
 827       switch (protocol)
 828         {
 829           case SNAT_PROTOCOL_ICMP:
 830             inner_icmp0 = (icmp46_header_t*)l4_header;
 831             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 832
 833             old_id0 = inner_echo0->identifier;
 834             new_id0 = sm0.port;
 835             inner_echo0->identifier = new_id0;
 836
 837             sum0 = icmp0->checksum;
 838             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 839                                    identifier);
 840             icmp0->checksum = ip_csum_fold (sum0);
 841             break;
 842           case SNAT_PROTOCOL_UDP:
 843           case SNAT_PROTOCOL_TCP:
 844             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 845             new_id0 = sm0.port;
 846             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 847
 848             sum0 = icmp0->checksum;
 849             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 850                                    dst_port);
 851             icmp0->checksum = ip_csum_fold (sum0);
 852             break;
 853           default:
 854             ASSERT(0);
 855         }
 856     }
 857
 858   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 859     {
 860       if (sm->deterministic ||
 861           0 != snat_icmp_hairpinning(sm, b0, ip0, icmp0,
 862                                      sm->endpoint_dependent))
 863         vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 864     }
 865
 866 out:
 867   return next0;
 868 }
 869
 870 /**
 871  * @brief Hairpinning
 872  *
 873  * Hairpinning allows two endpoints on the internal side of the NAT to
 874  * communicate even if they only use each other's external IP addresses
 875  * and ports.
 876  *
 877  * @param sm     NAT main.
 878  * @param b0     Vlib buffer.
 879  * @param ip0    IP header.
 880  * @param udp0   UDP header.
 881  * @param tcp0   TCP header.
 882  * @param proto0 NAT protocol.
 883  */
 884 static inline int
 885 snat_hairpinning (snat_main_t *sm,
 886                   vlib_buffer_t * b0,
 887                   ip4_header_t * ip0,
 888                   udp_header_t * udp0,
 889                   tcp_header_t * tcp0,
 890                   u32 proto0,
 891                   int is_ed)
 892 {
 893   snat_session_key_t key0, sm0;
 894   snat_session_t * s0;
 895   clib_bihash_kv_8_8_t kv0, value0;
 896   ip_csum_t sum0;
 897   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 898   u16 new_dst_port0, old_dst_port0;
 899   int rv;
 900
 901   key0.addr = ip0->dst_address;
 902   key0.port = udp0->dst_port;
 903   key0.protocol = proto0;
 904   key0.fib_index = sm->outside_fib_index;
 905   kv0.key = key0.as_u64;
 906
 907   /* Check if destination is static mappings */
 908   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
 909     {
 910       new_dst_addr0 = sm0.addr.as_u32;
 911       new_dst_port0 = sm0.port;
 912       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 913     }
 914   /* or active session */
 915   else
 916     {
 917       if (sm->num_workers > 1)
 918         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 919       else
 920         ti = sm->num_workers;
 921
 922       if (is_ed)
 923         {
 924           clib_bihash_kv_16_8_t ed_kv, ed_value;
 925           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 926                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 927                       udp0->src_port);
 928           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 929                                         &ed_kv, &ed_value);
 930           si = ed_value.value;
 931         }
 932       else
 933         {
 934           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 935                                        &value0);
 936           si = value0.value;
 937         }
 938       if (rv)
 939         return 0;
 940
 941       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 942       new_dst_addr0 = s0->in2out.addr.as_u32;
 943       new_dst_port0 = s0->in2out.port;
 944       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 945     }
 946
 947   /* Destination is behind the same NAT, use internal address and port */
 948   if (new_dst_addr0)
 949     {
 950       old_dst_addr0 = ip0->dst_address.as_u32;
 951       ip0->dst_address.as_u32 = new_dst_addr0;
 952       sum0 = ip0->checksum;
 953       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 954                              ip4_header_t, dst_address);
 955       ip0->checksum = ip_csum_fold (sum0);
 956
 957       old_dst_port0 = tcp0->dst;
 958       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 959         {
 960           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 961             {
 962               tcp0->dst = new_dst_port0;
 963               sum0 = tcp0->checksum;
 964               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 965                                      ip4_header_t, dst_address);
 966               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 967                                      ip4_header_t /* cheat */, length);
 968               tcp0->checksum = ip_csum_fold(sum0);
 969             }
 970           else
 971             {
 972               udp0->dst_port = new_dst_port0;
 973               udp0->checksum = 0;
 974             }
 975         }
 976       else
 977         {
 978           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 979             {
 980               sum0 = tcp0->checksum;
 981               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 982                                      ip4_header_t, dst_address);
 983               tcp0->checksum = ip_csum_fold(sum0);
 984             }
 985         }
 986       return 1;
 987     }
 988   return 0;
 989 }
 990
 991 static inline u32
 992 snat_icmp_hairpinning (snat_main_t *sm,
 993                        vlib_buffer_t * b0,
 994                        ip4_header_t * ip0,
 995                        icmp46_header_t * icmp0,
 996                        int is_ed)
 997 {
 998   snat_session_key_t key0;
 999   clib_bihash_kv_8_8_t kv0, value0;
1000   u32 old_dst_addr0, new_dst_addr0;
1001   u32 old_addr0, new_addr0;
1002   u16 old_port0, new_port0;
1003   u16 old_checksum0, new_checksum0;
1004   u32 si, ti = 0;
1005   ip_csum_t sum0;
1006   snat_session_t *s0;
1007   snat_static_mapping_t *m0;
1008
1009   if (icmp_is_error_message (icmp0))
1010     {
1011       ip4_header_t *inner_ip0 = 0;
1012       tcp_udp_header_t *l4_header = 0;
1013
1014       inner_ip0 = (ip4_header_t *)((icmp_echo_header_t *)(icmp0+1)+1);
1015       l4_header = ip4_next_header (inner_ip0);
1016       u32 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
1017
1018       if (protocol != SNAT_PROTOCOL_TCP && protocol != SNAT_PROTOCOL_UDP)
1019         return 1;
1020
1021       if (is_ed)
1022         {
1023           clib_bihash_kv_16_8_t ed_kv, ed_value;
1024           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
1025                       inner_ip0->protocol, sm->outside_fib_index,
1026                       l4_header->src_port, l4_header->dst_port);
1027           if (clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
1028                                        &ed_kv, &ed_value))
1029             return 1;
1030           si = ed_value.value;
1031         }
1032       else
1033         {
1034           key0.addr = ip0->dst_address;
1035           key0.port = l4_header->src_port;
1036           key0.protocol = protocol;
1037           key0.fib_index = sm->outside_fib_index;
1038           kv0.key = key0.as_u64;
1039           if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1040                                       &value0))
1041             return 1;
1042           si = value0.value;
1043         }
1044       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1045       new_dst_addr0 = s0->in2out.addr.as_u32;
1046       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1047
1048       /* update inner source IP address */
1049       old_addr0 = inner_ip0->src_address.as_u32;
1050       inner_ip0->src_address.as_u32 = new_dst_addr0;
1051       new_addr0 = inner_ip0->src_address.as_u32;
1052       sum0 = icmp0->checksum;
1053       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1054                              src_address);
1055       icmp0->checksum = ip_csum_fold (sum0);
1056
1057       /* update inner IP header checksum */
1058       old_checksum0 = inner_ip0->checksum;
1059       sum0 = inner_ip0->checksum;
1060       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1061                              src_address);
1062       inner_ip0->checksum = ip_csum_fold (sum0);
1063       new_checksum0 = inner_ip0->checksum;
1064       sum0 = icmp0->checksum;
1065       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
1066                              checksum);
1067       icmp0->checksum = ip_csum_fold (sum0);
1068
1069       /* update inner source port */
1070       old_port0 = l4_header->src_port;
1071       l4_header->src_port = s0->in2out.port;
1072       new_port0 = l4_header->src_port;
1073       sum0 = icmp0->checksum;
1074       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
1075                              src_port);
1076       icmp0->checksum = ip_csum_fold (sum0);
1077     }
1078   else
1079     {
1080       if (!is_ed)
1081         {
1082           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1083           u16 icmp_id0 = echo0->identifier;
1084           key0.addr = ip0->dst_address;
1085           key0.port = icmp_id0;
1086           key0.protocol = SNAT_PROTOCOL_ICMP;
1087           key0.fib_index = sm->outside_fib_index;
1088           kv0.key = key0.as_u64;
1089           if (sm->num_workers > 1)
1090             ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1091           else
1092             ti = sm->num_workers;
1093           int rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1094                                            &value0);
1095           if (!rv)
1096             {
1097               si = value0.value;
1098               s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1099               new_dst_addr0 = s0->in2out.addr.as_u32;
1100               vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1101               echo0->identifier = s0->in2out.port;
1102               sum0 = icmp0->checksum;
1103               sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1104                                      icmp_echo_header_t, identifier);
1105               icmp0->checksum = ip_csum_fold (sum0);
1106               goto change_addr;
1107             }
1108           ti = 0;
1109         }
1110
1111       key0.addr = ip0->dst_address;
1112       key0.port = 0;
1113       key0.protocol = 0;
1114       key0.fib_index = sm->outside_fib_index;
1115       kv0.key = key0.as_u64;
1116
1117       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
1118         return 1;
1119
1120       m0 = pool_elt_at_index (sm->static_mappings, value0.value);
1121
1122       new_dst_addr0 = m0->local_addr.as_u32;
1123       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
1124         vnet_buffer(b0)->sw_if_index[VLIB_TX] = m0->fib_index;
1125     }
1126 change_addr:
1127   /* Destination is behind the same NAT, use internal address and port */
1128   if (new_dst_addr0)
1129     {
1130       old_dst_addr0 = ip0->dst_address.as_u32;
1131       ip0->dst_address.as_u32 = new_dst_addr0;
1132       sum0 = ip0->checksum;
1133       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1134                              ip4_header_t, dst_address);
1135       ip0->checksum = ip_csum_fold (sum0);
1136     }
1137   return 0;
1138 }
1139
1140 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1141                                          vlib_buffer_t * b0,
1142                                          ip4_header_t * ip0,
1143                                          icmp46_header_t * icmp0,
1144                                          u32 sw_if_index0,
1145                                          u32 rx_fib_index0,
1146                                          vlib_node_runtime_t * node,
1147                                          u32 next0,
1148                                          f64 now,
1149                                          u32 thread_index,
1150                                          snat_session_t ** p_s0)
1151 {
1152   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1153                       next0, thread_index, p_s0, 0);
1154   snat_session_t * s0 = *p_s0;
1155   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1156     {
1157       /* Accounting */
1158       nat44_session_update_counters (s0, now,
1159                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
1160       /* Per-user LRU list maintenance */
1161       nat44_session_update_lru (sm, s0, thread_index);
1162     }
1163   return next0;
1164 }
1165
1166 static inline void
1167 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1168                                   vlib_buffer_t * b,
1169                                   ip4_header_t * ip)
1170 {
1171   clib_bihash_kv_8_8_t kv, value;
1172   snat_static_mapping_t *m;
1173   u32 old_addr, new_addr;
1174   ip_csum_t sum;
1175
1176   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1177   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1178     return;
1179
1180   m = pool_elt_at_index (sm->static_mappings, value.value);
1181
1182   old_addr = ip->dst_address.as_u32;
1183   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1184   sum = ip->checksum;
1185   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1186   ip->checksum = ip_csum_fold (sum);
1187
1188   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1189     vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1190 }
1191
1192 static int
1193 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1194                              vlib_buffer_t * b,
1195                              ip4_header_t * ip,
1196                              u32 rx_fib_index)
1197 {
1198   clib_bihash_kv_8_8_t kv, value;
1199   snat_static_mapping_t *m;
1200   snat_session_key_t m_key;
1201   u32 old_addr, new_addr;
1202   ip_csum_t sum;
1203
1204   m_key.addr = ip->src_address;
1205   m_key.port = 0;
1206   m_key.protocol = 0;
1207   m_key.fib_index = rx_fib_index;
1208   kv.key = m_key.as_u64;
1209   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1210     return 1;
1211
1212   m = pool_elt_at_index (sm->static_mappings, value.value);
1213
1214   old_addr = ip->src_address.as_u32;
1215   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1216   sum = ip->checksum;
1217   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1218   ip->checksum = ip_csum_fold (sum);
1219
1220
1221   /* Hairpinning */
1222   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1223     {
1224       vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1225       nat_hairpinning_sm_unknown_proto (sm, b, ip);
1226     }
1227
1228   return 0;
1229 }
1230
1231 static inline uword
1232 snat_in2out_node_fn_inline (vlib_main_t * vm,
1233                             vlib_node_runtime_t * node,
1234                             vlib_frame_t * frame, int is_slow_path,
1235                             int is_output_feature)
1236 {
1237   u32 n_left_from, * from, * to_next;
1238   snat_in2out_next_t next_index;
1239   u32 pkts_processed = 0;
1240   snat_main_t * sm = &snat_main;
1241   f64 now = vlib_time_now (vm);
1242   u32 stats_node_index;
1243   u32 thread_index = vm->thread_index;
1244
1245   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1246     snat_in2out_node.index;
1247
1248   from = vlib_frame_vector_args (frame);
1249   n_left_from = frame->n_vectors;
1250   next_index = node->cached_next_index;
1251
1252   while (n_left_from > 0)
1253     {
1254       u32 n_left_to_next;
1255
1256       vlib_get_next_frame (vm, node, next_index,
1257                            to_next, n_left_to_next);
1258
1259       while (n_left_from >= 4 && n_left_to_next >= 2)
1260         {
1261           u32 bi0, bi1;
1262           vlib_buffer_t * b0, * b1;
1263           u32 next0, next1;
1264           u32 sw_if_index0, sw_if_index1;
1265           ip4_header_t * ip0, * ip1;
1266           ip_csum_t sum0, sum1;
1267           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1268           u16 old_port0, new_port0, old_port1, new_port1;
1269           udp_header_t * udp0, * udp1;
1270           tcp_header_t * tcp0, * tcp1;
1271           icmp46_header_t * icmp0, * icmp1;
1272           snat_session_key_t key0, key1;
1273           u32 rx_fib_index0, rx_fib_index1;
1274           u32 proto0, proto1;
1275           snat_session_t * s0 = 0, * s1 = 0;
1276           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1277           u32 iph_offset0 = 0, iph_offset1 = 0;
1278
1279           /* Prefetch next iteration. */
1280           {
1281             vlib_buffer_t * p2, * p3;
1282
1283             p2 = vlib_get_buffer (vm, from[2]);
1284             p3 = vlib_get_buffer (vm, from[3]);
1285
1286             vlib_prefetch_buffer_header (p2, LOAD);
1287             vlib_prefetch_buffer_header (p3, LOAD);
1288
1289             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1290             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1291           }
1292
1293           /* speculatively enqueue b0 and b1 to the current next frame */
1294           to_next[0] = bi0 = from[0];
1295           to_next[1] = bi1 = from[1];
1296           from += 2;
1297           to_next += 2;
1298           n_left_from -= 2;
1299           n_left_to_next -= 2;
1300
1301           b0 = vlib_get_buffer (vm, bi0);
1302           b1 = vlib_get_buffer (vm, bi1);
1303
1304           if (is_output_feature)
1305             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1306
1307           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1308                  iph_offset0);
1309
1310           udp0 = ip4_next_header (ip0);
1311           tcp0 = (tcp_header_t *) udp0;
1312           icmp0 = (icmp46_header_t *) udp0;
1313
1314           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1315           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1316                                    sw_if_index0);
1317
1318           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1319
1320           if (PREDICT_FALSE(ip0->ttl == 1))
1321             {
1322               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1323               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1324                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1325                                            0);
1326               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1327               goto trace00;
1328             }
1329
1330           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1331
1332           /* Next configured feature, probably ip4-lookup */
1333           if (is_slow_path)
1334             {
1335               if (PREDICT_FALSE (proto0 == ~0))
1336                 {
1337                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1338                     {
1339                       next0 = SNAT_IN2OUT_NEXT_DROP;
1340                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1341                     }
1342                   goto trace00;
1343                 }
1344
1345               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1346                 {
1347                   next0 = icmp_in2out_slow_path
1348                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1349                      node, next0, now, thread_index, &s0);
1350                   goto trace00;
1351                 }
1352             }
1353           else
1354             {
1355               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1356                 {
1357                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1358                   goto trace00;
1359                 }
1360
1361               if (ip4_is_fragment (ip0))
1362                 {
1363                   next0 = SNAT_IN2OUT_NEXT_REASS;
1364                   goto trace00;
1365                 }
1366             }
1367
1368           key0.addr = ip0->src_address;
1369           key0.port = udp0->src_port;
1370           key0.protocol = proto0;
1371           key0.fib_index = rx_fib_index0;
1372
1373           kv0.key = key0.as_u64;
1374
1375           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1376               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1377             {
1378               if (is_slow_path)
1379                 {
1380                   if (is_output_feature)
1381                     {
1382                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1383                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1384                         goto trace00;
1385                     }
1386                   else
1387                     {
1388                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1389                           ip0, proto0, rx_fib_index0, thread_index)))
1390                         goto trace00;
1391                     }
1392
1393                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1394                                      &s0, node, next0, thread_index, now);
1395                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1396                     goto trace00;
1397                 }
1398               else
1399                 {
1400                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1401                   goto trace00;
1402                 }
1403             }
1404           else
1405             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1406                                     value0.value);
1407
1408           b0->flags |= VNET_BUFFER_F_IS_NATED;
1409
1410           old_addr0 = ip0->src_address.as_u32;
1411           ip0->src_address = s0->out2in.addr;
1412           new_addr0 = ip0->src_address.as_u32;
1413           if (!is_output_feature)
1414             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1415
1416           sum0 = ip0->checksum;
1417           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1418                                  ip4_header_t,
1419                                  src_address /* changed member */);
1420           ip0->checksum = ip_csum_fold (sum0);
1421
1422           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1423             {
1424               old_port0 = tcp0->src_port;
1425               tcp0->src_port = s0->out2in.port;
1426               new_port0 = tcp0->src_port;
1427
1428               sum0 = tcp0->checksum;
1429               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1430                                      ip4_header_t,
1431                                      dst_address /* changed member */);
1432               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1433                                      ip4_header_t /* cheat */,
1434                                      length /* changed member */);
1435               mss_clamping (sm, tcp0, &sum0);
1436               tcp0->checksum = ip_csum_fold(sum0);
1437             }
1438           else
1439             {
1440               old_port0 = udp0->src_port;
1441               udp0->src_port = s0->out2in.port;
1442               udp0->checksum = 0;
1443             }
1444
1445           /* Accounting */
1446           nat44_session_update_counters (s0, now,
1447                                          vlib_buffer_length_in_chain (vm, b0));
1448           /* Per-user LRU list maintenance */
1449           nat44_session_update_lru (sm, s0, thread_index);
1450         trace00:
1451
1452           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1453                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1454             {
1455               snat_in2out_trace_t *t =
1456                  vlib_add_trace (vm, node, b0, sizeof (*t));
1457               t->is_slow_path = is_slow_path;
1458               t->sw_if_index = sw_if_index0;
1459               t->next_index = next0;
1460                   t->session_index = ~0;
1461               if (s0)
1462                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1463             }
1464
1465           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1466
1467           if (is_output_feature)
1468             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1469
1470           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1471                  iph_offset1);
1472
1473           udp1 = ip4_next_header (ip1);
1474           tcp1 = (tcp_header_t *) udp1;
1475           icmp1 = (icmp46_header_t *) udp1;
1476
1477           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1478           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1479                                    sw_if_index1);
1480
1481           if (PREDICT_FALSE(ip1->ttl == 1))
1482             {
1483               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1484               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1485                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1486                                            0);
1487               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1488               goto trace01;
1489             }
1490
1491           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1492
1493           /* Next configured feature, probably ip4-lookup */
1494           if (is_slow_path)
1495             {
1496               if (PREDICT_FALSE (proto1 == ~0))
1497                 {
1498                   if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1499                     {
1500                       next1 = SNAT_IN2OUT_NEXT_DROP;
1501                       b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1502                     }
1503                   goto trace01;
1504                 }
1505
1506               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1507                 {
1508                   next1 = icmp_in2out_slow_path
1509                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1510                      next1, now, thread_index, &s1);
1511                   goto trace01;
1512                 }
1513             }
1514           else
1515             {
1516               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1517                 {
1518                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1519                   goto trace01;
1520                 }
1521
1522               if (ip4_is_fragment (ip1))
1523                 {
1524                   next1 = SNAT_IN2OUT_NEXT_REASS;
1525                   goto trace01;
1526                 }
1527             }
1528
1529           key1.addr = ip1->src_address;
1530           key1.port = udp1->src_port;
1531           key1.protocol = proto1;
1532           key1.fib_index = rx_fib_index1;
1533
1534           kv1.key = key1.as_u64;
1535
1536             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1537                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1538             {
1539               if (is_slow_path)
1540                 {
1541                   if (is_output_feature)
1542                     {
1543                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1544                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1545                         goto trace01;
1546                     }
1547                   else
1548                     {
1549                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1550                           ip1, proto1, rx_fib_index1, thread_index)))
1551                         goto trace01;
1552                     }
1553
1554                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1555                                      &s1, node, next1, thread_index, now);
1556                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1557                     goto trace01;
1558                 }
1559               else
1560                 {
1561                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1562                   goto trace01;
1563                 }
1564             }
1565           else
1566             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1567                                     value1.value);
1568
1569           b1->flags |= VNET_BUFFER_F_IS_NATED;
1570
1571           old_addr1 = ip1->src_address.as_u32;
1572           ip1->src_address = s1->out2in.addr;
1573           new_addr1 = ip1->src_address.as_u32;
1574           if (!is_output_feature)
1575             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1576
1577           sum1 = ip1->checksum;
1578           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1579                                  ip4_header_t,
1580                                  src_address /* changed member */);
1581           ip1->checksum = ip_csum_fold (sum1);
1582
1583           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1584             {
1585               old_port1 = tcp1->src_port;
1586               tcp1->src_port = s1->out2in.port;
1587               new_port1 = tcp1->src_port;
1588
1589               sum1 = tcp1->checksum;
1590               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1591                                      ip4_header_t,
1592                                      dst_address /* changed member */);
1593               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1594                                      ip4_header_t /* cheat */,
1595                                      length /* changed member */);
1596               mss_clamping (sm, tcp1, &sum1);
1597               tcp1->checksum = ip_csum_fold(sum1);
1598             }
1599           else
1600             {
1601               old_port1 = udp1->src_port;
1602               udp1->src_port = s1->out2in.port;
1603               udp1->checksum = 0;
1604             }
1605
1606           /* Accounting */
1607           nat44_session_update_counters (s1, now,
1608                                          vlib_buffer_length_in_chain (vm, b1));
1609           /* Per-user LRU list maintenance */
1610           nat44_session_update_lru (sm, s1, thread_index);
1611         trace01:
1612
1613           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1614                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1615             {
1616               snat_in2out_trace_t *t =
1617                  vlib_add_trace (vm, node, b1, sizeof (*t));
1618               t->sw_if_index = sw_if_index1;
1619               t->next_index = next1;
1620               t->session_index = ~0;
1621               if (s1)
1622                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1623             }
1624
1625           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1626
1627           /* verify speculative enqueues, maybe switch current next frame */
1628           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1629                                            to_next, n_left_to_next,
1630                                            bi0, bi1, next0, next1);
1631         }
1632
1633       while (n_left_from > 0 && n_left_to_next > 0)
1634         {
1635           u32 bi0;
1636           vlib_buffer_t * b0;
1637           u32 next0;
1638           u32 sw_if_index0;
1639           ip4_header_t * ip0;
1640           ip_csum_t sum0;
1641           u32 new_addr0, old_addr0;
1642           u16 old_port0, new_port0;
1643           udp_header_t * udp0;
1644           tcp_header_t * tcp0;
1645           icmp46_header_t * icmp0;
1646           snat_session_key_t key0;
1647           u32 rx_fib_index0;
1648           u32 proto0;
1649           snat_session_t * s0 = 0;
1650           clib_bihash_kv_8_8_t kv0, value0;
1651           u32 iph_offset0 = 0;
1652
1653           /* speculatively enqueue b0 to the current next frame */
1654           bi0 = from[0];
1655           to_next[0] = bi0;
1656           from += 1;
1657           to_next += 1;
1658           n_left_from -= 1;
1659           n_left_to_next -= 1;
1660
1661           b0 = vlib_get_buffer (vm, bi0);
1662           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1663
1664           if (is_output_feature)
1665             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1666
1667           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1668                  iph_offset0);
1669
1670           udp0 = ip4_next_header (ip0);
1671           tcp0 = (tcp_header_t *) udp0;
1672           icmp0 = (icmp46_header_t *) udp0;
1673
1674           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1675           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1676                                    sw_if_index0);
1677
1678           if (PREDICT_FALSE(ip0->ttl == 1))
1679             {
1680               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1681               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1682                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1683                                            0);
1684               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1685               goto trace0;
1686             }
1687
1688           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1689
1690           /* Next configured feature, probably ip4-lookup */
1691           if (is_slow_path)
1692             {
1693               if (PREDICT_FALSE (proto0 == ~0))
1694                 {
1695                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1696                     {
1697                       next0 = SNAT_IN2OUT_NEXT_DROP;
1698                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1699                     }
1700                   goto trace0;
1701                 }
1702
1703               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1704                 {
1705                   next0 = icmp_in2out_slow_path
1706                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1707                      next0, now, thread_index, &s0);
1708                   goto trace0;
1709                 }
1710             }
1711           else
1712             {
1713               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1714                 {
1715                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1716                   goto trace0;
1717                 }
1718
1719               if (ip4_is_fragment (ip0))
1720                 {
1721                   next0 = SNAT_IN2OUT_NEXT_REASS;
1722                   goto trace0;
1723                 }
1724             }
1725
1726           key0.addr = ip0->src_address;
1727           key0.port = udp0->src_port;
1728           key0.protocol = proto0;
1729           key0.fib_index = rx_fib_index0;
1730
1731           kv0.key = key0.as_u64;
1732
1733           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1734                                       &kv0, &value0))
1735             {
1736               if (is_slow_path)
1737                 {
1738                   if (is_output_feature)
1739                     {
1740                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1741                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1742                         goto trace0;
1743                     }
1744                   else
1745                     {
1746                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1747                           ip0, proto0, rx_fib_index0, thread_index)))
1748                         goto trace0;
1749                     }
1750
1751                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1752                                      &s0, node, next0, thread_index, now);
1753
1754                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1755                     goto trace0;
1756                 }
1757               else
1758                 {
1759                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1760                   goto trace0;
1761                 }
1762             }
1763           else
1764           s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1765                                   value0.value);
1766
1767           b0->flags |= VNET_BUFFER_F_IS_NATED;
1768
1769           old_addr0 = ip0->src_address.as_u32;
1770           ip0->src_address = s0->out2in.addr;
1771           new_addr0 = ip0->src_address.as_u32;
1772           if (!is_output_feature)
1773             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1774
1775           sum0 = ip0->checksum;
1776           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1777                                  ip4_header_t,
1778                                  src_address /* changed member */);
1779           ip0->checksum = ip_csum_fold (sum0);
1780
1781           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1782             {
1783               old_port0 = tcp0->src_port;
1784               tcp0->src_port = s0->out2in.port;
1785               new_port0 = tcp0->src_port;
1786
1787               sum0 = tcp0->checksum;
1788               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1789                                      ip4_header_t,
1790                                      dst_address /* changed member */);
1791               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1792                                      ip4_header_t /* cheat */,
1793                                      length /* changed member */);
1794               mss_clamping (sm, tcp0, &sum0);
1795               tcp0->checksum = ip_csum_fold(sum0);
1796             }
1797           else
1798             {
1799               old_port0 = udp0->src_port;
1800               udp0->src_port = s0->out2in.port;
1801               udp0->checksum = 0;
1802             }
1803
1804           /* Accounting */
1805           nat44_session_update_counters (s0, now,
1806                                          vlib_buffer_length_in_chain (vm, b0));
1807           /* Per-user LRU list maintenance */
1808           nat44_session_update_lru (sm, s0, thread_index);
1809
1810         trace0:
1811           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1812                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1813             {
1814               snat_in2out_trace_t *t =
1815                  vlib_add_trace (vm, node, b0, sizeof (*t));
1816               t->is_slow_path = is_slow_path;
1817               t->sw_if_index = sw_if_index0;
1818               t->next_index = next0;
1819                   t->session_index = ~0;
1820               if (s0)
1821                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1822             }
1823
1824           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1825
1826           /* verify speculative enqueue, maybe switch current next frame */
1827           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1828                                            to_next, n_left_to_next,
1829                                            bi0, next0);
1830         }
1831
1832       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1833     }
1834
1835   vlib_node_increment_counter (vm, stats_node_index,
1836                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1837                                pkts_processed);
1838   return frame->n_vectors;
1839 }
1840
1841 static uword
1842 snat_in2out_fast_path_fn (vlib_main_t * vm,
1843                           vlib_node_runtime_t * node,
1844                           vlib_frame_t * frame)
1845 {
1846   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1847 }
1848
1849 VLIB_REGISTER_NODE (snat_in2out_node) = {
1850   .function = snat_in2out_fast_path_fn,
1851   .name = "nat44-in2out",
1852   .vector_size = sizeof (u32),
1853   .format_trace = format_snat_in2out_trace,
1854   .type = VLIB_NODE_TYPE_INTERNAL,
1855
1856   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1857   .error_strings = snat_in2out_error_strings,
1858
1859   .runtime_data_bytes = sizeof (snat_runtime_t),
1860
1861   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1862
1863   /* edit / add dispositions here */
1864   .next_nodes = {
1865     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1866     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1867     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1868     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1869     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1870   },
1871 };
1872
1873 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1874
1875 static uword
1876 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1877                                  vlib_node_runtime_t * node,
1878                                  vlib_frame_t * frame)
1879 {
1880   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1881 }
1882
1883 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1884   .function = snat_in2out_output_fast_path_fn,
1885   .name = "nat44-in2out-output",
1886   .vector_size = sizeof (u32),
1887   .format_trace = format_snat_in2out_trace,
1888   .type = VLIB_NODE_TYPE_INTERNAL,
1889
1890   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1891   .error_strings = snat_in2out_error_strings,
1892
1893   .runtime_data_bytes = sizeof (snat_runtime_t),
1894
1895   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1896
1897   /* edit / add dispositions here */
1898   .next_nodes = {
1899     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1900     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1901     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1902     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1903     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1904   },
1905 };
1906
1907 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1908                               snat_in2out_output_fast_path_fn);
1909
1910 static uword
1911 snat_in2out_slow_path_fn (vlib_main_t * vm,
1912                           vlib_node_runtime_t * node,
1913                           vlib_frame_t * frame)
1914 {
1915   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1916 }
1917
1918 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1919   .function = snat_in2out_slow_path_fn,
1920   .name = "nat44-in2out-slowpath",
1921   .vector_size = sizeof (u32),
1922   .format_trace = format_snat_in2out_trace,
1923   .type = VLIB_NODE_TYPE_INTERNAL,
1924
1925   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1926   .error_strings = snat_in2out_error_strings,
1927
1928   .runtime_data_bytes = sizeof (snat_runtime_t),
1929
1930   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1931
1932   /* edit / add dispositions here */
1933   .next_nodes = {
1934     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1935     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1936     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1937     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1938     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1939   },
1940 };
1941
1942 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1943                               snat_in2out_slow_path_fn);
1944
1945 static uword
1946 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1947                                  vlib_node_runtime_t * node,
1948                                  vlib_frame_t * frame)
1949 {
1950   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1951 }
1952
1953 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1954   .function = snat_in2out_output_slow_path_fn,
1955   .name = "nat44-in2out-output-slowpath",
1956   .vector_size = sizeof (u32),
1957   .format_trace = format_snat_in2out_trace,
1958   .type = VLIB_NODE_TYPE_INTERNAL,
1959
1960   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1961   .error_strings = snat_in2out_error_strings,
1962
1963   .runtime_data_bytes = sizeof (snat_runtime_t),
1964
1965   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1966
1967   /* edit / add dispositions here */
1968   .next_nodes = {
1969     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1970     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1971     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1972     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1973     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1974   },
1975 };
1976
1977 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1978                               snat_in2out_output_slow_path_fn);
1979
1980 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1981
1982 static inline uword
1983 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1984                              vlib_node_runtime_t * node,
1985                              vlib_frame_t * frame,
1986                              int is_ed)
1987 {
1988   u32 n_left_from, * from, * to_next, stats_node_index;
1989   snat_in2out_next_t next_index;
1990   u32 pkts_processed = 0;
1991   snat_main_t * sm = &snat_main;
1992   vnet_feature_main_t *fm = &feature_main;
1993   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1994   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1995
1996   stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1997     nat44_hairpinning_node.index;
1998   from = vlib_frame_vector_args (frame);
1999   n_left_from = frame->n_vectors;
2000   next_index = node->cached_next_index;
2001
2002   while (n_left_from > 0)
2003     {
2004       u32 n_left_to_next;
2005
2006       vlib_get_next_frame (vm, node, next_index,
2007                            to_next, n_left_to_next);
2008
2009       while (n_left_from > 0 && n_left_to_next > 0)
2010         {
2011           u32 bi0;
2012           vlib_buffer_t * b0;
2013           u32 next0;
2014           ip4_header_t * ip0;
2015           u32 proto0;
2016           udp_header_t * udp0;
2017           tcp_header_t * tcp0;
2018
2019           /* speculatively enqueue b0 to the current next frame */
2020           bi0 = from[0];
2021           to_next[0] = bi0;
2022           from += 1;
2023           to_next += 1;
2024           n_left_from -= 1;
2025           n_left_to_next -= 1;
2026
2027           b0 = vlib_get_buffer (vm, bi0);
2028           ip0 = vlib_buffer_get_current (b0);
2029           udp0 = ip4_next_header (ip0);
2030           tcp0 = (tcp_header_t *) udp0;
2031
2032           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2033
2034           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2035                                 &next0, 0);
2036
2037           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
2038             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2039
2040           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2041
2042           /* verify speculative enqueue, maybe switch current next frame */
2043           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2044                                            to_next, n_left_to_next,
2045                                            bi0, next0);
2046          }
2047
2048       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2049     }
2050
2051   vlib_node_increment_counter (vm, stats_node_index,
2052                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2053                                pkts_processed);
2054   return frame->n_vectors;
2055 }
2056
2057 static uword
2058 nat44_hairpinning_fn (vlib_main_t * vm,
2059                       vlib_node_runtime_t * node,
2060                       vlib_frame_t * frame)
2061 {
2062   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
2063 }
2064
2065 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2066   .function = nat44_hairpinning_fn,
2067   .name = "nat44-hairpinning",
2068   .vector_size = sizeof (u32),
2069   .type = VLIB_NODE_TYPE_INTERNAL,
2070   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2071   .error_strings = snat_in2out_error_strings,
2072   .n_next_nodes = 2,
2073   .next_nodes = {
2074     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2075     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2076   },
2077 };
2078
2079 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2080                               nat44_hairpinning_fn);
2081
2082 static uword
2083 nat44_ed_hairpinning_fn (vlib_main_t * vm,
2084                          vlib_node_runtime_t * node,
2085                          vlib_frame_t * frame)
2086 {
2087   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
2088 }
2089
2090 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
2091   .function = nat44_ed_hairpinning_fn,
2092   .name = "nat44-ed-hairpinning",
2093   .vector_size = sizeof (u32),
2094   .type = VLIB_NODE_TYPE_INTERNAL,
2095   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2096   .error_strings = snat_in2out_error_strings,
2097   .n_next_nodes = 2,
2098   .next_nodes = {
2099     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2100     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2101   },
2102 };
2103
2104 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
2105                               nat44_ed_hairpinning_fn);
2106
2107 static inline void
2108 nat44_reass_hairpinning (snat_main_t *sm,
2109                          vlib_buffer_t * b0,
2110                          ip4_header_t * ip0,
2111                          u16 sport,
2112                          u16 dport,
2113                          u32 proto0,
2114                          int is_ed)
2115 {
2116   snat_session_key_t key0, sm0;
2117   snat_session_t * s0;
2118   clib_bihash_kv_8_8_t kv0, value0;
2119   ip_csum_t sum0;
2120   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2121   u16 new_dst_port0, old_dst_port0;
2122   udp_header_t * udp0;
2123   tcp_header_t * tcp0;
2124   int rv;
2125
2126   key0.addr = ip0->dst_address;
2127   key0.port = dport;
2128   key0.protocol = proto0;
2129   key0.fib_index = sm->outside_fib_index;
2130   kv0.key = key0.as_u64;
2131
2132   udp0 = ip4_next_header (ip0);
2133
2134   /* Check if destination is static mappings */
2135   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
2136     {
2137       new_dst_addr0 = sm0.addr.as_u32;
2138       new_dst_port0 = sm0.port;
2139       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2140     }
2141   /* or active sessions */
2142   else
2143     {
2144       if (sm->num_workers > 1)
2145         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2146       else
2147         ti = sm->num_workers;
2148
2149       if (is_ed)
2150         {
2151           clib_bihash_kv_16_8_t ed_kv, ed_value;
2152           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
2153                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
2154                       udp0->src_port);
2155           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
2156                                         &ed_kv, &ed_value);
2157           si = ed_value.value;
2158         }
2159       else
2160         {
2161           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
2162                                        &value0);
2163           si = value0.value;
2164         }
2165       if (!rv)
2166         {
2167           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2168           new_dst_addr0 = s0->in2out.addr.as_u32;
2169           new_dst_port0 = s0->in2out.port;
2170           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2171         }
2172     }
2173
2174   /* Destination is behind the same NAT, use internal address and port */
2175   if (new_dst_addr0)
2176     {
2177       old_dst_addr0 = ip0->dst_address.as_u32;
2178       ip0->dst_address.as_u32 = new_dst_addr0;
2179       sum0 = ip0->checksum;
2180       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2181                              ip4_header_t, dst_address);
2182       ip0->checksum = ip_csum_fold (sum0);
2183
2184       old_dst_port0 = dport;
2185       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2186                        ip4_is_first_fragment (ip0)))
2187         {
2188           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2189             {
2190               tcp0 = ip4_next_header (ip0);
2191               tcp0->dst = new_dst_port0;
2192               sum0 = tcp0->checksum;
2193               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2194                                      ip4_header_t, dst_address);
2195               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2196                                      ip4_header_t /* cheat */, length);
2197               tcp0->checksum = ip_csum_fold(sum0);
2198             }
2199           else
2200             {
2201               udp0->dst_port = new_dst_port0;
2202               udp0->checksum = 0;
2203             }
2204         }
2205       else
2206         {
2207           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2208             {
2209               tcp0 = ip4_next_header (ip0);
2210               sum0 = tcp0->checksum;
2211               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2212                                      ip4_header_t, dst_address);
2213               tcp0->checksum = ip_csum_fold(sum0);
2214             }
2215         }
2216     }
2217 }
2218
2219 static uword
2220 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2221                             vlib_node_runtime_t * node,
2222                             vlib_frame_t * frame)
2223 {
2224   u32 n_left_from, *from, *to_next;
2225   snat_in2out_next_t next_index;
2226   u32 pkts_processed = 0;
2227   snat_main_t *sm = &snat_main;
2228   f64 now = vlib_time_now (vm);
2229   u32 thread_index = vm->thread_index;
2230   snat_main_per_thread_data_t *per_thread_data =
2231     &sm->per_thread_data[thread_index];
2232   u32 *fragments_to_drop = 0;
2233   u32 *fragments_to_loopback = 0;
2234
2235   from = vlib_frame_vector_args (frame);
2236   n_left_from = frame->n_vectors;
2237   next_index = node->cached_next_index;
2238
2239   while (n_left_from > 0)
2240     {
2241       u32 n_left_to_next;
2242
2243       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2244
2245       while (n_left_from > 0 && n_left_to_next > 0)
2246        {
2247           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2248           vlib_buffer_t *b0;
2249           u32 next0;
2250           u8 cached0 = 0;
2251           ip4_header_t *ip0;
2252           nat_reass_ip4_t *reass0;
2253           udp_header_t * udp0;
2254           tcp_header_t * tcp0;
2255           snat_session_key_t key0;
2256           clib_bihash_kv_8_8_t kv0, value0;
2257           snat_session_t * s0 = 0;
2258           u16 old_port0, new_port0;
2259           ip_csum_t sum0;
2260
2261           /* speculatively enqueue b0 to the current next frame */
2262           bi0 = from[0];
2263           to_next[0] = bi0;
2264           from += 1;
2265           to_next += 1;
2266           n_left_from -= 1;
2267           n_left_to_next -= 1;
2268
2269           b0 = vlib_get_buffer (vm, bi0);
2270           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2271
2272           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2273           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2274                                                                sw_if_index0);
2275
2276           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2277             {
2278               next0 = SNAT_IN2OUT_NEXT_DROP;
2279               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2280               goto trace0;
2281             }
2282
2283           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2284           udp0 = ip4_next_header (ip0);
2285           tcp0 = (tcp_header_t *) udp0;
2286           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2287
2288           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2289                                                  ip0->dst_address,
2290                                                  ip0->fragment_id,
2291                                                  ip0->protocol,
2292                                                  1,
2293                                                  &fragments_to_drop);
2294
2295           if (PREDICT_FALSE (!reass0))
2296             {
2297               next0 = SNAT_IN2OUT_NEXT_DROP;
2298               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2299               nat_log_notice ("maximum reassemblies exceeded");
2300               goto trace0;
2301             }
2302
2303           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2304             {
2305               key0.addr = ip0->src_address;
2306               key0.port = udp0->src_port;
2307               key0.protocol = proto0;
2308               key0.fib_index = rx_fib_index0;
2309               kv0.key = key0.as_u64;
2310
2311               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2312                 {
2313                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2314                       ip0, proto0, rx_fib_index0, thread_index)))
2315                     goto trace0;
2316
2317                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2318                                      &s0, node, next0, thread_index, now);
2319
2320                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2321                     goto trace0;
2322
2323                   reass0->sess_index = s0 - per_thread_data->sessions;
2324                 }
2325               else
2326                 {
2327                   s0 = pool_elt_at_index (per_thread_data->sessions,
2328                                           value0.value);
2329                   reass0->sess_index = value0.value;
2330                 }
2331               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2332             }
2333           else
2334             {
2335               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2336                 {
2337                   if (nat_ip4_reass_add_fragment (reass0, bi0, &fragments_to_drop))
2338                     {
2339                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2340                       nat_log_notice ("maximum fragments per reassembly exceeded");
2341                       next0 = SNAT_IN2OUT_NEXT_DROP;
2342                       goto trace0;
2343                     }
2344                   cached0 = 1;
2345                   goto trace0;
2346                 }
2347               s0 = pool_elt_at_index (per_thread_data->sessions,
2348                                       reass0->sess_index);
2349             }
2350
2351           old_addr0 = ip0->src_address.as_u32;
2352           ip0->src_address = s0->out2in.addr;
2353           new_addr0 = ip0->src_address.as_u32;
2354           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2355
2356           sum0 = ip0->checksum;
2357           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2358                                  ip4_header_t,
2359                                  src_address /* changed member */);
2360           ip0->checksum = ip_csum_fold (sum0);
2361
2362           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2363             {
2364               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2365                 {
2366                   old_port0 = tcp0->src_port;
2367                   tcp0->src_port = s0->out2in.port;
2368                   new_port0 = tcp0->src_port;
2369
2370                   sum0 = tcp0->checksum;
2371                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2372                                          ip4_header_t,
2373                                          dst_address /* changed member */);
2374                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2375                                          ip4_header_t /* cheat */,
2376                                          length /* changed member */);
2377                   tcp0->checksum = ip_csum_fold(sum0);
2378                 }
2379               else
2380                 {
2381                   old_port0 = udp0->src_port;
2382                   udp0->src_port = s0->out2in.port;
2383                   udp0->checksum = 0;
2384                 }
2385             }
2386
2387           /* Hairpinning */
2388           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2389                                    s0->ext_host_port, proto0, 0);
2390
2391           /* Accounting */
2392           nat44_session_update_counters (s0, now,
2393                                          vlib_buffer_length_in_chain (vm, b0));
2394           /* Per-user LRU list maintenance */
2395           nat44_session_update_lru (sm, s0, thread_index);
2396
2397         trace0:
2398           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2399                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2400             {
2401               nat44_in2out_reass_trace_t *t =
2402                  vlib_add_trace (vm, node, b0, sizeof (*t));
2403               t->cached = cached0;
2404               t->sw_if_index = sw_if_index0;
2405               t->next_index = next0;
2406             }
2407
2408           if (cached0)
2409             {
2410               n_left_to_next++;
2411               to_next--;
2412             }
2413           else
2414             {
2415               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2416
2417               /* verify speculative enqueue, maybe switch current next frame */
2418               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2419                                                to_next, n_left_to_next,
2420                                                bi0, next0);
2421             }
2422
2423           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2424             {
2425               from = vlib_frame_vector_args (frame);
2426               u32 len = vec_len (fragments_to_loopback);
2427               if (len <= VLIB_FRAME_SIZE)
2428                 {
2429                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2430                   n_left_from = len;
2431                   vec_reset_length (fragments_to_loopback);
2432                 }
2433               else
2434                 {
2435                   clib_memcpy (from,
2436                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2437                                sizeof (u32) * VLIB_FRAME_SIZE);
2438                   n_left_from = VLIB_FRAME_SIZE;
2439                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2440                 }
2441             }
2442        }
2443
2444       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2445     }
2446
2447   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2448                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2449                                pkts_processed);
2450
2451   nat_send_all_to_node (vm, fragments_to_drop, node,
2452                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2453                         SNAT_IN2OUT_NEXT_DROP);
2454
2455   vec_free (fragments_to_drop);
2456   vec_free (fragments_to_loopback);
2457   return frame->n_vectors;
2458 }
2459
2460 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2461   .function = nat44_in2out_reass_node_fn,
2462   .name = "nat44-in2out-reass",
2463   .vector_size = sizeof (u32),
2464   .format_trace = format_nat44_in2out_reass_trace,
2465   .type = VLIB_NODE_TYPE_INTERNAL,
2466
2467   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2468   .error_strings = snat_in2out_error_strings,
2469
2470   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2471   .next_nodes = {
2472     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2473     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2474     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2475     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2476     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2477   },
2478 };
2479
2480 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2481                               nat44_in2out_reass_node_fn);
2482
2483 /*******************************/
2484 /*** endpoint-dependent mode ***/
2485 /*******************************/
2486
2487 static_always_inline int
2488 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2489 {
2490   icmp46_header_t *icmp0;
2491   nat_ed_ses_key_t key0;
2492   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2493   ip4_header_t *inner_ip0 = 0;
2494   void *l4_header = 0;
2495   icmp46_header_t *inner_icmp0;
2496
2497   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2498   echo0 = (icmp_echo_header_t *)(icmp0+1);
2499
2500   if (!icmp_is_error_message (icmp0))
2501     {
2502       key0.proto = IP_PROTOCOL_ICMP;
2503       key0.l_addr = ip0->src_address;
2504       key0.r_addr = ip0->dst_address;
2505       key0.l_port = echo0->identifier;
2506       key0.r_port = 0;
2507     }
2508   else
2509     {
2510       inner_ip0 = (ip4_header_t *)(echo0+1);
2511       l4_header = ip4_next_header (inner_ip0);
2512       key0.proto = inner_ip0->protocol;
2513       key0.r_addr = inner_ip0->src_address;
2514       key0.l_addr = inner_ip0->dst_address;
2515       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2516         {
2517         case SNAT_PROTOCOL_ICMP:
2518           inner_icmp0 = (icmp46_header_t*)l4_header;
2519           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2520           key0.r_port = 0;
2521           key0.l_port = inner_echo0->identifier;
2522           break;
2523         case SNAT_PROTOCOL_UDP:
2524         case SNAT_PROTOCOL_TCP:
2525           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2526           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2527           break;
2528         default:
2529           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2530         }
2531     }
2532   *p_key0 = key0;
2533   return 0;
2534 }
2535
2536 int
2537 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void * arg)
2538 {
2539   snat_main_t *sm = &snat_main;
2540   nat44_is_idle_session_ctx_t *ctx = arg;
2541   snat_session_t *s;
2542   u64 sess_timeout_time;
2543   nat_ed_ses_key_t ed_key;
2544   clib_bihash_kv_16_8_t ed_kv;
2545   int i;
2546   snat_address_t *a;
2547   snat_session_key_t key;
2548   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
2549                                                        ctx->thread_index);
2550
2551   s = pool_elt_at_index (tsm->sessions, kv->value);
2552   sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
2553   if (ctx->now >= sess_timeout_time)
2554     {
2555       if (is_fwd_bypass_session (s))
2556         goto delete;
2557
2558       ed_key.l_addr = s->out2in.addr;
2559       ed_key.r_addr = s->ext_host_addr;
2560       ed_key.fib_index = s->out2in.fib_index;
2561       if (snat_is_unk_proto_session (s))
2562         {
2563           ed_key.proto = s->in2out.port;
2564           ed_key.r_port = 0;
2565           ed_key.l_port = 0;
2566         }
2567       else
2568         {
2569           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
2570           ed_key.l_port = s->out2in.port;
2571           ed_key.r_port = s->ext_host_port;
2572         }
2573       ed_kv.key[0] = ed_key.as_u64[0];
2574       ed_kv.key[1] = ed_key.as_u64[1];
2575       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
2576         nat_log_warn ("out2in_ed key del failed");
2577
2578       if (snat_is_unk_proto_session (s))
2579         goto delete;
2580
2581       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
2582                                           s->out2in.addr.as_u32,
2583                                           s->in2out.protocol,
2584                                           s->in2out.port,
2585                                           s->out2in.port,
2586                                           s->in2out.fib_index);
2587
2588       if (is_twice_nat_session (s))
2589         {
2590           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
2591             {
2592               key.protocol = s->in2out.protocol;
2593               key.port = s->ext_host_nat_port;
2594               a = sm->twice_nat_addresses + i;
2595               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
2596                 {
2597                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
2598                                                       ctx->thread_index, &key);
2599                   break;
2600                 }
2601             }
2602         }
2603
2604       if (snat_is_session_static (s))
2605         goto delete;
2606
2607       if (s->outside_address_index != ~0)
2608         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
2609                                             &s->out2in);
2610     delete:
2611       nat44_delete_session (sm, s, ctx->thread_index);
2612       return 1;
2613     }
2614
2615   return 0;
2616 }
2617
2618 static inline u32
2619 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
2620                           ip4_header_t * ip0, icmp46_header_t * icmp0,
2621                           u32 sw_if_index0, u32 rx_fib_index0,
2622                           vlib_node_runtime_t * node, u32 next0, f64 now,
2623                           u32 thread_index, snat_session_t ** p_s0)
2624 {
2625   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2626                       next0, thread_index, p_s0, 0);
2627   snat_session_t * s0 = *p_s0;
2628   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
2629     {
2630       /* Hairpinning */
2631       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
2632         snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
2633       /* Accounting */
2634       nat44_session_update_counters (s0, now,
2635                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
2636     }
2637   return next0;
2638 }
2639
2640 static u32
2641 slow_path_ed (snat_main_t *sm,
2642               vlib_buffer_t *b,
2643               u32 rx_fib_index,
2644               clib_bihash_kv_16_8_t *kv,
2645               snat_session_t ** sessionp,
2646               vlib_node_runtime_t * node,
2647               u32 next,
2648               u32 thread_index,
2649               f64 now)
2650 {
2651   snat_session_t *s;
2652   snat_user_t *u;
2653   snat_session_key_t key0, key1;
2654   lb_nat_type_t lb = 0, is_sm = 0;
2655   u32 address_index = ~0;
2656   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2657   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2658   u32 proto = ip_proto_to_snat_proto (key->proto);
2659   nat_outside_fib_t *outside_fib;
2660   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2661   fib_prefix_t pfx = {
2662     .fp_proto = FIB_PROTOCOL_IP4,
2663     .fp_len = 32,
2664     .fp_addr = {
2665         .ip4.as_u32 = key->r_addr.as_u32,
2666     },
2667   };
2668   nat44_is_idle_session_ctx_t ctx;
2669
2670   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2671     {
2672       b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2673       nat_ipfix_logging_max_sessions(sm->max_translations);
2674       nat_log_notice ("maximum sessions exceeded");
2675       return SNAT_IN2OUT_NEXT_DROP;
2676     }
2677
2678   key0.addr = key->l_addr;
2679   key0.port = key->l_port;
2680   key1.protocol = key0.protocol = proto;
2681   key0.fib_index = rx_fib_index;
2682   key1.fib_index = sm->outside_fib_index;
2683   /* First try to match static mapping by local address and port */
2684   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0))
2685     {
2686       /* Try to create dynamic translation */
2687       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2688                                                thread_index, &key1,
2689                                                &address_index,
2690                                                sm->port_per_thread,
2691                                                tsm->snat_thread_index))
2692         {
2693           nat_log_notice ("addresses exhausted");
2694           b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2695           return SNAT_IN2OUT_NEXT_DROP;
2696         }
2697     }
2698   else
2699     is_sm = 1;
2700
2701   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2702   if (!u)
2703     {
2704       nat_log_warn ("create NAT user failed");
2705       if (!is_sm)
2706         snat_free_outside_address_and_port (sm->addresses,
2707                                             thread_index, &key1);
2708       return SNAT_IN2OUT_NEXT_DROP;
2709     }
2710
2711   s = nat_ed_session_alloc (sm, u, thread_index);
2712   if (!s)
2713     {
2714       nat44_delete_user_with_no_session (sm, u, thread_index);
2715       nat_log_warn ("create NAT session failed");
2716       if (!is_sm)
2717         snat_free_outside_address_and_port (sm->addresses,
2718                                             thread_index, &key1);
2719       return SNAT_IN2OUT_NEXT_DROP;
2720     }
2721
2722   user_session_increment (sm, u, is_sm);
2723   if (is_sm)
2724     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2725   if (lb)
2726     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2727   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2728   s->outside_address_index = address_index;
2729   s->ext_host_addr = key->r_addr;
2730   s->ext_host_port = key->r_port;
2731   s->in2out = key0;
2732   s->out2in = key1;
2733   s->out2in.protocol = key0.protocol;
2734
2735   switch (vec_len (sm->outside_fibs))
2736     {
2737     case 0:
2738       s->out2in.fib_index = sm->outside_fib_index;
2739       break;
2740     case 1:
2741       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2742       break;
2743     default:
2744       vec_foreach (outside_fib, sm->outside_fibs)
2745         {
2746           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2747           if (FIB_NODE_INDEX_INVALID != fei)
2748             {
2749               if (fib_entry_get_resolving_interface (fei) != ~0)
2750                 {
2751                   s->out2in.fib_index = outside_fib->fib_index;
2752                   break;
2753                 }
2754             }
2755         }
2756       break;
2757     }
2758
2759   /* Add to lookup tables */
2760   kv->value = s - tsm->sessions;
2761   ctx.now = now;
2762   ctx.thread_index = thread_index;
2763   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
2764                                                nat44_i2o_ed_is_idle_session_cb,
2765                                                &ctx))
2766     nat_log_notice ("in2out-ed key add failed");
2767
2768   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2769               key1.port, key->r_port);
2770   kv->value = s - tsm->sessions;
2771   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
2772                                                nat44_o2i_ed_is_idle_session_cb,
2773                                                &ctx))
2774     nat_log_notice ("out2in-ed key add failed");
2775
2776   *sessionp = s;
2777
2778   /* log NAT event */
2779   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2780                                       s->out2in.addr.as_u32,
2781                                       s->in2out.protocol,
2782                                       s->in2out.port,
2783                                       s->out2in.port,
2784                                       s->in2out.fib_index);
2785   return next;
2786 }
2787
2788 static_always_inline int
2789 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2790                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
2791                         u32 rx_fib_index, u32 thread_index)
2792 {
2793   udp_header_t *udp = ip4_next_header (ip);
2794   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2795   clib_bihash_kv_16_8_t kv, value;
2796   snat_session_key_t key0, key1;
2797
2798   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2799               sm->outside_fib_index, udp->dst_port, udp->src_port);
2800
2801   /* NAT packet aimed at external address if */
2802   /* has active sessions */
2803   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2804     {
2805       key0.addr = ip->dst_address;
2806       key0.port = udp->dst_port;
2807       key0.protocol = proto;
2808       key0.fib_index = sm->outside_fib_index;
2809       /* or is static mappings */
2810       if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0, 0))
2811         return 0;
2812     }
2813   else
2814     return 0;
2815
2816   if (sm->forwarding_enabled)
2817     return 1;
2818
2819   return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2820 }
2821
2822 static_always_inline int
2823 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2824                                       u32 thread_index, f64 now,
2825                                       vlib_main_t * vm, vlib_buffer_t * b)
2826 {
2827   nat_ed_ses_key_t key;
2828   clib_bihash_kv_16_8_t kv, value;
2829   udp_header_t *udp;
2830   snat_session_t *s = 0;
2831   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2832
2833   if (!sm->forwarding_enabled)
2834     return 0;
2835
2836   if (ip->protocol == IP_PROTOCOL_ICMP)
2837     {
2838       key.as_u64[0] = key.as_u64[1] = 0;
2839       if (icmp_get_ed_key (ip, &key))
2840         return 0;
2841       key.fib_index = 0;
2842       kv.key[0] = key.as_u64[0];
2843       kv.key[1] = key.as_u64[1];
2844     }
2845   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2846     {
2847       udp = ip4_next_header(ip);
2848       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2849                   udp->src_port, udp->dst_port);
2850     }
2851   else
2852     {
2853       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2854                   0);
2855     }
2856
2857   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2858     {
2859       s = pool_elt_at_index (tsm->sessions, value.value);
2860       if (is_fwd_bypass_session (s))
2861         {
2862           if (ip->protocol == IP_PROTOCOL_TCP)
2863             {
2864               tcp_header_t *tcp = ip4_next_header(ip);
2865               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2866                 return 1;
2867             }
2868           /* Accounting */
2869           nat44_session_update_counters (s, now,
2870                                          vlib_buffer_length_in_chain (vm, b));
2871           return 1;
2872         }
2873       else
2874         return 0;
2875     }
2876
2877   return 0;
2878 }
2879
2880 static_always_inline int
2881 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2882                                        u8 proto, u16 src_port, u16 dst_port,
2883                                        u32 thread_index, u32 rx_sw_if_index,
2884                                        u32 tx_sw_if_index)
2885 {
2886   clib_bihash_kv_16_8_t kv, value;
2887   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2888   snat_interface_t *i;
2889   snat_session_t *s;
2890   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
2891   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
2892
2893   /* src NAT check */
2894   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
2895               src_port, dst_port);
2896   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2897     return 1;
2898
2899   /* dst NAT check */
2900   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
2901               dst_port, src_port);
2902   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2903   {
2904     s = pool_elt_at_index (tsm->sessions, value.value);
2905     if (is_fwd_bypass_session (s))
2906       return 0;
2907
2908     /* hairpinning */
2909     pool_foreach (i, sm->output_feature_interfaces,
2910     ({
2911       if ((nat_interface_is_inside(i)) && (rx_sw_if_index == i->sw_if_index))
2912         return 0;
2913     }));
2914     return 1;
2915   }
2916
2917   return 0;
2918 }
2919
2920 u32
2921 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2922                      u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2923                      u8 *p_proto, snat_session_key_t *p_value,
2924                      u8 *p_dont_translate, void *d, void *e)
2925 {
2926   icmp46_header_t *icmp;
2927   u32 sw_if_index;
2928   u32 rx_fib_index;
2929   nat_ed_ses_key_t key;
2930   snat_session_t *s = 0;
2931   u8 dont_translate = 0;
2932   clib_bihash_kv_16_8_t kv, value;
2933   u32 next = ~0;
2934   int err;
2935   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2936
2937   icmp = (icmp46_header_t *) ip4_next_header (ip);
2938   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2939   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2940
2941   key.as_u64[0] = key.as_u64[1] = 0;
2942   err = icmp_get_ed_key (ip, &key);
2943   if (err != 0)
2944     {
2945       b->error = node->errors[err];
2946       next = SNAT_IN2OUT_NEXT_DROP;
2947       goto out;
2948     }
2949   key.fib_index = rx_fib_index;
2950
2951   kv.key[0] = key.as_u64[0];
2952   kv.key[1] = key.as_u64[1];
2953
2954   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2955     {
2956       if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2957         {
2958           if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2959               key.proto, key.l_port, key.r_port, thread_index, sw_if_index,
2960               vnet_buffer(b)->sw_if_index[VLIB_TX])))
2961             {
2962               dont_translate = 1;
2963               goto out;
2964             }
2965         }
2966       else
2967         {
2968           if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2969               ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2970             {
2971               dont_translate = 1;
2972               goto out;
2973             }
2974         }
2975
2976       if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2977         {
2978           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2979           next = SNAT_IN2OUT_NEXT_DROP;
2980           goto out;
2981         }
2982
2983       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2984                            thread_index, vlib_time_now (sm->vlib_main));
2985
2986       if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2987         goto out;
2988     }
2989   else
2990     {
2991       if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2992                         icmp->type != ICMP4_echo_reply &&
2993                         !icmp_is_error_message (icmp)))
2994         {
2995           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2996           next = SNAT_IN2OUT_NEXT_DROP;
2997           goto out;
2998         }
2999
3000       s = pool_elt_at_index (tsm->sessions, value.value);
3001     }
3002
3003   *p_proto = ip_proto_to_snat_proto (key.proto);
3004 out:
3005   if (s)
3006     *p_value = s->out2in;
3007   *p_dont_translate = dont_translate;
3008   if (d)
3009     *(snat_session_t**)d = s;
3010   return next;
3011 }
3012
3013 static inline void
3014 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
3015                                     vlib_buffer_t * b,
3016                                     ip4_header_t * ip)
3017 {
3018   u32 old_addr, new_addr = 0, ti = 0;
3019   clib_bihash_kv_8_8_t kv, value;
3020   clib_bihash_kv_16_8_t s_kv, s_value;
3021   snat_static_mapping_t *m;
3022   ip_csum_t sum;
3023   snat_session_t *s;
3024   snat_main_per_thread_data_t *tsm;
3025
3026   if (sm->num_workers > 1)
3027     ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
3028   else
3029     ti = sm->num_workers;
3030   tsm = &sm->per_thread_data[ti];
3031
3032   old_addr = ip->dst_address.as_u32;
3033   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
3034               sm->outside_fib_index, 0, 0);
3035   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3036     {
3037       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3038       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3039         return;
3040
3041       m = pool_elt_at_index (sm->static_mappings, value.value);
3042       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3043         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
3044       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
3045     }
3046   else
3047     {
3048       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
3049       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3050         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
3051       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
3052     }
3053   sum = ip->checksum;
3054   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
3055   ip->checksum = ip_csum_fold (sum);
3056 }
3057
3058 static snat_session_t *
3059 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
3060                                vlib_buffer_t * b,
3061                                ip4_header_t * ip,
3062                                u32 rx_fib_index,
3063                                u32 thread_index,
3064                                f64 now,
3065                                vlib_main_t * vm,
3066                                vlib_node_runtime_t * node)
3067 {
3068   clib_bihash_kv_8_8_t kv, value;
3069   clib_bihash_kv_16_8_t s_kv, s_value;
3070   snat_static_mapping_t *m;
3071   u32 old_addr, new_addr = 0;
3072   ip_csum_t sum;
3073   snat_user_t *u;
3074   dlist_elt_t *head, *elt;
3075   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3076   u32 elt_index, head_index, ses_index;
3077   snat_session_t * s;
3078   u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
3079   int i;
3080   u8 is_sm = 0;
3081   nat_outside_fib_t *outside_fib;
3082   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3083   fib_prefix_t pfx = {
3084     .fp_proto = FIB_PROTOCOL_IP4,
3085     .fp_len = 32,
3086     .fp_addr = {
3087         .ip4.as_u32 = ip->dst_address.as_u32,
3088     },
3089   };
3090
3091   switch (vec_len (sm->outside_fibs))
3092     {
3093     case 0:
3094       outside_fib_index = sm->outside_fib_index;
3095       break;
3096     case 1:
3097       outside_fib_index = sm->outside_fibs[0].fib_index;
3098       break;
3099     default:
3100       vec_foreach (outside_fib, sm->outside_fibs)
3101         {
3102           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3103           if (FIB_NODE_INDEX_INVALID != fei)
3104             {
3105               if (fib_entry_get_resolving_interface (fei) != ~0)
3106                 {
3107                   outside_fib_index = outside_fib->fib_index;
3108                   break;
3109                 }
3110             }
3111         }
3112       break;
3113     }
3114   old_addr = ip->src_address.as_u32;
3115
3116   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
3117               rx_fib_index, 0, 0);
3118
3119   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
3120     {
3121       s = pool_elt_at_index (tsm->sessions, s_value.value);
3122       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
3123     }
3124   else
3125     {
3126       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
3127         {
3128           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
3129           nat_ipfix_logging_max_sessions(sm->max_translations);
3130           nat_log_notice ("maximum sessions exceeded");
3131           return 0;
3132         }
3133
3134       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
3135                                   thread_index);
3136       if (!u)
3137         {
3138           nat_log_warn ("create NAT user failed");
3139           return 0;
3140         }
3141
3142       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
3143
3144       /* Try to find static mapping first */
3145       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
3146         {
3147           m = pool_elt_at_index (sm->static_mappings, value.value);
3148           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
3149           is_sm = 1;
3150           goto create_ses;
3151         }
3152       /* Fallback to 3-tuple key */
3153       else
3154         {
3155           /* Choose same out address as for TCP/UDP session to same destination */
3156           head_index = u->sessions_per_user_list_head_index;
3157           head = pool_elt_at_index (tsm->list_pool, head_index);
3158           elt_index = head->next;
3159           if (PREDICT_FALSE (elt_index == ~0))
3160             ses_index = ~0;
3161           else
3162             {
3163               elt = pool_elt_at_index (tsm->list_pool, elt_index);
3164               ses_index = elt->value;
3165             }
3166
3167           while (ses_index != ~0)
3168             {
3169               s =  pool_elt_at_index (tsm->sessions, ses_index);
3170               elt_index = elt->next;
3171               elt = pool_elt_at_index (tsm->list_pool, elt_index);
3172               ses_index = elt->value;
3173
3174               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
3175                 {
3176                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
3177                   address_index = s->outside_address_index;
3178
3179                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
3180                               ip->protocol, outside_fib_index, 0, 0);
3181                   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3182                     goto create_ses;
3183
3184                   break;
3185                 }
3186             }
3187
3188           for (i = 0; i < vec_len (sm->addresses); i++)
3189             {
3190               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
3191                           ip->protocol, outside_fib_index, 0, 0);
3192               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3193                 {
3194                   new_addr = ip->src_address.as_u32 =
3195                     sm->addresses[i].addr.as_u32;
3196                   address_index = i;
3197                   goto create_ses;
3198                 }
3199             }
3200           return 0;
3201         }
3202
3203 create_ses:
3204       s = nat_ed_session_alloc (sm, u, thread_index);
3205       if (!s)
3206         {
3207           nat44_delete_user_with_no_session (sm, u, thread_index);
3208           nat_log_warn ("create NAT session failed");
3209           return 0;
3210         }
3211
3212       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
3213       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
3214       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
3215       s->outside_address_index = address_index;
3216       s->out2in.addr.as_u32 = new_addr;
3217       s->out2in.fib_index = outside_fib_index;
3218       s->in2out.addr.as_u32 = old_addr;
3219       s->in2out.fib_index = rx_fib_index;
3220       s->in2out.port = s->out2in.port = ip->protocol;
3221       if (is_sm)
3222         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
3223       user_session_increment (sm, u, is_sm);
3224
3225       /* Add to lookup tables */
3226       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
3227                   rx_fib_index, 0, 0);
3228       s_kv.value = s - tsm->sessions;
3229       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
3230         nat_log_notice ("in2out key add failed");
3231
3232       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
3233                   outside_fib_index, 0, 0);
3234       s_kv.value = s - tsm->sessions;
3235       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
3236         nat_log_notice ("out2in key add failed");
3237   }
3238
3239   /* Update IP checksum */
3240   sum = ip->checksum;
3241   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
3242   ip->checksum = ip_csum_fold (sum);
3243
3244   /* Accounting */
3245   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
3246
3247   /* Hairpinning */
3248   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3249     nat44_ed_hairpinning_unknown_proto(sm, b, ip);
3250
3251   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3252     vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
3253
3254   return s;
3255 }
3256
3257 static inline uword
3258 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
3259                                 vlib_node_runtime_t * node,
3260                                 vlib_frame_t * frame, int is_slow_path,
3261                                 int is_output_feature)
3262 {
3263   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
3264   snat_in2out_next_t next_index;
3265   snat_main_t *sm = &snat_main;
3266   f64 now = vlib_time_now (vm);
3267   u32 thread_index = vm->thread_index;
3268   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3269
3270   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
3271     nat44_ed_in2out_node.index;
3272
3273   from = vlib_frame_vector_args (frame);
3274   n_left_from = frame->n_vectors;
3275   next_index = node->cached_next_index;
3276
3277   while (n_left_from > 0)
3278     {
3279       u32 n_left_to_next;
3280
3281       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3282
3283       while (n_left_from >= 4 && n_left_to_next >= 2)
3284         {
3285           u32 bi0, bi1;
3286           vlib_buffer_t *b0, *b1;
3287           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3288               new_addr0, old_addr0;
3289           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3290               new_addr1, old_addr1;
3291           u16 old_port0, new_port0, old_port1, new_port1;
3292           ip4_header_t *ip0, *ip1;
3293           udp_header_t *udp0, *udp1;
3294           tcp_header_t *tcp0, *tcp1;
3295           icmp46_header_t *icmp0, *icmp1;
3296           snat_session_t *s0 = 0, *s1 = 0;
3297           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3298           ip_csum_t sum0, sum1;
3299
3300           /* Prefetch next iteration. */
3301           {
3302             vlib_buffer_t * p2, * p3;
3303
3304             p2 = vlib_get_buffer (vm, from[2]);
3305             p3 = vlib_get_buffer (vm, from[3]);
3306
3307             vlib_prefetch_buffer_header (p2, LOAD);
3308             vlib_prefetch_buffer_header (p3, LOAD);
3309
3310             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3311             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3312           }
3313
3314           /* speculatively enqueue b0 and b1 to the current next frame */
3315           to_next[0] = bi0 = from[0];
3316           to_next[1] = bi1 = from[1];
3317           from += 2;
3318           to_next += 2;
3319           n_left_from -= 2;
3320           n_left_to_next -= 2;
3321
3322           b0 = vlib_get_buffer (vm, bi0);
3323           b1 = vlib_get_buffer (vm, bi1);
3324
3325           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3326
3327           if (is_output_feature)
3328             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3329
3330           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3331                  iph_offset0);
3332
3333           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3334           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3335                                                                sw_if_index0);
3336
3337           if (PREDICT_FALSE(ip0->ttl == 1))
3338             {
3339               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3340               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3341                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3342                                            0);
3343               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3344               goto trace00;
3345             }
3346
3347           udp0 = ip4_next_header (ip0);
3348           tcp0 = (tcp_header_t *) udp0;
3349           icmp0 = (icmp46_header_t *) udp0;
3350           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3351
3352           if (is_slow_path)
3353             {
3354               if (PREDICT_FALSE (proto0 == ~0))
3355                 {
3356                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3357                                                       rx_fib_index0,
3358                                                       thread_index, now, vm,
3359                                                       node);
3360                   if (!s0)
3361                     next0 = SNAT_IN2OUT_NEXT_DROP;
3362                   goto trace00;
3363                 }
3364
3365               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3366                 {
3367                   next0 = icmp_in2out_ed_slow_path
3368                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3369                      next0, now, thread_index, &s0);
3370                   goto trace00;
3371                 }
3372             }
3373           else
3374             {
3375                if (PREDICT_FALSE (proto0 == ~0))
3376                  {
3377                    next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3378                    goto trace00;
3379                  }
3380
3381                if (ip4_is_fragment (ip0))
3382                  {
3383                    next0 = SNAT_IN2OUT_NEXT_REASS;
3384                    goto trace00;
3385                  }
3386
3387                if (is_output_feature)
3388                 {
3389                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3390                       sm, ip0, thread_index, now, vm, b0)))
3391                     goto trace00;
3392                 }
3393
3394                if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3395                  {
3396                    next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3397                    goto trace00;
3398                  }
3399             }
3400
3401           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3402                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3403
3404           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3405             {
3406               if (is_slow_path)
3407                 {
3408                   if (is_output_feature)
3409                     {
3410                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3411                           sm, ip0, ip0->protocol, udp0->src_port,
3412                           udp0->dst_port, thread_index, sw_if_index0,
3413                           vnet_buffer(b0)->sw_if_index[VLIB_TX])))
3414                         goto trace00;
3415                     }
3416                   else
3417                     {
3418                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3419                           sw_if_index0, ip0, proto0, rx_fib_index0,
3420                           thread_index)))
3421                         goto trace00;
3422                     }
3423
3424                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3425                                         next0, thread_index, now);
3426
3427                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3428                     goto trace00;
3429                 }
3430               else
3431                 {
3432                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3433                   goto trace00;
3434                 }
3435             }
3436           else
3437             {
3438               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3439             }
3440
3441           b0->flags |= VNET_BUFFER_F_IS_NATED;
3442
3443           if (!is_output_feature)
3444             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3445
3446           old_addr0 = ip0->src_address.as_u32;
3447           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3448           sum0 = ip0->checksum;
3449           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3450                                  src_address);
3451           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3452             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3453                                    s0->ext_host_addr.as_u32, ip4_header_t,
3454                                    dst_address);
3455           ip0->checksum = ip_csum_fold (sum0);
3456
3457           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3458             {
3459               old_port0 = tcp0->src_port;
3460               new_port0 = tcp0->src_port = s0->out2in.port;
3461
3462               sum0 = tcp0->checksum;
3463               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3464                                      dst_address);
3465               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3466                                      length);
3467               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3468                 {
3469                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3470                                          s0->ext_host_addr.as_u32,
3471                                          ip4_header_t, dst_address);
3472                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3473                                          s0->ext_host_port, ip4_header_t,
3474                                          length);
3475                   tcp0->dst_port = s0->ext_host_port;
3476                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3477                 }
3478               mss_clamping (sm, tcp0, &sum0);
3479               tcp0->checksum = ip_csum_fold(sum0);
3480               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3481                 goto trace00;
3482             }
3483           else
3484             {
3485               udp0->src_port = s0->out2in.port;
3486               udp0->checksum = 0;
3487               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3488                 {
3489                   udp0->dst_port = s0->ext_host_port;
3490                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3491                 }
3492             }
3493
3494           /* Accounting */
3495           nat44_session_update_counters (s0, now,
3496                                          vlib_buffer_length_in_chain (vm, b0));
3497
3498         trace00:
3499           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3500                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3501             {
3502               snat_in2out_trace_t *t =
3503                 vlib_add_trace (vm, node, b0, sizeof (*t));
3504               t->is_slow_path = is_slow_path;
3505               t->sw_if_index = sw_if_index0;
3506               t->next_index = next0;
3507               t->session_index = ~0;
3508               if (s0)
3509                 t->session_index = s0 - tsm->sessions;
3510             }
3511
3512           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3513
3514
3515           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3516
3517           if (is_output_feature)
3518             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3519
3520           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3521                  iph_offset1);
3522
3523           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3524           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3525                                                                sw_if_index1);
3526
3527           if (PREDICT_FALSE(ip1->ttl == 1))
3528             {
3529               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3530               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3531                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3532                                            0);
3533               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3534               goto trace01;
3535             }
3536
3537           udp1 = ip4_next_header (ip1);
3538           tcp1 = (tcp_header_t *) udp1;
3539           icmp1 = (icmp46_header_t *) udp1;
3540           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3541
3542           if (is_slow_path)
3543             {
3544               if (PREDICT_FALSE (proto1 == ~0))
3545                 {
3546                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3547                                                       rx_fib_index1,
3548                                                       thread_index, now, vm,
3549                                                       node);
3550                   if (!s1)
3551                     next1 = SNAT_IN2OUT_NEXT_DROP;
3552                   goto trace01;
3553                 }
3554
3555               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3556                 {
3557                   next1 = icmp_in2out_ed_slow_path
3558                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3559                      next1, now, thread_index, &s1);
3560                   goto trace01;
3561                 }
3562             }
3563           else
3564             {
3565                if (PREDICT_FALSE (proto1 == ~0))
3566                  {
3567                    next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3568                    goto trace01;
3569                  }
3570
3571                if (ip4_is_fragment (ip1))
3572                  {
3573                    next1 = SNAT_IN2OUT_NEXT_REASS;
3574                    goto trace01;
3575                  }
3576
3577                if (is_output_feature)
3578                 {
3579                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3580                       sm, ip1, thread_index, now, vm, b1)))
3581                     goto trace01;
3582                 }
3583
3584                if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3585                  {
3586                    next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3587                    goto trace01;
3588                  }
3589             }
3590
3591           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3592                       rx_fib_index1, udp1->src_port, udp1->dst_port);
3593
3594           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3595             {
3596               if (is_slow_path)
3597                 {
3598                   if (is_output_feature)
3599                     {
3600                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3601                           sm, ip1, ip1->protocol, udp1->src_port,
3602                           udp1->dst_port, thread_index, sw_if_index1,
3603                           vnet_buffer(b1)->sw_if_index[VLIB_TX])))
3604                         goto trace01;
3605                     }
3606                   else
3607                     {
3608                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3609                           sw_if_index1, ip1, proto1, rx_fib_index1,
3610                           thread_index)))
3611                         goto trace01;
3612                     }
3613
3614                   next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3615                                         next1, thread_index, now);
3616
3617                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3618                     goto trace01;
3619                 }
3620               else
3621                 {
3622                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3623                   goto trace01;
3624                 }
3625             }
3626           else
3627             {
3628               s1 = pool_elt_at_index (tsm->sessions, value1.value);
3629             }
3630
3631           b1->flags |= VNET_BUFFER_F_IS_NATED;
3632
3633           if (!is_output_feature)
3634             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3635
3636           old_addr1 = ip1->src_address.as_u32;
3637           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3638           sum1 = ip1->checksum;
3639           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3640                                  src_address);
3641           if (PREDICT_FALSE (is_twice_nat_session (s1)))
3642             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3643                                    s1->ext_host_addr.as_u32, ip4_header_t,
3644                                    dst_address);
3645           ip1->checksum = ip_csum_fold (sum1);
3646
3647           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3648             {
3649               old_port1 = tcp1->src_port;
3650               new_port1 = tcp1->src_port = s1->out2in.port;
3651
3652               sum1 = tcp1->checksum;
3653               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3654                                      dst_address);
3655               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3656                                      length);
3657               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3658                 {
3659                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3660                                          s1->ext_host_addr.as_u32,
3661                                          ip4_header_t, dst_address);
3662                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
3663                                          s1->ext_host_port, ip4_header_t,
3664                                          length);
3665                   tcp1->dst_port = s1->ext_host_port;
3666                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3667                 }
3668               tcp1->checksum = ip_csum_fold(sum1);
3669               mss_clamping (sm, tcp1, &sum1);
3670               if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3671                 goto trace01;
3672             }
3673           else
3674             {
3675               udp1->src_port = s1->out2in.port;
3676               udp1->checksum = 0;
3677               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3678                 {
3679                   udp1->dst_port = s1->ext_host_port;
3680                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3681                 }
3682             }
3683
3684           /* Accounting */
3685           nat44_session_update_counters (s1, now,
3686                                          vlib_buffer_length_in_chain (vm, b1));
3687
3688         trace01:
3689           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3690                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3691             {
3692               snat_in2out_trace_t *t =
3693                 vlib_add_trace (vm, node, b1, sizeof (*t));
3694               t->is_slow_path = is_slow_path;
3695               t->sw_if_index = sw_if_index1;
3696               t->next_index = next1;
3697               t->session_index = ~0;
3698               if (s1)
3699                 t->session_index = s1 - tsm->sessions;
3700             }
3701
3702           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3703
3704           /* verify speculative enqueues, maybe switch current next frame */
3705           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3706                                            to_next, n_left_to_next,
3707                                            bi0, bi1, next0, next1);
3708         }
3709
3710       while (n_left_from > 0 && n_left_to_next > 0)
3711         {
3712           u32 bi0;
3713           vlib_buffer_t *b0;
3714           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3715               new_addr0, old_addr0;
3716           u16 old_port0, new_port0;
3717           ip4_header_t *ip0;
3718           udp_header_t *udp0;
3719           tcp_header_t *tcp0;
3720           icmp46_header_t * icmp0;
3721           snat_session_t *s0 = 0;
3722           clib_bihash_kv_16_8_t kv0, value0;
3723           ip_csum_t sum0;
3724
3725           /* speculatively enqueue b0 to the current next frame */
3726           bi0 = from[0];
3727           to_next[0] = bi0;
3728           from += 1;
3729           to_next += 1;
3730           n_left_from -= 1;
3731           n_left_to_next -= 1;
3732
3733           b0 = vlib_get_buffer (vm, bi0);
3734           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3735
3736           if (is_output_feature)
3737             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3738
3739           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3740                  iph_offset0);
3741
3742           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3743           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3744                                                                sw_if_index0);
3745
3746           if (PREDICT_FALSE(ip0->ttl == 1))
3747             {
3748               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3749               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3750                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3751                                            0);
3752               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3753               goto trace0;
3754             }
3755
3756           udp0 = ip4_next_header (ip0);
3757           tcp0 = (tcp_header_t *) udp0;
3758           icmp0 = (icmp46_header_t *) udp0;
3759           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3760
3761           if (is_slow_path)
3762             {
3763               if (PREDICT_FALSE (proto0 == ~0))
3764                 {
3765                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3766                                                       rx_fib_index0,
3767                                                       thread_index, now, vm,
3768                                                       node);
3769                   if (!s0)
3770                     next0 = SNAT_IN2OUT_NEXT_DROP;
3771                   goto trace0;
3772                 }
3773
3774               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3775                 {
3776                   next0 = icmp_in2out_ed_slow_path
3777                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3778                      next0, now, thread_index, &s0);
3779                   goto trace0;
3780                 }
3781             }
3782           else
3783             {
3784               if (PREDICT_FALSE (proto0 == ~0))
3785                 {
3786                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3787                   goto trace0;
3788                 }
3789
3790               if (ip4_is_fragment (ip0))
3791                 {
3792                   next0 = SNAT_IN2OUT_NEXT_REASS;
3793                   goto trace0;
3794                 }
3795
3796               if (is_output_feature)
3797                {
3798                  if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3799                      sm, ip0, thread_index, now, vm, b0)))
3800                    goto trace0;
3801                }
3802
3803               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3804                 {
3805                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3806                   goto trace0;
3807                 }
3808             }
3809
3810           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3811                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3812
3813           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3814             {
3815               if (is_slow_path)
3816                 {
3817                   if (is_output_feature)
3818                     {
3819                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3820                           sm, ip0, ip0->protocol, udp0->src_port,
3821                           udp0->dst_port, thread_index, sw_if_index0,
3822                           vnet_buffer(b0)->sw_if_index[VLIB_TX])))
3823                         goto trace0;
3824                     }
3825                   else
3826                     {
3827                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3828                           sw_if_index0, ip0, proto0, rx_fib_index0,
3829                           thread_index)))
3830                         goto trace0;
3831                     }
3832
3833                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3834                                         next0, thread_index, now);
3835
3836                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3837                     goto trace0;
3838                 }
3839               else
3840                 {
3841                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3842                   goto trace0;
3843                 }
3844             }
3845           else
3846             {
3847               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3848             }
3849
3850           b0->flags |= VNET_BUFFER_F_IS_NATED;
3851
3852           if (!is_output_feature)
3853             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3854
3855           old_addr0 = ip0->src_address.as_u32;
3856           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3857           sum0 = ip0->checksum;
3858           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3859                                  src_address);
3860           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3861             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3862                                    s0->ext_host_addr.as_u32, ip4_header_t,
3863                                    dst_address);
3864           ip0->checksum = ip_csum_fold (sum0);
3865
3866           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3867             {
3868               old_port0 = tcp0->src_port;
3869               new_port0 = tcp0->src_port = s0->out2in.port;
3870
3871               sum0 = tcp0->checksum;
3872               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3873                                      dst_address);
3874               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3875                                      length);
3876               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3877                 {
3878                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3879                                          s0->ext_host_addr.as_u32,
3880                                          ip4_header_t, dst_address);
3881                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3882                                          s0->ext_host_port, ip4_header_t,
3883                                          length);
3884                   tcp0->dst_port = s0->ext_host_port;
3885                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3886                 }
3887               mss_clamping (sm, tcp0, &sum0);
3888               tcp0->checksum = ip_csum_fold(sum0);
3889               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3890                 goto trace0;
3891             }
3892           else
3893             {
3894               udp0->src_port = s0->out2in.port;
3895               udp0->checksum = 0;
3896               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3897                 {
3898                   udp0->dst_port = s0->ext_host_port;
3899                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3900                 }
3901             }
3902
3903           /* Accounting */
3904           nat44_session_update_counters (s0, now,
3905                                          vlib_buffer_length_in_chain (vm, b0));
3906
3907         trace0:
3908           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3909                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3910             {
3911               snat_in2out_trace_t *t =
3912                 vlib_add_trace (vm, node, b0, sizeof (*t));
3913               t->is_slow_path = is_slow_path;
3914               t->sw_if_index = sw_if_index0;
3915               t->next_index = next0;
3916               t->session_index = ~0;
3917               if (s0)
3918                 t->session_index = s0 - tsm->sessions;
3919             }
3920
3921           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3922
3923           /* verify speculative enqueue, maybe switch current next frame */
3924           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3925                                            to_next, n_left_to_next,
3926                                            bi0, next0);
3927         }
3928
3929       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3930     }
3931
3932   vlib_node_increment_counter (vm, stats_node_index,
3933                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3934                                pkts_processed);
3935   return frame->n_vectors;
3936 }
3937
3938 static uword
3939 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3940                               vlib_node_runtime_t * node,
3941                               vlib_frame_t * frame)
3942 {
3943   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3944 }
3945
3946 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3947   .function = nat44_ed_in2out_fast_path_fn,
3948   .name = "nat44-ed-in2out",
3949   .vector_size = sizeof (u32),
3950   .format_trace = format_snat_in2out_trace,
3951   .type = VLIB_NODE_TYPE_INTERNAL,
3952
3953   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3954   .error_strings = snat_in2out_error_strings,
3955
3956   .runtime_data_bytes = sizeof (snat_runtime_t),
3957
3958   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3959
3960   /* edit / add dispositions here */
3961   .next_nodes = {
3962     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3963     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3964     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3965     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3966     [SNAT_IN2OUT_NEXT_REASS] = "nat44-ed-in2out-reass",
3967   },
3968 };
3969
3970 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3971
3972 static uword
3973 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3974                                      vlib_node_runtime_t * node,
3975                                      vlib_frame_t * frame)
3976 {
3977   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3978 }
3979
3980 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3981   .function = nat44_ed_in2out_output_fast_path_fn,
3982   .name = "nat44-ed-in2out-output",
3983   .vector_size = sizeof (u32),
3984   .format_trace = format_snat_in2out_trace,
3985   .type = VLIB_NODE_TYPE_INTERNAL,
3986
3987   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3988   .error_strings = snat_in2out_error_strings,
3989
3990   .runtime_data_bytes = sizeof (snat_runtime_t),
3991
3992   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3993
3994   /* edit / add dispositions here */
3995   .next_nodes = {
3996     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3997     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3998     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3999     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4000     [SNAT_IN2OUT_NEXT_REASS] = "nat44-ed-in2out-reass-output",
4001   },
4002 };
4003
4004 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
4005                               nat44_ed_in2out_output_fast_path_fn);
4006
4007 static uword
4008 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
4009                               vlib_node_runtime_t * node,
4010                               vlib_frame_t * frame)
4011 {
4012   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
4013 }
4014
4015 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
4016   .function = nat44_ed_in2out_slow_path_fn,
4017   .name = "nat44-ed-in2out-slowpath",
4018   .vector_size = sizeof (u32),
4019   .format_trace = format_snat_in2out_trace,
4020   .type = VLIB_NODE_TYPE_INTERNAL,
4021
4022   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4023   .error_strings = snat_in2out_error_strings,
4024
4025   .runtime_data_bytes = sizeof (snat_runtime_t),
4026
4027   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4028
4029   /* edit / add dispositions here */
4030   .next_nodes = {
4031     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4032     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4033     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4034     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4035     [SNAT_IN2OUT_NEXT_REASS] = "nat44-ed-in2out-reass",
4036   },
4037 };
4038
4039 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
4040                               nat44_ed_in2out_slow_path_fn);
4041
4042 static uword
4043 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
4044                                      vlib_node_runtime_t * node,
4045                                      vlib_frame_t * frame)
4046 {
4047   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
4048 }
4049
4050 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
4051   .function = nat44_ed_in2out_output_slow_path_fn,
4052   .name = "nat44-ed-in2out-output-slowpath",
4053   .vector_size = sizeof (u32),
4054   .format_trace = format_snat_in2out_trace,
4055   .type = VLIB_NODE_TYPE_INTERNAL,
4056
4057   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4058   .error_strings = snat_in2out_error_strings,
4059
4060   .runtime_data_bytes = sizeof (snat_runtime_t),
4061
4062   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4063
4064   /* edit / add dispositions here */
4065   .next_nodes = {
4066     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4067     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
4068     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4069     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4070     [SNAT_IN2OUT_NEXT_REASS] = "nat44-ed-in2out-reass",
4071   },
4072 };
4073
4074 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
4075                               nat44_ed_in2out_output_slow_path_fn);
4076
4077 static inline uword
4078 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
4079                                       vlib_node_runtime_t * node,
4080                                       vlib_frame_t * frame,
4081                                       int is_output_feature)
4082 {
4083   u32 n_left_from, *from, *to_next;
4084   snat_in2out_next_t next_index;
4085   u32 pkts_processed = 0;
4086   snat_main_t *sm = &snat_main;
4087   f64 now = vlib_time_now (vm);
4088   u32 thread_index = vm->thread_index;
4089   snat_main_per_thread_data_t *per_thread_data =
4090     &sm->per_thread_data[thread_index];
4091   u32 *fragments_to_drop = 0;
4092   u32 *fragments_to_loopback = 0;
4093
4094   from = vlib_frame_vector_args (frame);
4095   n_left_from = frame->n_vectors;
4096   next_index = node->cached_next_index;
4097
4098   while (n_left_from > 0)
4099     {
4100       u32 n_left_to_next;
4101
4102       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
4103
4104       while (n_left_from > 0 && n_left_to_next > 0)
4105        {
4106           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
4107           u32 iph_offset0 = 0;
4108           vlib_buffer_t *b0;
4109           u32 next0;
4110           u8 cached0 = 0;
4111           ip4_header_t *ip0 = 0;
4112           nat_reass_ip4_t *reass0;
4113           udp_header_t * udp0;
4114           tcp_header_t * tcp0;
4115           icmp46_header_t * icmp0;
4116           clib_bihash_kv_16_8_t kv0, value0;
4117           snat_session_t * s0 = 0;
4118           u16 old_port0, new_port0;
4119           ip_csum_t sum0;
4120
4121           /* speculatively enqueue b0 to the current next frame */
4122           bi0 = from[0];
4123           to_next[0] = bi0;
4124           from += 1;
4125           to_next += 1;
4126           n_left_from -= 1;
4127           n_left_to_next -= 1;
4128
4129           b0 = vlib_get_buffer (vm, bi0);
4130
4131           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4132
4133           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4134           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
4135                                                                sw_if_index0);
4136
4137           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
4138             {
4139               next0 = SNAT_IN2OUT_NEXT_DROP;
4140               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
4141               goto trace0;
4142             }
4143
4144           if (is_output_feature)
4145             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
4146
4147           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
4148                  iph_offset0);
4149
4150           udp0 = ip4_next_header (ip0);
4151           tcp0 = (tcp_header_t *) udp0;
4152           icmp0 = (icmp46_header_t *) udp0;
4153           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4154
4155           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
4156                                                  ip0->dst_address,
4157                                                  ip0->fragment_id,
4158                                                  ip0->protocol,
4159                                                  1,
4160                                                  &fragments_to_drop);
4161
4162           if (PREDICT_FALSE (!reass0))
4163             {
4164               next0 = SNAT_IN2OUT_NEXT_DROP;
4165               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
4166               nat_log_notice ("maximum reassemblies exceeded");
4167               goto trace0;
4168             }
4169
4170           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
4171             {
4172               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4173                 {
4174                   if (is_output_feature)
4175                     {
4176                       if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
4177                           sm, ip0, thread_index, now, vm, b0)))
4178                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
4179                         goto trace0;
4180                     }
4181
4182                   next0 = icmp_in2out_slow_path
4183                       (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
4184                        next0, now, thread_index, &s0);
4185
4186                   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP))
4187                     {
4188                       if (s0)
4189                         reass0->sess_index = s0 - per_thread_data->sessions;
4190                       else
4191                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
4192                       nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
4193                     }
4194
4195                   goto trace0;
4196                 }
4197
4198               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
4199                           rx_fib_index0, udp0->src_port, udp0->dst_port);
4200
4201               if (clib_bihash_search_16_8 (&per_thread_data->in2out_ed, &kv0, &value0))
4202                 {
4203                   if (is_output_feature)
4204                     {
4205                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
4206                           sm, ip0, ip0->protocol, udp0->src_port,
4207                           udp0->dst_port, thread_index, sw_if_index0,
4208                           vnet_buffer(b0)->sw_if_index[VLIB_TX])))
4209                         {
4210                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
4211                           nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
4212                           goto trace0;
4213                         }
4214                     }
4215                   else
4216                     {
4217                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
4218                           sw_if_index0, ip0, proto0, rx_fib_index0,
4219                           thread_index)))
4220                         {
4221                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
4222                           nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
4223                           goto trace0;
4224                         }
4225                     }
4226
4227                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
4228                                         &s0, node, next0, thread_index, now);
4229
4230                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
4231                     goto trace0;
4232
4233                   reass0->sess_index = s0 - per_thread_data->sessions;
4234                 }
4235               else
4236                 {
4237                   s0 = pool_elt_at_index (per_thread_data->sessions,
4238                                           value0.value);
4239                   reass0->sess_index = value0.value;
4240                 }
4241               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
4242             }
4243           else
4244             {
4245               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
4246                 goto trace0;
4247               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
4248                 {
4249                   if (nat_ip4_reass_add_fragment (reass0, bi0, &fragments_to_drop))
4250                     {
4251                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
4252                       nat_log_notice ("maximum fragments per reassembly exceeded");
4253                       next0 = SNAT_IN2OUT_NEXT_DROP;
4254                       goto trace0;
4255                     }
4256                   cached0 = 1;
4257                   goto trace0;
4258                 }
4259               s0 = pool_elt_at_index (per_thread_data->sessions,
4260                                       reass0->sess_index);
4261             }
4262
4263           old_addr0 = ip0->src_address.as_u32;
4264           ip0->src_address = s0->out2in.addr;
4265           new_addr0 = ip0->src_address.as_u32;
4266           if (!is_output_feature)
4267             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
4268
4269           sum0 = ip0->checksum;
4270           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4271                                  ip4_header_t,
4272                                  src_address /* changed member */);
4273           if (PREDICT_FALSE (is_twice_nat_session (s0)))
4274             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
4275                                    s0->ext_host_addr.as_u32, ip4_header_t,
4276                                    dst_address);
4277           ip0->checksum = ip_csum_fold (sum0);
4278
4279           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
4280             {
4281               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4282                 {
4283                   old_port0 = tcp0->src_port;
4284                   tcp0->src_port = s0->out2in.port;
4285                   new_port0 = tcp0->src_port;
4286
4287                   sum0 = tcp0->checksum;
4288                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4289                                          ip4_header_t,
4290                                          dst_address /* changed member */);
4291                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4292                                          ip4_header_t /* cheat */,
4293                                          length /* changed member */);
4294                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
4295                     {
4296                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
4297                                              s0->ext_host_addr.as_u32,
4298                                              ip4_header_t, dst_address);
4299                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
4300                                              s0->ext_host_port, ip4_header_t,
4301                                              length);
4302                       tcp0->dst_port = s0->ext_host_port;
4303                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
4304                     }
4305                   tcp0->checksum = ip_csum_fold(sum0);
4306                 }
4307               else
4308                 {
4309                   old_port0 = udp0->src_port;
4310                   udp0->src_port = s0->out2in.port;
4311                   udp0->checksum = 0;
4312                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
4313                     {
4314                       udp0->dst_port = s0->ext_host_port;
4315                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
4316                     }
4317                 }
4318             }
4319
4320           /* Hairpinning */
4321           if (PREDICT_TRUE(proto0 != SNAT_PROTOCOL_ICMP))
4322             nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
4323                                      s0->ext_host_port, proto0, 1);
4324           else
4325             snat_icmp_hairpinning(sm, b0, ip0, icmp0, 1);
4326
4327           /* Accounting */
4328           nat44_session_update_counters (s0, now,
4329                                          vlib_buffer_length_in_chain (vm, b0));
4330           /* Per-user LRU list maintenance */
4331           nat44_session_update_lru (sm, s0, thread_index);
4332
4333         trace0:
4334           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4335                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4336             {
4337               nat44_in2out_reass_trace_t *t =
4338                  vlib_add_trace (vm, node, b0, sizeof (*t));
4339               t->cached = cached0;
4340               t->sw_if_index = sw_if_index0;
4341               t->next_index = next0;
4342             }
4343
4344           if (cached0)
4345             {
4346               n_left_to_next++;
4347               to_next--;
4348             }
4349           else
4350             {
4351               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4352
4353               /* verify speculative enqueue, maybe switch current next frame */
4354               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4355                                                to_next, n_left_to_next,
4356                                                bi0, next0);
4357             }
4358
4359           if (n_left_from == 0 && vec_len (fragments_to_loopback))
4360             {
4361               from = vlib_frame_vector_args (frame);
4362               u32 len = vec_len (fragments_to_loopback);
4363               if (len <= VLIB_FRAME_SIZE)
4364                 {
4365                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
4366                   n_left_from = len;
4367                   vec_reset_length (fragments_to_loopback);
4368                 }
4369               else
4370                 {
4371                   clib_memcpy (from,
4372                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
4373                                sizeof (u32) * VLIB_FRAME_SIZE);
4374                   n_left_from = VLIB_FRAME_SIZE;
4375                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
4376                 }
4377             }
4378        }
4379
4380       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4381     }
4382
4383   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
4384                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4385                                pkts_processed);
4386
4387   nat_send_all_to_node (vm, fragments_to_drop, node,
4388                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
4389                         SNAT_IN2OUT_NEXT_DROP);
4390
4391   vec_free (fragments_to_drop);
4392   vec_free (fragments_to_loopback);
4393   return frame->n_vectors;
4394 }
4395
4396 static uword
4397 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
4398                                vlib_node_runtime_t * node,
4399                                vlib_frame_t * frame)
4400 {
4401   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
4402 }
4403
4404 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
4405   .function = nat44_ed_in2out_reass_node_fn,
4406   .name = "nat44-ed-in2out-reass",
4407   .vector_size = sizeof (u32),
4408   .format_trace = format_nat44_in2out_reass_trace,
4409   .type = VLIB_NODE_TYPE_INTERNAL,
4410
4411   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4412   .error_strings = snat_in2out_error_strings,
4413
4414   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4415   .next_nodes = {
4416     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4417     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4418     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4419     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4420     [SNAT_IN2OUT_NEXT_REASS] = "nat44-ed-in2out-reass",
4421   },
4422 };
4423
4424 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
4425                               nat44_ed_in2out_reass_node_fn);
4426
4427 static uword
4428 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
4429                                       vlib_node_runtime_t * node,
4430                                       vlib_frame_t * frame)
4431 {
4432   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
4433 }
4434
4435 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
4436   .function = nat44_ed_in2out_reass_output_node_fn,
4437   .name = "nat44-ed-in2out-reass-output",
4438   .vector_size = sizeof (u32),
4439   .format_trace = format_nat44_in2out_reass_trace,
4440   .type = VLIB_NODE_TYPE_INTERNAL,
4441
4442   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4443   .error_strings = snat_in2out_error_strings,
4444
4445   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4446   .next_nodes = {
4447     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4448     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
4449     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4450     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4451     [SNAT_IN2OUT_NEXT_REASS] = "nat44-ed-in2out-reass",
4452   },
4453 };
4454
4455 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
4456                               nat44_ed_in2out_reass_output_node_fn);
4457
4458 /**************************/
4459 /*** deterministic mode ***/
4460 /**************************/
4461 static uword
4462 snat_det_in2out_node_fn (vlib_main_t * vm,
4463                          vlib_node_runtime_t * node,
4464                          vlib_frame_t * frame)
4465 {
4466   u32 n_left_from, * from, * to_next;
4467   snat_in2out_next_t next_index;
4468   u32 pkts_processed = 0;
4469   snat_main_t * sm = &snat_main;
4470   u32 now = (u32) vlib_time_now (vm);
4471   u32 thread_index = vm->thread_index;
4472
4473   from = vlib_frame_vector_args (frame);
4474   n_left_from = frame->n_vectors;
4475   next_index = node->cached_next_index;
4476
4477   while (n_left_from > 0)
4478     {
4479       u32 n_left_to_next;
4480
4481       vlib_get_next_frame (vm, node, next_index,
4482                            to_next, n_left_to_next);
4483
4484       while (n_left_from >= 4 && n_left_to_next >= 2)
4485         {
4486           u32 bi0, bi1;
4487           vlib_buffer_t * b0, * b1;
4488           u32 next0, next1;
4489           u32 sw_if_index0, sw_if_index1;
4490           ip4_header_t * ip0, * ip1;
4491           ip_csum_t sum0, sum1;
4492           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
4493           u16 old_port0, new_port0, lo_port0, i0;
4494           u16 old_port1, new_port1, lo_port1, i1;
4495           udp_header_t * udp0, * udp1;
4496           tcp_header_t * tcp0, * tcp1;
4497           u32 proto0, proto1;
4498           snat_det_out_key_t key0, key1;
4499           snat_det_map_t * dm0, * dm1;
4500           snat_det_session_t * ses0 = 0, * ses1 = 0;
4501           u32 rx_fib_index0, rx_fib_index1;
4502           icmp46_header_t * icmp0, * icmp1;
4503
4504           /* Prefetch next iteration. */
4505           {
4506             vlib_buffer_t * p2, * p3;
4507
4508             p2 = vlib_get_buffer (vm, from[2]);
4509             p3 = vlib_get_buffer (vm, from[3]);
4510
4511             vlib_prefetch_buffer_header (p2, LOAD);
4512             vlib_prefetch_buffer_header (p3, LOAD);
4513
4514             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
4515             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
4516           }
4517
4518           /* speculatively enqueue b0 and b1 to the current next frame */
4519           to_next[0] = bi0 = from[0];
4520           to_next[1] = bi1 = from[1];
4521           from += 2;
4522           to_next += 2;
4523           n_left_from -= 2;
4524           n_left_to_next -= 2;
4525
4526           b0 = vlib_get_buffer (vm, bi0);
4527           b1 = vlib_get_buffer (vm, bi1);
4528
4529           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4530           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
4531
4532           ip0 = vlib_buffer_get_current (b0);
4533           udp0 = ip4_next_header (ip0);
4534           tcp0 = (tcp_header_t *) udp0;
4535
4536           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4537
4538           if (PREDICT_FALSE(ip0->ttl == 1))
4539             {
4540               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4541               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4542                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4543                                            0);
4544               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4545               goto trace0;
4546             }
4547
4548           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4549
4550           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4551             {
4552               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4553               icmp0 = (icmp46_header_t *) udp0;
4554
4555               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4556                                   rx_fib_index0, node, next0, thread_index,
4557                                   &ses0, &dm0);
4558               goto trace0;
4559             }
4560
4561           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4562           if (PREDICT_FALSE(!dm0))
4563             {
4564               nat_log_info ("no match for internal host %U",
4565                             format_ip4_address, &ip0->src_address);
4566               next0 = SNAT_IN2OUT_NEXT_DROP;
4567               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4568               goto trace0;
4569             }
4570
4571           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4572
4573           key0.ext_host_addr = ip0->dst_address;
4574           key0.ext_host_port = tcp0->dst;
4575
4576           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4577           if (PREDICT_FALSE(!ses0))
4578             {
4579               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4580                 {
4581                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
4582                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4583
4584                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4585                     continue;
4586
4587                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4588                   break;
4589                 }
4590               if (PREDICT_FALSE(!ses0))
4591                 {
4592                   /* too many sessions for user, send ICMP error packet */
4593
4594                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4595                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4596                                                ICMP4_destination_unreachable_destination_unreachable_host,
4597                                                0);
4598                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4599                   goto trace0;
4600                 }
4601             }
4602
4603           new_port0 = ses0->out.out_port;
4604
4605           old_addr0.as_u32 = ip0->src_address.as_u32;
4606           ip0->src_address.as_u32 = new_addr0.as_u32;
4607           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4608
4609           sum0 = ip0->checksum;
4610           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4611                                  ip4_header_t,
4612                                  src_address /* changed member */);
4613           ip0->checksum = ip_csum_fold (sum0);
4614
4615           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4616             {
4617               if (tcp0->flags & TCP_FLAG_SYN)
4618                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4619               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4620                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4621               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4622                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4623               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4624                 snat_det_ses_close(dm0, ses0);
4625               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4626                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4627               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4628                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4629
4630               old_port0 = tcp0->src;
4631               tcp0->src = new_port0;
4632
4633               sum0 = tcp0->checksum;
4634               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4635                                      ip4_header_t,
4636                                      dst_address /* changed member */);
4637               sum0 = ip_csum_update (sum0, old_port0, new_port0,
4638                                      ip4_header_t /* cheat */,
4639                                      length /* changed member */);
4640               mss_clamping (sm, tcp0, &sum0);
4641               tcp0->checksum = ip_csum_fold(sum0);
4642             }
4643           else
4644             {
4645               ses0->state = SNAT_SESSION_UDP_ACTIVE;
4646               old_port0 = udp0->src_port;
4647               udp0->src_port = new_port0;
4648               udp0->checksum = 0;
4649             }
4650
4651           switch(ses0->state)
4652             {
4653             case SNAT_SESSION_UDP_ACTIVE:
4654                 ses0->expire = now + sm->udp_timeout;
4655                 break;
4656             case SNAT_SESSION_TCP_SYN_SENT:
4657             case SNAT_SESSION_TCP_FIN_WAIT:
4658             case SNAT_SESSION_TCP_CLOSE_WAIT:
4659             case SNAT_SESSION_TCP_LAST_ACK:
4660                 ses0->expire = now + sm->tcp_transitory_timeout;
4661                 break;
4662             case SNAT_SESSION_TCP_ESTABLISHED:
4663                 ses0->expire = now + sm->tcp_established_timeout;
4664                 break;
4665             }
4666
4667         trace0:
4668           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4669                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4670             {
4671               snat_in2out_trace_t *t =
4672                  vlib_add_trace (vm, node, b0, sizeof (*t));
4673               t->is_slow_path = 0;
4674               t->sw_if_index = sw_if_index0;
4675               t->next_index = next0;
4676               t->session_index = ~0;
4677               if (ses0)
4678                 t->session_index = ses0 - dm0->sessions;
4679             }
4680
4681           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4682
4683           ip1 = vlib_buffer_get_current (b1);
4684           udp1 = ip4_next_header (ip1);
4685           tcp1 = (tcp_header_t *) udp1;
4686
4687           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4688
4689           if (PREDICT_FALSE(ip1->ttl == 1))
4690             {
4691               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4692               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4693                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4694                                            0);
4695               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4696               goto trace1;
4697             }
4698
4699           proto1 = ip_proto_to_snat_proto (ip1->protocol);
4700
4701           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4702             {
4703               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4704               icmp1 = (icmp46_header_t *) udp1;
4705
4706               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4707                                   rx_fib_index1, node, next1, thread_index,
4708                                   &ses1, &dm1);
4709               goto trace1;
4710             }
4711
4712           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4713           if (PREDICT_FALSE(!dm1))
4714             {
4715               nat_log_info ("no match for internal host %U",
4716                             format_ip4_address, &ip0->src_address);
4717               next1 = SNAT_IN2OUT_NEXT_DROP;
4718               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4719               goto trace1;
4720             }
4721
4722           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4723
4724           key1.ext_host_addr = ip1->dst_address;
4725           key1.ext_host_port = tcp1->dst;
4726
4727           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4728           if (PREDICT_FALSE(!ses1))
4729             {
4730               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4731                 {
4732                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
4733                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4734
4735                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4736                     continue;
4737
4738                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4739                   break;
4740                 }
4741               if (PREDICT_FALSE(!ses1))
4742                 {
4743                   /* too many sessions for user, send ICMP error packet */
4744
4745                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4746                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4747                                                ICMP4_destination_unreachable_destination_unreachable_host,
4748                                                0);
4749                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4750                   goto trace1;
4751                 }
4752             }
4753
4754           new_port1 = ses1->out.out_port;
4755
4756           old_addr1.as_u32 = ip1->src_address.as_u32;
4757           ip1->src_address.as_u32 = new_addr1.as_u32;
4758           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4759
4760           sum1 = ip1->checksum;
4761           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4762                                  ip4_header_t,
4763                                  src_address /* changed member */);
4764           ip1->checksum = ip_csum_fold (sum1);
4765
4766           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4767             {
4768               if (tcp1->flags & TCP_FLAG_SYN)
4769                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4770               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4771                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4772               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4773                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4774               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4775                 snat_det_ses_close(dm1, ses1);
4776               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4777                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4778               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4779                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4780
4781               old_port1 = tcp1->src;
4782               tcp1->src = new_port1;
4783
4784               sum1 = tcp1->checksum;
4785               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4786                                      ip4_header_t,
4787                                      dst_address /* changed member */);
4788               sum1 = ip_csum_update (sum1, old_port1, new_port1,
4789                                      ip4_header_t /* cheat */,
4790                                      length /* changed member */);
4791               mss_clamping (sm, tcp1, &sum1);
4792               tcp1->checksum = ip_csum_fold(sum1);
4793             }
4794           else
4795             {
4796               ses1->state = SNAT_SESSION_UDP_ACTIVE;
4797               old_port1 = udp1->src_port;
4798               udp1->src_port = new_port1;
4799               udp1->checksum = 0;
4800             }
4801
4802           switch(ses1->state)
4803             {
4804             case SNAT_SESSION_UDP_ACTIVE:
4805                 ses1->expire = now + sm->udp_timeout;
4806                 break;
4807             case SNAT_SESSION_TCP_SYN_SENT:
4808             case SNAT_SESSION_TCP_FIN_WAIT:
4809             case SNAT_SESSION_TCP_CLOSE_WAIT:
4810             case SNAT_SESSION_TCP_LAST_ACK:
4811                 ses1->expire = now + sm->tcp_transitory_timeout;
4812                 break;
4813             case SNAT_SESSION_TCP_ESTABLISHED:
4814                 ses1->expire = now + sm->tcp_established_timeout;
4815                 break;
4816             }
4817
4818         trace1:
4819           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4820                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4821             {
4822               snat_in2out_trace_t *t =
4823                  vlib_add_trace (vm, node, b1, sizeof (*t));
4824               t->is_slow_path = 0;
4825               t->sw_if_index = sw_if_index1;
4826               t->next_index = next1;
4827               t->session_index = ~0;
4828               if (ses1)
4829                 t->session_index = ses1 - dm1->sessions;
4830             }
4831
4832           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4833
4834           /* verify speculative enqueues, maybe switch current next frame */
4835           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4836                                            to_next, n_left_to_next,
4837                                            bi0, bi1, next0, next1);
4838          }
4839
4840       while (n_left_from > 0 && n_left_to_next > 0)
4841         {
4842           u32 bi0;
4843           vlib_buffer_t * b0;
4844           u32 next0;
4845           u32 sw_if_index0;
4846           ip4_header_t * ip0;
4847           ip_csum_t sum0;
4848           ip4_address_t new_addr0, old_addr0;
4849           u16 old_port0, new_port0, lo_port0, i0;
4850           udp_header_t * udp0;
4851           tcp_header_t * tcp0;
4852           u32 proto0;
4853           snat_det_out_key_t key0;
4854           snat_det_map_t * dm0;
4855           snat_det_session_t * ses0 = 0;
4856           u32 rx_fib_index0;
4857           icmp46_header_t * icmp0;
4858
4859           /* speculatively enqueue b0 to the current next frame */
4860           bi0 = from[0];
4861           to_next[0] = bi0;
4862           from += 1;
4863           to_next += 1;
4864           n_left_from -= 1;
4865           n_left_to_next -= 1;
4866
4867           b0 = vlib_get_buffer (vm, bi0);
4868           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4869
4870           ip0 = vlib_buffer_get_current (b0);
4871           udp0 = ip4_next_header (ip0);
4872           tcp0 = (tcp_header_t *) udp0;
4873
4874           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4875
4876           if (PREDICT_FALSE(ip0->ttl == 1))
4877             {
4878               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4879               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4880                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4881                                            0);
4882               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4883               goto trace00;
4884             }
4885
4886           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4887
4888           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4889             {
4890               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4891               icmp0 = (icmp46_header_t *) udp0;
4892
4893               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4894                                   rx_fib_index0, node, next0, thread_index,
4895                                   &ses0, &dm0);
4896               goto trace00;
4897             }
4898
4899           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4900           if (PREDICT_FALSE(!dm0))
4901             {
4902               nat_log_info ("no match for internal host %U",
4903                             format_ip4_address, &ip0->src_address);
4904               next0 = SNAT_IN2OUT_NEXT_DROP;
4905               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4906               goto trace00;
4907             }
4908
4909           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4910
4911           key0.ext_host_addr = ip0->dst_address;
4912           key0.ext_host_port = tcp0->dst;
4913
4914           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4915           if (PREDICT_FALSE(!ses0))
4916             {
4917               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4918                 {
4919                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
4920                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4921
4922                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4923                     continue;
4924
4925                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4926                   break;
4927                 }
4928               if (PREDICT_FALSE(!ses0))
4929                 {
4930                   /* too many sessions for user, send ICMP error packet */
4931
4932                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4933                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4934                                                ICMP4_destination_unreachable_destination_unreachable_host,
4935                                                0);
4936                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4937                   goto trace00;
4938                 }
4939             }
4940
4941           new_port0 = ses0->out.out_port;
4942
4943           old_addr0.as_u32 = ip0->src_address.as_u32;
4944           ip0->src_address.as_u32 = new_addr0.as_u32;
4945           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4946
4947           sum0 = ip0->checksum;
4948           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4949                                  ip4_header_t,
4950                                  src_address /* changed member */);
4951           ip0->checksum = ip_csum_fold (sum0);
4952
4953           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4954             {
4955               if (tcp0->flags & TCP_FLAG_SYN)
4956                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4957               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4958                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4959               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4960                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4961               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4962                 snat_det_ses_close(dm0, ses0);
4963               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4964                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4965               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4966                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4967
4968               old_port0 = tcp0->src;
4969               tcp0->src = new_port0;
4970
4971               sum0 = tcp0->checksum;
4972               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4973                                      ip4_header_t,
4974                                      dst_address /* changed member */);
4975               sum0 = ip_csum_update (sum0, old_port0, new_port0,
4976                                      ip4_header_t /* cheat */,
4977                                      length /* changed member */);
4978               mss_clamping (sm, tcp0, &sum0);
4979               tcp0->checksum = ip_csum_fold(sum0);
4980             }
4981           else
4982             {
4983               ses0->state = SNAT_SESSION_UDP_ACTIVE;
4984               old_port0 = udp0->src_port;
4985               udp0->src_port = new_port0;
4986               udp0->checksum = 0;
4987             }
4988
4989           switch(ses0->state)
4990             {
4991             case SNAT_SESSION_UDP_ACTIVE:
4992                 ses0->expire = now + sm->udp_timeout;
4993                 break;
4994             case SNAT_SESSION_TCP_SYN_SENT:
4995             case SNAT_SESSION_TCP_FIN_WAIT:
4996             case SNAT_SESSION_TCP_CLOSE_WAIT:
4997             case SNAT_SESSION_TCP_LAST_ACK:
4998                 ses0->expire = now + sm->tcp_transitory_timeout;
4999                 break;
5000             case SNAT_SESSION_TCP_ESTABLISHED:
5001                 ses0->expire = now + sm->tcp_established_timeout;
5002                 break;
5003             }
5004
5005         trace00:
5006           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5007                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5008             {
5009               snat_in2out_trace_t *t =
5010                  vlib_add_trace (vm, node, b0, sizeof (*t));
5011               t->is_slow_path = 0;
5012               t->sw_if_index = sw_if_index0;
5013               t->next_index = next0;
5014               t->session_index = ~0;
5015               if (ses0)
5016                 t->session_index = ses0 - dm0->sessions;
5017             }
5018
5019           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5020
5021           /* verify speculative enqueue, maybe switch current next frame */
5022           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5023                                            to_next, n_left_to_next,
5024                                            bi0, next0);
5025         }
5026
5027       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5028     }
5029
5030   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
5031                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5032                                pkts_processed);
5033   return frame->n_vectors;
5034 }
5035
5036 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
5037   .function = snat_det_in2out_node_fn,
5038   .name = "nat44-det-in2out",
5039   .vector_size = sizeof (u32),
5040   .format_trace = format_snat_in2out_trace,
5041   .type = VLIB_NODE_TYPE_INTERNAL,
5042
5043   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5044   .error_strings = snat_in2out_error_strings,
5045
5046   .runtime_data_bytes = sizeof (snat_runtime_t),
5047
5048   .n_next_nodes = 3,
5049
5050   /* edit / add dispositions here */
5051   .next_nodes = {
5052     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5053     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5054     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5055   },
5056 };
5057
5058 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
5059
5060 /**
5061  * Get address and port values to be used for ICMP packet translation
5062  * and create session if needed
5063  *
5064  * @param[in,out] sm             NAT main
5065  * @param[in,out] node           NAT node runtime
5066  * @param[in] thread_index       thread index
5067  * @param[in,out] b0             buffer containing packet to be translated
5068  * @param[out] p_proto           protocol used for matching
5069  * @param[out] p_value           address and port after NAT translation
5070  * @param[out] p_dont_translate  if packet should not be translated
5071  * @param d                      optional parameter
5072  * @param e                      optional parameter
5073  */
5074 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
5075                           u32 thread_index, vlib_buffer_t *b0,
5076                           ip4_header_t *ip0, u8 *p_proto,
5077                           snat_session_key_t *p_value,
5078                           u8 *p_dont_translate, void *d, void *e)
5079 {
5080   icmp46_header_t *icmp0;
5081   u32 sw_if_index0;
5082   u32 rx_fib_index0;
5083   u8 protocol;
5084   snat_det_out_key_t key0;
5085   u8 dont_translate = 0;
5086   u32 next0 = ~0;
5087   icmp_echo_header_t *echo0, *inner_echo0 = 0;
5088   ip4_header_t *inner_ip0;
5089   void *l4_header = 0;
5090   icmp46_header_t *inner_icmp0;
5091   snat_det_map_t * dm0 = 0;
5092   ip4_address_t new_addr0;
5093   u16 lo_port0, i0;
5094   snat_det_session_t * ses0 = 0;
5095   ip4_address_t in_addr;
5096   u16 in_port;
5097
5098   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
5099   echo0 = (icmp_echo_header_t *)(icmp0+1);
5100   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5101   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
5102
5103   if (!icmp_is_error_message (icmp0))
5104     {
5105       protocol = SNAT_PROTOCOL_ICMP;
5106       in_addr = ip0->src_address;
5107       in_port = echo0->identifier;
5108     }
5109   else
5110     {
5111       inner_ip0 = (ip4_header_t *)(echo0+1);
5112       l4_header = ip4_next_header (inner_ip0);
5113       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
5114       in_addr = inner_ip0->dst_address;
5115       switch (protocol)
5116         {
5117         case SNAT_PROTOCOL_ICMP:
5118           inner_icmp0 = (icmp46_header_t*)l4_header;
5119           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
5120           in_port = inner_echo0->identifier;
5121           break;
5122         case SNAT_PROTOCOL_UDP:
5123         case SNAT_PROTOCOL_TCP:
5124           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
5125           break;
5126         default:
5127           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
5128           next0 = SNAT_IN2OUT_NEXT_DROP;
5129           goto out;
5130         }
5131     }
5132
5133   dm0 = snat_det_map_by_user(sm, &in_addr);
5134   if (PREDICT_FALSE(!dm0))
5135     {
5136       nat_log_info ("no match for internal host %U",
5137                     format_ip4_address, &in_addr);
5138       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
5139           IP_PROTOCOL_ICMP, rx_fib_index0)))
5140         {
5141           dont_translate = 1;
5142           goto out;
5143         }
5144       next0 = SNAT_IN2OUT_NEXT_DROP;
5145       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5146       goto out;
5147     }
5148
5149   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
5150
5151   key0.ext_host_addr = ip0->dst_address;
5152   key0.ext_host_port = 0;
5153
5154   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
5155   if (PREDICT_FALSE(!ses0))
5156     {
5157       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
5158           IP_PROTOCOL_ICMP, rx_fib_index0)))
5159         {
5160           dont_translate = 1;
5161           goto out;
5162         }
5163       if (icmp0->type != ICMP4_echo_request)
5164         {
5165           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
5166           next0 = SNAT_IN2OUT_NEXT_DROP;
5167           goto out;
5168         }
5169       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
5170         {
5171           key0.out_port = clib_host_to_net_u16 (lo_port0 +
5172             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
5173
5174           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
5175             continue;
5176
5177           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
5178           break;
5179         }
5180       if (PREDICT_FALSE(!ses0))
5181         {
5182           next0 = SNAT_IN2OUT_NEXT_DROP;
5183           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
5184           goto out;
5185         }
5186     }
5187
5188   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
5189                     !icmp_is_error_message (icmp0)))
5190     {
5191       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
5192       next0 = SNAT_IN2OUT_NEXT_DROP;
5193       goto out;
5194     }
5195
5196   u32 now = (u32) vlib_time_now (sm->vlib_main);
5197
5198   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
5199   ses0->expire = now + sm->icmp_timeout;
5200
5201 out:
5202   *p_proto = protocol;
5203   if (ses0)
5204     {
5205       p_value->addr = new_addr0;
5206       p_value->fib_index = sm->outside_fib_index;
5207       p_value->port = ses0->out.out_port;
5208     }
5209   *p_dont_translate = dont_translate;
5210   if (d)
5211     *(snat_det_session_t**)d = ses0;
5212   if (e)
5213     *(snat_det_map_t**)e = dm0;
5214   return next0;
5215 }
5216
5217 /**********************/
5218 /*** worker handoff ***/
5219 /**********************/
5220 static inline uword
5221 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
5222                                       vlib_node_runtime_t * node,
5223                                       vlib_frame_t * frame,
5224                                       u8 is_output)
5225 {
5226   snat_main_t *sm = &snat_main;
5227   vlib_thread_main_t *tm = vlib_get_thread_main ();
5228   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
5229   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
5230   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
5231     = 0;
5232   vlib_frame_queue_elt_t *hf = 0;
5233   vlib_frame_queue_t *fq;
5234   vlib_frame_t *f = 0;
5235   int i;
5236   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
5237   u32 next_worker_index = 0;
5238   u32 current_worker_index = ~0;
5239   u32 thread_index = vm->thread_index;
5240   u32 fq_index;
5241   u32 to_node_index;
5242   vlib_frame_t *d = 0;
5243
5244   ASSERT (vec_len (sm->workers));
5245
5246   if (is_output)
5247     {
5248       fq_index = sm->fq_in2out_output_index;
5249       to_node_index = sm->in2out_output_node_index;
5250     }
5251   else
5252     {
5253       fq_index = sm->fq_in2out_index;
5254       to_node_index = sm->in2out_node_index;
5255     }
5256
5257   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
5258     {
5259       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
5260
5261       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
5262                                tm->n_vlib_mains - 1,
5263                                (vlib_frame_queue_t *) (~0));
5264     }
5265
5266   from = vlib_frame_vector_args (frame);
5267   n_left_from = frame->n_vectors;
5268
5269   while (n_left_from > 0)
5270     {
5271       u32 bi0;
5272       vlib_buffer_t *b0;
5273       u32 sw_if_index0;
5274       u32 rx_fib_index0;
5275       ip4_header_t * ip0;
5276       u8 do_handoff;
5277
5278       bi0 = from[0];
5279       from += 1;
5280       n_left_from -= 1;
5281
5282       b0 = vlib_get_buffer (vm, bi0);
5283
5284       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
5285       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5286
5287       ip0 = vlib_buffer_get_current (b0);
5288
5289       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
5290
5291       if (PREDICT_FALSE (next_worker_index != thread_index))
5292         {
5293           do_handoff = 1;
5294
5295           if (next_worker_index != current_worker_index)
5296             {
5297               fq = is_vlib_frame_queue_congested (
5298                 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
5299                 congested_handoff_queue_by_worker_index);
5300
5301               if (fq)
5302                 {
5303                   /* if this is 1st frame */
5304                   if (!d)
5305                     {
5306                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
5307                       to_next_drop = vlib_frame_vector_args (d);
5308                     }
5309
5310                   to_next_drop[0] = bi0;
5311                   to_next_drop += 1;
5312                   d->n_vectors++;
5313                   b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
5314                   goto trace0;
5315                 }
5316
5317               if (hf)
5318                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
5319
5320               hf = vlib_get_worker_handoff_queue_elt (fq_index,
5321                                                       next_worker_index,
5322                                                       handoff_queue_elt_by_worker_index);
5323
5324               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
5325               to_next_worker = &hf->buffer_index[hf->n_vectors];
5326               current_worker_index = next_worker_index;
5327             }
5328
5329           /* enqueue to correct worker thread */
5330           to_next_worker[0] = bi0;
5331           to_next_worker++;
5332           n_left_to_next_worker--;
5333
5334           if (n_left_to_next_worker == 0)
5335             {
5336               hf->n_vectors = VLIB_FRAME_SIZE;
5337               vlib_put_frame_queue_elt (hf);
5338               current_worker_index = ~0;
5339               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
5340               hf = 0;
5341             }
5342         }
5343       else
5344         {
5345           do_handoff = 0;
5346           /* if this is 1st frame */
5347           if (!f)
5348             {
5349               f = vlib_get_frame_to_node (vm, to_node_index);
5350               to_next = vlib_frame_vector_args (f);
5351             }
5352
5353           to_next[0] = bi0;
5354           to_next += 1;
5355           f->n_vectors++;
5356         }
5357
5358 trace0:
5359       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
5360                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5361         {
5362           snat_in2out_worker_handoff_trace_t *t =
5363             vlib_add_trace (vm, node, b0, sizeof (*t));
5364           t->next_worker_index = next_worker_index;
5365           t->do_handoff = do_handoff;
5366         }
5367     }
5368
5369   if (f)
5370     vlib_put_frame_to_node (vm, to_node_index, f);
5371
5372   if (d)
5373     vlib_put_frame_to_node (vm, sm->error_node_index, d);
5374
5375   if (hf)
5376     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
5377
5378   /* Ship frames to the worker nodes */
5379   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
5380     {
5381       if (handoff_queue_elt_by_worker_index[i])
5382         {
5383           hf = handoff_queue_elt_by_worker_index[i];
5384           /*
5385            * It works better to let the handoff node
5386            * rate-adapt, always ship the handoff queue element.
5387            */
5388           if (1 || hf->n_vectors == hf->last_n_vectors)
5389             {
5390               vlib_put_frame_queue_elt (hf);
5391               handoff_queue_elt_by_worker_index[i] = 0;
5392             }
5393           else
5394             hf->last_n_vectors = hf->n_vectors;
5395         }
5396       congested_handoff_queue_by_worker_index[i] =
5397         (vlib_frame_queue_t *) (~0);
5398     }
5399   hf = 0;
5400   current_worker_index = ~0;
5401   return frame->n_vectors;
5402 }
5403
5404 static uword
5405 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
5406                                vlib_node_runtime_t * node,
5407                                vlib_frame_t * frame)
5408 {
5409   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
5410 }
5411
5412 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
5413   .function = snat_in2out_worker_handoff_fn,
5414   .name = "nat44-in2out-worker-handoff",
5415   .vector_size = sizeof (u32),
5416   .format_trace = format_snat_in2out_worker_handoff_trace,
5417   .type = VLIB_NODE_TYPE_INTERNAL,
5418
5419   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5420   .error_strings = snat_in2out_error_strings,
5421
5422   .n_next_nodes = 1,
5423
5424   .next_nodes = {
5425     [0] = "error-drop",
5426   },
5427 };
5428
5429 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
5430                               snat_in2out_worker_handoff_fn);
5431
5432 static uword
5433 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
5434                                       vlib_node_runtime_t * node,
5435                                       vlib_frame_t * frame)
5436 {
5437   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
5438 }
5439
5440 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
5441   .function = snat_in2out_output_worker_handoff_fn,
5442   .name = "nat44-in2out-output-worker-handoff",
5443   .vector_size = sizeof (u32),
5444   .format_trace = format_snat_in2out_worker_handoff_trace,
5445   .type = VLIB_NODE_TYPE_INTERNAL,
5446
5447   .n_next_nodes = 1,
5448
5449   .next_nodes = {
5450     [0] = "error-drop",
5451   },
5452 };
5453
5454 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
5455                               snat_in2out_output_worker_handoff_fn);
5456
5457 static_always_inline int
5458 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
5459 {
5460   snat_address_t * ap;
5461   clib_bihash_kv_8_8_t kv, value;
5462   snat_session_key_t m_key;
5463
5464   vec_foreach (ap, sm->addresses)
5465     {
5466       if (ap->addr.as_u32 == dst_addr->as_u32)
5467         return 1;
5468     }
5469
5470   m_key.addr.as_u32 = dst_addr->as_u32;
5471   m_key.fib_index = 0;
5472   m_key.port = 0;
5473   m_key.protocol = 0;
5474   kv.key = m_key.as_u64;
5475   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
5476     return 1;
5477
5478   return 0;
5479 }
5480
5481 static inline uword
5482 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
5483                             vlib_node_runtime_t * node,
5484                             vlib_frame_t * frame,
5485                             int is_ed)
5486 {
5487   u32 n_left_from, * from, * to_next, stats_node_index;
5488   snat_in2out_next_t next_index;
5489   u32 pkts_processed = 0;
5490   snat_main_t * sm = &snat_main;
5491
5492   stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
5493     snat_hairpin_dst_node.index;
5494
5495   from = vlib_frame_vector_args (frame);
5496   n_left_from = frame->n_vectors;
5497   next_index = node->cached_next_index;
5498
5499   while (n_left_from > 0)
5500     {
5501       u32 n_left_to_next;
5502
5503       vlib_get_next_frame (vm, node, next_index,
5504                            to_next, n_left_to_next);
5505
5506       while (n_left_from > 0 && n_left_to_next > 0)
5507         {
5508           u32 bi0;
5509           vlib_buffer_t * b0;
5510           u32 next0;
5511           ip4_header_t * ip0;
5512           u32 proto0;
5513
5514           /* speculatively enqueue b0 to the current next frame */
5515           bi0 = from[0];
5516           to_next[0] = bi0;
5517           from += 1;
5518           to_next += 1;
5519           n_left_from -= 1;
5520           n_left_to_next -= 1;
5521
5522           b0 = vlib_get_buffer (vm, bi0);
5523           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5524           ip0 = vlib_buffer_get_current (b0);
5525
5526           proto0 = ip_proto_to_snat_proto (ip0->protocol);
5527
5528           vnet_buffer (b0)->snat.flags = 0;
5529           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
5530             {
5531               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
5532                 {
5533                   udp_header_t * udp0 = ip4_next_header (ip0);
5534                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
5535
5536                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
5537                 }
5538               else if (proto0 == SNAT_PROTOCOL_ICMP)
5539                 {
5540                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
5541
5542                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
5543                 }
5544               else
5545                 {
5546                   if (is_ed)
5547                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
5548                   else
5549                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
5550                 }
5551
5552               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
5553             }
5554
5555           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5556
5557           /* verify speculative enqueue, maybe switch current next frame */
5558           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5559                                            to_next, n_left_to_next,
5560                                            bi0, next0);
5561          }
5562
5563       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5564     }
5565
5566   vlib_node_increment_counter (vm, stats_node_index,
5567                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5568                                pkts_processed);
5569   return frame->n_vectors;
5570 }
5571
5572 static uword
5573 snat_hairpin_dst_fn (vlib_main_t * vm,
5574                      vlib_node_runtime_t * node,
5575                      vlib_frame_t * frame)
5576 {
5577   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
5578 }
5579
5580 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
5581   .function = snat_hairpin_dst_fn,
5582   .name = "nat44-hairpin-dst",
5583   .vector_size = sizeof (u32),
5584   .type = VLIB_NODE_TYPE_INTERNAL,
5585   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5586   .error_strings = snat_in2out_error_strings,
5587   .n_next_nodes = 2,
5588   .next_nodes = {
5589     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5590     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5591   },
5592 };
5593
5594 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
5595                               snat_hairpin_dst_fn);
5596
5597 static uword
5598 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
5599                          vlib_node_runtime_t * node,
5600                          vlib_frame_t * frame)
5601 {
5602   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
5603 }
5604
5605 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
5606   .function = nat44_ed_hairpin_dst_fn,
5607   .name = "nat44-ed-hairpin-dst",
5608   .vector_size = sizeof (u32),
5609   .type = VLIB_NODE_TYPE_INTERNAL,
5610   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5611   .error_strings = snat_in2out_error_strings,
5612   .n_next_nodes = 2,
5613   .next_nodes = {
5614     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5615     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5616   },
5617 };
5618
5619 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
5620                               nat44_ed_hairpin_dst_fn);
5621
5622 static inline uword
5623 snat_hairpin_src_fn_inline (vlib_main_t * vm,
5624                             vlib_node_runtime_t * node,
5625                             vlib_frame_t * frame,
5626                             int is_ed)
5627 {
5628   u32 n_left_from, * from, * to_next, stats_node_index;
5629   snat_in2out_next_t next_index;
5630   u32 pkts_processed = 0;
5631   snat_main_t *sm = &snat_main;
5632
5633   stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
5634     snat_hairpin_src_node.index;
5635
5636   from = vlib_frame_vector_args (frame);
5637   n_left_from = frame->n_vectors;
5638   next_index = node->cached_next_index;
5639
5640   while (n_left_from > 0)
5641     {
5642       u32 n_left_to_next;
5643
5644       vlib_get_next_frame (vm, node, next_index,
5645                            to_next, n_left_to_next);
5646
5647       while (n_left_from > 0 && n_left_to_next > 0)
5648         {
5649           u32 bi0;
5650           vlib_buffer_t * b0;
5651           u32 next0;
5652           snat_interface_t *i;
5653           u32 sw_if_index0;
5654
5655           /* speculatively enqueue b0 to the current next frame */
5656           bi0 = from[0];
5657           to_next[0] = bi0;
5658           from += 1;
5659           to_next += 1;
5660           n_left_from -= 1;
5661           n_left_to_next -= 1;
5662
5663           b0 = vlib_get_buffer (vm, bi0);
5664           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5665           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
5666
5667           pool_foreach (i, sm->output_feature_interfaces,
5668           ({
5669             /* Only packets from NAT inside interface */
5670             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
5671               {
5672                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
5673                                     SNAT_FLAG_HAIRPINNING))
5674                   {
5675                     if (PREDICT_TRUE (sm->num_workers > 1))
5676                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5677                     else
5678                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5679                   }
5680                 break;
5681               }
5682           }));
5683
5684           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5685
5686           /* verify speculative enqueue, maybe switch current next frame */
5687           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5688                                            to_next, n_left_to_next,
5689                                            bi0, next0);
5690          }
5691
5692       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5693     }
5694
5695   vlib_node_increment_counter (vm, stats_node_index,
5696                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5697                                pkts_processed);
5698   return frame->n_vectors;
5699 }
5700
5701 static uword
5702 snat_hairpin_src_fn (vlib_main_t * vm,
5703                      vlib_node_runtime_t * node,
5704                      vlib_frame_t * frame)
5705 {
5706   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5707 }
5708
5709 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5710   .function = snat_hairpin_src_fn,
5711   .name = "nat44-hairpin-src",
5712   .vector_size = sizeof (u32),
5713   .type = VLIB_NODE_TYPE_INTERNAL,
5714   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5715   .error_strings = snat_in2out_error_strings,
5716   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5717   .next_nodes = {
5718      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5719      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5720      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5721      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5722   },
5723 };
5724
5725 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5726                               snat_hairpin_src_fn);
5727
5728 static uword
5729 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5730                          vlib_node_runtime_t * node,
5731                          vlib_frame_t * frame)
5732 {
5733   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5734 }
5735
5736 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5737   .function = nat44_ed_hairpin_src_fn,
5738   .name = "nat44-ed-hairpin-src",
5739   .vector_size = sizeof (u32),
5740   .type = VLIB_NODE_TYPE_INTERNAL,
5741   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5742   .error_strings = snat_in2out_error_strings,
5743   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5744   .next_nodes = {
5745      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5746      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5747      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5748      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5749   },
5750 };
5751
5752 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5753                               nat44_ed_hairpin_src_fn);
5754
5755 static uword
5756 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5757                                 vlib_node_runtime_t * node,
5758                                 vlib_frame_t * frame)
5759 {
5760   u32 n_left_from, * from, * to_next;
5761   snat_in2out_next_t next_index;
5762   u32 pkts_processed = 0;
5763   snat_main_t * sm = &snat_main;
5764   u32 stats_node_index;
5765
5766   stats_node_index = snat_in2out_fast_node.index;
5767
5768   from = vlib_frame_vector_args (frame);
5769   n_left_from = frame->n_vectors;
5770   next_index = node->cached_next_index;
5771
5772   while (n_left_from > 0)
5773     {
5774       u32 n_left_to_next;
5775
5776       vlib_get_next_frame (vm, node, next_index,
5777                            to_next, n_left_to_next);
5778
5779       while (n_left_from > 0 && n_left_to_next > 0)
5780         {
5781           u32 bi0;
5782           vlib_buffer_t * b0;
5783           u32 next0;
5784           u32 sw_if_index0;
5785           ip4_header_t * ip0;
5786           ip_csum_t sum0;
5787           u32 new_addr0, old_addr0;
5788           u16 old_port0, new_port0;
5789           udp_header_t * udp0;
5790           tcp_header_t * tcp0;
5791           icmp46_header_t * icmp0;
5792           snat_session_key_t key0, sm0;
5793           u32 proto0;
5794           u32 rx_fib_index0;
5795
5796           /* speculatively enqueue b0 to the current next frame */
5797           bi0 = from[0];
5798           to_next[0] = bi0;
5799           from += 1;
5800           to_next += 1;
5801           n_left_from -= 1;
5802           n_left_to_next -= 1;
5803
5804           b0 = vlib_get_buffer (vm, bi0);
5805           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5806
5807           ip0 = vlib_buffer_get_current (b0);
5808           udp0 = ip4_next_header (ip0);
5809           tcp0 = (tcp_header_t *) udp0;
5810           icmp0 = (icmp46_header_t *) udp0;
5811
5812           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5813           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5814
5815           if (PREDICT_FALSE(ip0->ttl == 1))
5816             {
5817               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5818               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5819                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
5820                                            0);
5821               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5822               goto trace0;
5823             }
5824
5825           proto0 = ip_proto_to_snat_proto (ip0->protocol);
5826
5827           if (PREDICT_FALSE (proto0 == ~0))
5828               goto trace0;
5829
5830           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5831             {
5832               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5833                                   rx_fib_index0, node, next0, ~0, 0, 0);
5834               goto trace0;
5835             }
5836
5837           key0.addr = ip0->src_address;
5838           key0.protocol = proto0;
5839           key0.port = udp0->src_port;
5840           key0.fib_index = rx_fib_index0;
5841
5842           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0, 0))
5843             {
5844               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5845               next0= SNAT_IN2OUT_NEXT_DROP;
5846               goto trace0;
5847             }
5848
5849           new_addr0 = sm0.addr.as_u32;
5850           new_port0 = sm0.port;
5851           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5852           old_addr0 = ip0->src_address.as_u32;
5853           ip0->src_address.as_u32 = new_addr0;
5854
5855           sum0 = ip0->checksum;
5856           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5857                                  ip4_header_t,
5858                                  src_address /* changed member */);
5859           ip0->checksum = ip_csum_fold (sum0);
5860
5861           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5862             {
5863               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5864                 {
5865                   old_port0 = tcp0->src_port;
5866                   tcp0->src_port = new_port0;
5867
5868                   sum0 = tcp0->checksum;
5869                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5870                                          ip4_header_t,
5871                                          dst_address /* changed member */);
5872                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
5873                                          ip4_header_t /* cheat */,
5874                                          length /* changed member */);
5875                   mss_clamping (sm, tcp0, &sum0);
5876                   tcp0->checksum = ip_csum_fold(sum0);
5877                 }
5878               else
5879                 {
5880                   old_port0 = udp0->src_port;
5881                   udp0->src_port = new_port0;
5882                   udp0->checksum = 0;
5883                 }
5884             }
5885           else
5886             {
5887               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5888                 {
5889                   sum0 = tcp0->checksum;
5890                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5891                                          ip4_header_t,
5892                                          dst_address /* changed member */);
5893                   mss_clamping (sm, tcp0, &sum0);
5894                   tcp0->checksum = ip_csum_fold(sum0);
5895                 }
5896             }
5897
5898           /* Hairpinning */
5899           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5900
5901         trace0:
5902           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5903                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5904             {
5905               snat_in2out_trace_t *t =
5906                  vlib_add_trace (vm, node, b0, sizeof (*t));
5907               t->sw_if_index = sw_if_index0;
5908               t->next_index = next0;
5909             }
5910
5911           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5912
5913           /* verify speculative enqueue, maybe switch current next frame */
5914           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5915                                            to_next, n_left_to_next,
5916                                            bi0, next0);
5917         }
5918
5919       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5920     }
5921
5922   vlib_node_increment_counter (vm, stats_node_index,
5923                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5924                                pkts_processed);
5925   return frame->n_vectors;
5926 }
5927
5928
5929 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5930   .function = snat_in2out_fast_static_map_fn,
5931   .name = "nat44-in2out-fast",
5932   .vector_size = sizeof (u32),
5933   .format_trace = format_snat_in2out_fast_trace,
5934   .type = VLIB_NODE_TYPE_INTERNAL,
5935
5936   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5937   .error_strings = snat_in2out_error_strings,
5938
5939   .runtime_data_bytes = sizeof (snat_runtime_t),
5940
5941   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5942
5943   /* edit / add dispositions here */
5944   .next_nodes = {
5945     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5946     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5947     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5948     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5949     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5950   },
5951 };
5952
5953 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);