src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/lib/ipfix_logging.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat44/inlines.h>
  31 #include <nat/lib/nat_syslog.h>
  32 #include <nat/nat44/ed_inlines.h>
  33 #include <nat/lib/nat_inlines.h>
  34
  35 /* number of attempts to get a port for ED overloading algorithm, if rolling
  36  * a dice this many times doesn't produce a free port, it's treated
  37  * as if there were no free ports available to conserve resources */
  38 #define ED_PORT_ALLOC_ATTEMPTS (10)
  39
  40 static char *nat_in2out_ed_error_strings[] = {
  41 #define _(sym,string) string,
  42   foreach_nat_in2out_ed_error
  43 #undef _
  44 };
  45
  46 typedef struct
  47 {
  48   u32 sw_if_index;
  49   u32 next_index;
  50   u32 session_index;
  51   u32 is_slow_path;
  52 } nat_in2out_ed_trace_t;
  53
  54 static u8 *
  55 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  56 {
  57   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  58   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  59   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  60   char *tag;
  61
  62   tag =
  63     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  64     "NAT44_IN2OUT_ED_FAST_PATH";
  65
  66   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  67               t->sw_if_index, t->next_index, t->session_index);
  68
  69   return s;
  70 }
  71
  72 #ifndef CLIB_MARCH_VARIANT
  73 int
  74 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  75 {
  76   snat_main_t *sm = &snat_main;
  77   nat44_is_idle_session_ctx_t *ctx = arg;
  78   snat_session_t *s;
  79   u64 sess_timeout_time;
  80   u8 proto;
  81   u16 r_port, l_port;
  82   ip4_address_t *l_addr, *r_addr;
  83   u32 fib_index;
  84   clib_bihash_kv_16_8_t ed_kv;
  85   int i;
  86   snat_address_t *a;
  87   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  88                                                        ctx->thread_index);
  89
  90   ASSERT (ctx->thread_index == ed_value_get_thread_index (kv));
  91   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (kv));
  92   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  93   if (ctx->now >= sess_timeout_time)
  94     {
  95       if (is_fwd_bypass_session (s))
  96         goto delete;
  97
  98       l_addr = &s->out2in.addr;
  99       r_addr = &s->ext_host_addr;
 100       fib_index = s->out2in.fib_index;
 101       if (snat_is_unk_proto_session (s))
 102         {
 103           proto = s->in2out.port;
 104           r_port = 0;
 105           l_port = 0;
 106         }
 107       else
 108         {
 109           proto = nat_proto_to_ip_proto (s->nat_proto);
 110           l_port = s->out2in.port;
 111           r_port = s->ext_host_port;
 112         }
 113       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 114       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 115         nat_elog_warn ("out2in_ed key del failed");
 116
 117       if (snat_is_unk_proto_session (s))
 118         goto delete;
 119
 120       nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 121                                           s->in2out.addr.as_u32,
 122                                           s->out2in.addr.as_u32,
 123                                           s->nat_proto,
 124                                           s->in2out.port,
 125                                           s->out2in.port,
 126                                           s->in2out.fib_index);
 127
 128       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 129                              &s->in2out.addr, s->in2out.port,
 130                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 131                              &s->out2in.addr, s->out2in.port,
 132                              &s->ext_host_addr, s->ext_host_port,
 133                              s->nat_proto, is_twice_nat_session (s));
 134
 135       if (is_twice_nat_session (s))
 136         {
 137           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 138             {
 139               // TODO FIXME this is obviously broken - which address should be
 140               // freed here?!
 141               a = sm->twice_nat_addresses + i;
 142               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 143                 {
 144                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 145                                                       ctx->thread_index,
 146                                                       &s->ext_host_nat_addr,
 147                                                       s->ext_host_nat_port,
 148                                                       s->nat_proto);
 149                   break;
 150                 }
 151             }
 152         }
 153
 154       if (snat_is_session_static (s))
 155         goto delete;
 156
 157       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 158                                           &s->out2in.addr, s->out2in.port,
 159                                           s->nat_proto);
 160     delete:
 161       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 162       return 1;
 163     }
 164
 165   return 0;
 166 }
 167 #endif
 168
 169 static inline u32
 170 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 171                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 172                           u32 sw_if_index0, u32 rx_fib_index0,
 173                           vlib_node_runtime_t * node, u32 next0, f64 now,
 174                           u32 thread_index, snat_session_t ** p_s0)
 175 {
 176   vlib_main_t *vm = vlib_get_main ();
 177
 178   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 179                        next0, thread_index, p_s0, 0);
 180   snat_session_t *s0 = *p_s0;
 181   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 182     {
 183       /* Accounting */
 184       nat44_session_update_counters (s0, now,
 185                                      vlib_buffer_length_in_chain
 186                                      (vm, b0), thread_index);
 187       /* Per-user LRU list maintenance */
 188       nat44_session_update_lru (sm, s0, thread_index);
 189     }
 190   return next0;
 191 }
 192
 193 static int
 194 nat_ed_alloc_addr_and_port (snat_main_t * sm, u32 rx_fib_index,
 195                             u32 nat_proto, u32 thread_index,
 196                             ip4_address_t r_addr, u16 r_port, u8 proto,
 197                             u16 port_per_thread, u32 snat_thread_index,
 198                             snat_session_t * s,
 199                             ip4_address_t * outside_addr,
 200                             u16 * outside_port,
 201                             clib_bihash_kv_16_8_t * out2in_ed_kv)
 202 {
 203   int i;
 204   snat_address_t *a, *ga = 0;
 205   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 206
 207   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 208
 209   for (i = 0; i < vec_len (sm->addresses); i++)
 210     {
 211       a = sm->addresses + i;
 212       switch (nat_proto)
 213         {
 214 #define _(N, j, n, unused)                                                   \
 215   case NAT_PROTOCOL_##N:                                                     \
 216     if (a->fib_index == rx_fib_index)                                        \
 217       {                                                                      \
 218         /* first try port suggested by caller */                             \
 219         u16 port = clib_net_to_host_u16 (*outside_port);                     \
 220         u16 port_offset = port - port_thread_offset;                         \
 221         if (port <= port_thread_offset ||                                    \
 222             port > port_thread_offset + port_per_thread)                     \
 223           {                                                                  \
 224             /* need to pick a different port, suggested port doesn't fit in  \
 225              * this thread's port range */                                   \
 226             port_offset = snat_random_port (0, port_per_thread - 1);         \
 227             port = port_thread_offset + port_offset;                         \
 228           }                                                                  \
 229         u16 attempts = ED_PORT_ALLOC_ATTEMPTS;                               \
 230         do                                                                   \
 231           {                                                                  \
 232             init_ed_kv (out2in_ed_kv, a->addr, clib_host_to_net_u16 (port),  \
 233                         r_addr, r_port, s->out2in.fib_index, proto,          \
 234                         thread_index, s - tsm->sessions);                    \
 235             int rv = clib_bihash_add_del_16_8 (&sm->out2in_ed, out2in_ed_kv, \
 236                                                2 /* is_add */);              \
 237             if (0 == rv)                                                     \
 238               {                                                              \
 239                 ++a->busy_##n##_port_refcounts[port];                        \
 240                 a->busy_##n##_ports_per_thread[thread_index]++;              \
 241                 a->busy_##n##_ports++;                                       \
 242                 *outside_addr = a->addr;                                     \
 243                 *outside_port = clib_host_to_net_u16 (port);                 \
 244                 return 0;                                                    \
 245               }                                                              \
 246             port_offset = snat_random_port (0, port_per_thread - 1);         \
 247             port = port_thread_offset + port_offset;                         \
 248             --attempts;                                                      \
 249           }                                                                  \
 250         while (attempts > 0);                                                \
 251       }                                                                      \
 252     else if (a->fib_index == ~0)                                             \
 253       {                                                                      \
 254         ga = a;                                                              \
 255       }                                                                      \
 256     break;
 257
 258           foreach_nat_protocol;
 259         default:
 260           nat_elog_info ("unknown protocol");
 261           return 1;
 262         }
 263     }
 264
 265   if (ga)
 266     {
 267       /* fake fib_index to reuse macro */
 268       rx_fib_index = ~0;
 269       a = ga;
 270       switch (nat_proto)
 271         {
 272           foreach_nat_protocol;
 273         default:
 274           nat_elog_info ("unknown protocol");
 275           return 1;
 276         }
 277     }
 278
 279 #undef _
 280
 281   /* Totally out of translations to use... */
 282   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 283   return 1;
 284 }
 285
 286 static_always_inline u32
 287 nat_outside_fib_index_lookup (snat_main_t * sm, ip4_address_t addr)
 288 {
 289   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 290   nat_outside_fib_t *outside_fib;
 291   fib_prefix_t pfx = {
 292     .fp_proto = FIB_PROTOCOL_IP4,
 293     .fp_len = 32,
 294     .fp_addr = {.ip4.as_u32 = addr.as_u32,}
 295     ,
 296   };
 297   // TODO: multiple vrfs none can resolve addr
 298   /* *INDENT-OFF* */
 299   vec_foreach (outside_fib, sm->outside_fibs)
 300     {
 301       fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 302       if (FIB_NODE_INDEX_INVALID != fei)
 303         {
 304           if (fib_entry_get_resolving_interface (fei) != ~0)
 305             {
 306               return outside_fib->fib_index;
 307             }
 308         }
 309     }
 310   /* *INDENT-ON* */
 311   return ~0;
 312 }
 313
 314 static u32
 315 slow_path_ed (snat_main_t * sm,
 316               vlib_buffer_t * b,
 317               ip4_address_t l_addr,
 318               ip4_address_t r_addr,
 319               u16 l_port,
 320               u16 r_port,
 321               u8 proto,
 322               u32 rx_fib_index,
 323               snat_session_t ** sessionp,
 324               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 325 {
 326   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 327   clib_bihash_kv_16_8_t out2in_ed_kv;
 328   nat44_is_idle_session_ctx_t ctx;
 329   ip4_address_t outside_addr;
 330   u16 outside_port;
 331   u8 identity_nat;
 332
 333   u32 nat_proto = ip_proto_to_nat_proto (proto);
 334   snat_session_t *s = NULL;
 335   lb_nat_type_t lb = 0;
 336
 337   if (PREDICT_TRUE (nat_proto == NAT_PROTOCOL_TCP))
 338     {
 339       if (PREDICT_FALSE
 340           (!tcp_flags_is_init
 341            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 342         {
 343           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 344           return NAT_NEXT_DROP;
 345         }
 346     }
 347
 348   if (PREDICT_FALSE
 349       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 350     {
 351       if (!nat_lru_free_one (sm, thread_index, now))
 352         {
 353           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 354           nat_ipfix_logging_max_sessions (thread_index,
 355                                           sm->max_translations_per_thread);
 356           nat_elog_notice ("maximum sessions exceeded");
 357           return NAT_NEXT_DROP;
 358         }
 359     }
 360
 361   ip4_address_t sm_addr;
 362   u16 sm_port;
 363   u32 sm_fib_index;
 364   /* First try to match static mapping by local address and port */
 365   if (snat_static_mapping_match
 366       (sm, l_addr, l_port, rx_fib_index, nat_proto, &sm_addr, &sm_port,
 367        &sm_fib_index, 0, 0, 0, &lb, 0, &identity_nat, 0))
 368     {
 369       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 370       ASSERT (s);
 371       s->in2out.addr = l_addr;
 372       s->in2out.port = l_port;
 373       s->nat_proto = nat_proto;
 374       s->in2out.fib_index = rx_fib_index;
 375       s->out2in.fib_index = sm->outside_fib_index;
 376
 377       switch (vec_len (sm->outside_fibs))
 378         {
 379         case 0:
 380           s->out2in.fib_index = sm->outside_fib_index;
 381           break;
 382         case 1:
 383           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 384           break;
 385         default:
 386           s->out2in.fib_index = nat_outside_fib_index_lookup (sm, r_addr);
 387           break;
 388         }
 389
 390       /* Try to create dynamic translation */
 391       outside_port = l_port;    // suggest using local port to allocation function
 392       if (nat_ed_alloc_addr_and_port (sm, rx_fib_index, nat_proto,
 393                                       thread_index, r_addr, r_port, proto,
 394                                       sm->port_per_thread,
 395                                       tsm->snat_thread_index, s,
 396                                       &outside_addr,
 397                                       &outside_port, &out2in_ed_kv))
 398         {
 399           nat_elog_notice ("addresses exhausted");
 400           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 401           nat_ed_session_delete (sm, s, thread_index, 1);
 402           return NAT_NEXT_DROP;
 403         }
 404       s->out2in.addr = outside_addr;
 405       s->out2in.port = outside_port;
 406     }
 407   else
 408     {
 409       if (PREDICT_FALSE (identity_nat))
 410         {
 411           *sessionp = NULL;
 412           return next;
 413         }
 414       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 415       ASSERT (s);
 416       s->out2in.addr = sm_addr;
 417       s->out2in.port = sm_port;
 418       s->in2out.addr = l_addr;
 419       s->in2out.port = l_port;
 420       s->nat_proto = nat_proto;
 421       s->in2out.fib_index = rx_fib_index;
 422       s->out2in.fib_index = sm->outside_fib_index;
 423       switch (vec_len (sm->outside_fibs))
 424         {
 425         case 0:
 426           s->out2in.fib_index = sm->outside_fib_index;
 427           break;
 428         case 1:
 429           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 430           break;
 431         default:
 432           s->out2in.fib_index = nat_outside_fib_index_lookup (sm, r_addr);
 433           break;
 434         }
 435
 436       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 437
 438       init_ed_kv (&out2in_ed_kv, sm_addr, sm_port, r_addr, r_port,
 439                   s->out2in.fib_index, proto, thread_index,
 440                   s - tsm->sessions);
 441       if (clib_bihash_add_or_overwrite_stale_16_8
 442           (&sm->out2in_ed, &out2in_ed_kv, nat44_o2i_ed_is_idle_session_cb,
 443            &ctx))
 444         nat_elog_notice ("out2in-ed key add failed");
 445     }
 446
 447   if (lb)
 448     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 449   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 450   s->ext_host_addr = r_addr;
 451   s->ext_host_port = r_port;
 452
 453   clib_bihash_kv_16_8_t in2out_ed_kv;
 454   init_ed_kv (&in2out_ed_kv, l_addr, l_port, r_addr, r_port, rx_fib_index,
 455               proto, thread_index, s - tsm->sessions);
 456   ctx.now = now;
 457   ctx.thread_index = thread_index;
 458   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &in2out_ed_kv,
 459                                                nat44_i2o_ed_is_idle_session_cb,
 460                                                &ctx))
 461     nat_elog_notice ("in2out-ed key add failed");
 462
 463   *sessionp = s;
 464
 465   /* log NAT event */
 466   nat_ipfix_logging_nat44_ses_create (thread_index,
 467                                       s->in2out.addr.as_u32,
 468                                       s->out2in.addr.as_u32,
 469                                       s->nat_proto,
 470                                       s->in2out.port,
 471                                       s->out2in.port, s->in2out.fib_index);
 472
 473   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 474                          &s->in2out.addr, s->in2out.port,
 475                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 476                          &s->out2in.addr, s->out2in.port,
 477                          &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 478                          0);
 479
 480   per_vrf_sessions_register_session (s, thread_index);
 481
 482   return next;
 483 }
 484
 485 static_always_inline int
 486 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 487                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 488                         u32 rx_fib_index, u32 thread_index)
 489 {
 490   udp_header_t *udp = ip4_next_header (ip);
 491   clib_bihash_kv_16_8_t kv, value;
 492
 493   init_ed_k (&kv, ip->dst_address, udp->dst_port, ip->src_address,
 494              udp->src_port, sm->outside_fib_index, ip->protocol);
 495
 496   /* NAT packet aimed at external address if has active sessions */
 497   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 498     {
 499       /* or is static mappings */
 500       ip4_address_t placeholder_addr;
 501       u16 placeholder_port;
 502       u32 placeholder_fib_index;
 503       if (!snat_static_mapping_match
 504           (sm, ip->dst_address, udp->dst_port, sm->outside_fib_index, proto,
 505            &placeholder_addr, &placeholder_port, &placeholder_fib_index, 1, 0,
 506            0, 0, 0, 0, 0))
 507         return 0;
 508     }
 509   else
 510     return 0;
 511
 512   if (sm->forwarding_enabled)
 513     return 1;
 514
 515   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 516                                   rx_fib_index);
 517 }
 518
 519 static_always_inline int
 520 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 521                                       u32 thread_index, f64 now,
 522                                       vlib_main_t * vm, vlib_buffer_t * b)
 523 {
 524   clib_bihash_kv_16_8_t kv, value;
 525   snat_session_t *s = 0;
 526   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 527
 528   if (!sm->forwarding_enabled)
 529     return 0;
 530
 531   if (ip->protocol == IP_PROTOCOL_ICMP)
 532     {
 533       if (get_icmp_i2o_ed_key (b, ip, 0, ~0, ~0, 0, 0, 0, &kv))
 534         return 0;
 535     }
 536   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 537     {
 538       init_ed_k (&kv, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
 539                  ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port, 0,
 540                  ip->protocol);
 541     }
 542   else
 543     {
 544       init_ed_k (&kv, ip->src_address, 0, ip->dst_address, 0, 0,
 545                  ip->protocol);
 546     }
 547
 548   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 549     {
 550       ASSERT (thread_index == ed_value_get_thread_index (&value));
 551       s =
 552         pool_elt_at_index (tsm->sessions,
 553                            ed_value_get_session_index (&value));
 554       if (is_fwd_bypass_session (s))
 555         {
 556           if (ip->protocol == IP_PROTOCOL_TCP)
 557             {
 558               nat44_set_tcp_session_state_i2o (sm, now, s, b, thread_index);
 559             }
 560           /* Accounting */
 561           nat44_session_update_counters (s, now,
 562                                          vlib_buffer_length_in_chain (vm, b),
 563                                          thread_index);
 564           /* Per-user LRU list maintenance */
 565           nat44_session_update_lru (sm, s, thread_index);
 566           return 1;
 567         }
 568       else
 569         return 0;
 570     }
 571
 572   return 0;
 573 }
 574
 575 static_always_inline int
 576 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 577                                        u16 src_port, u16 dst_port,
 578                                        u32 thread_index, u32 rx_sw_if_index,
 579                                        u32 tx_sw_if_index, f64 now)
 580 {
 581   clib_bihash_kv_16_8_t kv, value;
 582   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 583   snat_interface_t *i;
 584   snat_session_t *s;
 585   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 586   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 587
 588   /* src NAT check */
 589   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 590              tx_fib_index, ip->protocol);
 591   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 592     {
 593       ASSERT (thread_index == ed_value_get_thread_index (&value));
 594       s =
 595         pool_elt_at_index (tsm->sessions,
 596                            ed_value_get_session_index (&value));
 597       if (nat44_is_ses_closed (s)
 598           && (!s->tcp_closed_timestamp || now >= s->tcp_closed_timestamp))
 599         {
 600           nat_free_session_data (sm, s, thread_index, 0);
 601           nat_ed_session_delete (sm, s, thread_index, 1);
 602         }
 603       return 1;
 604     }
 605
 606   /* dst NAT check */
 607   init_ed_k (&kv, ip->dst_address, dst_port, ip->src_address, src_port,
 608              rx_fib_index, ip->protocol);
 609   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 610     {
 611       ASSERT (thread_index == ed_value_get_thread_index (&value));
 612       s =
 613         pool_elt_at_index (tsm->sessions,
 614                            ed_value_get_session_index (&value));
 615       if (is_fwd_bypass_session (s))
 616         return 0;
 617
 618       /* hairpinning */
 619       /* *INDENT-OFF* */
 620       pool_foreach (i, sm->output_feature_interfaces)
 621        {
 622         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 623            return 0;
 624       }
 625       /* *INDENT-ON* */
 626       return 1;
 627     }
 628
 629   return 0;
 630 }
 631
 632 #ifndef CLIB_MARCH_VARIANT
 633 u32
 634 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 635                       u32 thread_index, vlib_buffer_t * b,
 636                       ip4_header_t * ip, ip4_address_t * addr,
 637                       u16 * port, u32 * fib_index, nat_protocol_t * proto,
 638                       void *d, void *e, u8 * dont_translate)
 639 {
 640   u32 sw_if_index;
 641   u32 rx_fib_index;
 642   clib_bihash_kv_16_8_t kv, value;
 643   u32 next = ~0;
 644   int err;
 645   snat_session_t *s = NULL;
 646   u16 l_port = 0, r_port = 0;   // initialize to workaround gcc warning
 647   vlib_main_t *vm = vlib_get_main ();
 648   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 649   *dont_translate = 0;
 650   f64 now = vlib_time_now (vm);
 651
 652   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 653   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 654
 655   err =
 656     get_icmp_i2o_ed_key (b, ip, rx_fib_index, ~0, ~0, proto, &l_port,
 657                          &r_port, &kv);
 658   if (err != 0)
 659     {
 660       b->error = node->errors[err];
 661       next = NAT_NEXT_DROP;
 662       goto out;
 663     }
 664
 665   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 666     {
 667       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 668         {
 669           if (PREDICT_FALSE
 670               (nat44_ed_not_translate_output_feature
 671                (sm, ip, l_port, r_port, thread_index,
 672                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX], now)))
 673             {
 674               *dont_translate = 1;
 675               goto out;
 676             }
 677         }
 678       else
 679         {
 680           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 681                                                      ip, NAT_PROTOCOL_ICMP,
 682                                                      rx_fib_index,
 683                                                      thread_index)))
 684             {
 685               *dont_translate = 1;
 686               goto out;
 687             }
 688         }
 689
 690       if (PREDICT_FALSE
 691           (icmp_type_is_error_message
 692            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 693         {
 694           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 695           next = NAT_NEXT_DROP;
 696           goto out;
 697         }
 698
 699       next =
 700         slow_path_ed (sm, b, ip->src_address, ip->dst_address, l_port, r_port,
 701                       ip->protocol, rx_fib_index, &s, node, next,
 702                       thread_index, vlib_time_now (vm));
 703
 704       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 705         goto out;
 706
 707       if (!s)
 708         {
 709           *dont_translate = 1;
 710           goto out;
 711         }
 712     }
 713   else
 714     {
 715       if (PREDICT_FALSE
 716           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 717            ICMP4_echo_request
 718            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 719            ICMP4_echo_reply
 720            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 721                                            reass.icmp_type_or_tcp_flags)))
 722         {
 723           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 724           next = NAT_NEXT_DROP;
 725           goto out;
 726         }
 727
 728       ASSERT (thread_index == ed_value_get_thread_index (&value));
 729       s =
 730         pool_elt_at_index (tsm->sessions,
 731                            ed_value_get_session_index (&value));
 732     }
 733 out:
 734   if (s)
 735     {
 736       *addr = s->out2in.addr;
 737       *port = s->out2in.port;
 738       *fib_index = s->out2in.fib_index;
 739     }
 740   if (d)
 741     {
 742       *(snat_session_t **) d = s;
 743     }
 744   return next;
 745 }
 746 #endif
 747
 748 static snat_session_t *
 749 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 750                                vlib_buffer_t * b,
 751                                ip4_header_t * ip,
 752                                u32 rx_fib_index,
 753                                u32 thread_index,
 754                                f64 now,
 755                                vlib_main_t * vm, vlib_node_runtime_t * node)
 756 {
 757   clib_bihash_kv_8_8_t kv, value;
 758   clib_bihash_kv_16_8_t s_kv, s_value;
 759   snat_static_mapping_t *m;
 760   u32 old_addr, new_addr = 0;
 761   ip_csum_t sum;
 762   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 763   snat_session_t *s;
 764   u32 outside_fib_index = sm->outside_fib_index;
 765   int i;
 766   u8 is_sm = 0;
 767
 768   switch (vec_len (sm->outside_fibs))
 769     {
 770     case 0:
 771       outside_fib_index = sm->outside_fib_index;
 772       break;
 773     case 1:
 774       outside_fib_index = sm->outside_fibs[0].fib_index;
 775       break;
 776     default:
 777       outside_fib_index = nat_outside_fib_index_lookup (sm, ip->dst_address);
 778       break;
 779     }
 780   old_addr = ip->src_address.as_u32;
 781
 782   init_ed_k (&s_kv, ip->src_address, 0, ip->dst_address, 0, rx_fib_index,
 783              ip->protocol);
 784
 785   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 786     {
 787       ASSERT (thread_index == ed_value_get_thread_index (&s_value));
 788       s =
 789         pool_elt_at_index (tsm->sessions,
 790                            ed_value_get_session_index (&s_value));
 791       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 792     }
 793   else
 794     {
 795       if (PREDICT_FALSE
 796           (nat44_ed_maximum_sessions_exceeded
 797            (sm, rx_fib_index, thread_index)))
 798         {
 799           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 800           nat_ipfix_logging_max_sessions (thread_index,
 801                                           sm->max_translations_per_thread);
 802           nat_elog_notice ("maximum sessions exceeded");
 803           return 0;
 804         }
 805
 806       init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 807
 808       /* Try to find static mapping first */
 809       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 810         {
 811           m = pool_elt_at_index (sm->static_mappings, value.value);
 812           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 813           is_sm = 1;
 814           goto create_ses;
 815         }
 816       else
 817         {
 818           /* *INDENT-OFF* */
 819           pool_foreach (s, tsm->sessions) {
 820             if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 821               {
 822                 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 823
 824                 init_ed_k(&s_kv, s->out2in.addr, 0, ip->dst_address, 0, outside_fib_index, ip->protocol);
 825                 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 826                   goto create_ses;
 827
 828                 break;
 829               }
 830           }
 831           /* *INDENT-ON* */
 832
 833           for (i = 0; i < vec_len (sm->addresses); i++)
 834             {
 835               init_ed_k (&s_kv, sm->addresses[i].addr, 0, ip->dst_address, 0,
 836                          outside_fib_index, ip->protocol);
 837               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 838                 {
 839                   new_addr = ip->src_address.as_u32 =
 840                     sm->addresses[i].addr.as_u32;
 841                   goto create_ses;
 842                 }
 843             }
 844           return 0;
 845         }
 846
 847     create_ses:
 848       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 849       if (!s)
 850         {
 851           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 852           nat_elog_warn ("create NAT session failed");
 853           return 0;
 854         }
 855
 856       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 857       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 858       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 859       s->out2in.addr.as_u32 = new_addr;
 860       s->out2in.fib_index = outside_fib_index;
 861       s->in2out.addr.as_u32 = old_addr;
 862       s->in2out.fib_index = rx_fib_index;
 863       s->in2out.port = s->out2in.port = ip->protocol;
 864       if (is_sm)
 865         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 866
 867       /* Add to lookup tables */
 868       init_ed_kv (&s_kv, s->in2out.addr, 0, ip->dst_address, 0, rx_fib_index,
 869                   ip->protocol, thread_index, s - tsm->sessions);
 870       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 871         nat_elog_notice ("in2out key add failed");
 872
 873       init_ed_kv (&s_kv, s->out2in.addr, 0, ip->dst_address, 0,
 874                   outside_fib_index, ip->protocol, thread_index,
 875                   s - tsm->sessions);
 876       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 877         nat_elog_notice ("out2in key add failed");
 878
 879       per_vrf_sessions_register_session (s, thread_index);
 880     }
 881
 882   /* Update IP checksum */
 883   sum = ip->checksum;
 884   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 885   ip->checksum = ip_csum_fold (sum);
 886
 887   /* Accounting */
 888   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 889                                  thread_index);
 890   /* Per-user LRU list maintenance */
 891   nat44_session_update_lru (sm, s, thread_index);
 892
 893   /* Hairpinning */
 894   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 895     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 896
 897   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 898     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 899
 900   return s;
 901 }
 902
 903 static inline uword
 904 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 905                                           vlib_node_runtime_t * node,
 906                                           vlib_frame_t * frame,
 907                                           int is_output_feature)
 908 {
 909   u32 n_left_from, *from;
 910   snat_main_t *sm = &snat_main;
 911   f64 now = vlib_time_now (vm);
 912   u32 thread_index = vm->thread_index;
 913   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 914   u32 def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH
 915     : NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 916
 917   from = vlib_frame_vector_args (frame);
 918   n_left_from = frame->n_vectors;
 919
 920   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 921   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 922   vlib_get_buffers (vm, from, b, n_left_from);
 923
 924   while (n_left_from > 0)
 925     {
 926       vlib_buffer_t *b0;
 927       u32 sw_if_index0, rx_fib_index0, proto0, new_addr0, old_addr0,
 928         iph_offset0 = 0;
 929       u16 old_port0, new_port0;
 930       ip4_header_t *ip0;
 931       udp_header_t *udp0;
 932       tcp_header_t *tcp0;
 933       snat_session_t *s0 = 0;
 934       clib_bihash_kv_16_8_t kv0, value0;
 935       ip_csum_t sum0;
 936
 937       b0 = *b;
 938       b++;
 939
 940       /* Prefetch next iteration. */
 941       if (PREDICT_TRUE (n_left_from >= 2))
 942         {
 943           vlib_buffer_t *p2;
 944
 945           p2 = *b;
 946
 947           vlib_prefetch_buffer_header (p2, LOAD);
 948
 949           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 950         }
 951
 952       if (is_output_feature)
 953         {
 954           iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 955         }
 956
 957       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 958
 959       ip0 =
 960         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
 961
 962       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 963       rx_fib_index0 =
 964         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 965
 966       if (PREDICT_FALSE (ip0->ttl == 1))
 967         {
 968           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 969           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 970                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 971                                        0);
 972           next[0] = NAT_NEXT_ICMP_ERROR;
 973           goto trace0;
 974         }
 975
 976       udp0 = ip4_next_header (ip0);
 977       tcp0 = (tcp_header_t *) udp0;
 978       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 979
 980       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 981         {
 982           next[0] = def_slow;
 983           goto trace0;
 984         }
 985
 986       if (is_output_feature)
 987         {
 988           if (PREDICT_FALSE
 989               (nat_not_translate_output_feature_fwd
 990                (sm, ip0, thread_index, now, vm, b0)))
 991             goto trace0;
 992         }
 993
 994       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 995         {
 996           next[0] = def_slow;
 997           goto trace0;
 998         }
 999
1000       init_ed_k (&kv0, ip0->src_address,
1001                  vnet_buffer (b0)->ip.reass.l4_src_port, ip0->dst_address,
1002                  vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1003                  ip0->protocol);
1004
1005       // lookup for session
1006       if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1007         {
1008           // session does not exist go slow path
1009           next[0] = def_slow;
1010           goto trace0;
1011         }
1012       ASSERT (thread_index == ed_value_get_thread_index (&value0));
1013       s0 =
1014         pool_elt_at_index (tsm->sessions,
1015                            ed_value_get_session_index (&value0));
1016
1017       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
1018         {
1019           // session is closed, go slow path
1020           nat_free_session_data (sm, s0, thread_index, 0);
1021           nat_ed_session_delete (sm, s0, thread_index, 1);
1022           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1023           goto trace0;
1024         }
1025
1026       if (s0->tcp_closed_timestamp)
1027         {
1028           if (now >= s0->tcp_closed_timestamp)
1029             {
1030               // session is closed, go slow path, freed in slow path
1031               next[0] = def_slow;
1032             }
1033           else
1034             {
1035               // session in transitory timeout, drop
1036               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1037               next[0] = NAT_NEXT_DROP;
1038             }
1039           goto trace0;
1040         }
1041
1042       // drop if session expired
1043       u64 sess_timeout_time;
1044       sess_timeout_time =
1045         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
1046       if (now >= sess_timeout_time)
1047         {
1048           nat_free_session_data (sm, s0, thread_index, 0);
1049           nat_ed_session_delete (sm, s0, thread_index, 1);
1050           // session is closed, go slow path
1051           next[0] = def_slow;
1052           goto trace0;
1053         }
1054
1055       b0->flags |= VNET_BUFFER_F_IS_NATED;
1056
1057       if (!is_output_feature)
1058         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1059
1060       old_addr0 = ip0->src_address.as_u32;
1061       new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1062       sum0 = ip0->checksum;
1063       sum0 =
1064         ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1065                         src_address);
1066       if (PREDICT_FALSE (is_twice_nat_session (s0)))
1067         sum0 =
1068           ip_csum_update (sum0, ip0->dst_address.as_u32,
1069                           s0->ext_host_addr.as_u32, ip4_header_t,
1070                           dst_address);
1071       ip0->checksum = ip_csum_fold (sum0);
1072
1073       old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1074
1075       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1076         {
1077           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1078             {
1079               new_port0 = udp0->src_port = s0->out2in.port;
1080               sum0 = tcp0->checksum;
1081               sum0 =
1082                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1083                                 dst_address);
1084               sum0 =
1085                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1086                                 length);
1087               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1088                 {
1089                   sum0 =
1090                     ip_csum_update (sum0, ip0->dst_address.as_u32,
1091                                     s0->ext_host_addr.as_u32, ip4_header_t,
1092                                     dst_address);
1093                   sum0 =
1094                     ip_csum_update (sum0,
1095                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1096                                     s0->ext_host_port, ip4_header_t, length);
1097                   tcp0->dst_port = s0->ext_host_port;
1098                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1099                 }
1100               mss_clamping (sm->mss_clamping, tcp0, &sum0);
1101               tcp0->checksum = ip_csum_fold (sum0);
1102             }
1103           vlib_increment_simple_counter (&sm->counters.fastpath.in2out_ed.tcp,
1104                                          thread_index, sw_if_index0, 1);
1105           nat44_set_tcp_session_state_i2o (sm, now, s0, b0, thread_index);
1106         }
1107       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1108                && udp0->checksum)
1109         {
1110           new_port0 = udp0->src_port = s0->out2in.port;
1111           sum0 = udp0->checksum;
1112           sum0 =
1113             ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1114                             dst_address);
1115           sum0 =
1116             ip_csum_update (sum0, old_port0, new_port0, ip4_header_t, length);
1117           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1118             {
1119               sum0 =
1120                 ip_csum_update (sum0, ip0->dst_address.as_u32,
1121                                 s0->ext_host_addr.as_u32, ip4_header_t,
1122                                 dst_address);
1123               sum0 =
1124                 ip_csum_update (sum0, vnet_buffer (b0)->ip.reass.l4_dst_port,
1125                                 s0->ext_host_port, ip4_header_t, length);
1126               udp0->dst_port = s0->ext_host_port;
1127               ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1128             }
1129           udp0->checksum = ip_csum_fold (sum0);
1130           vlib_increment_simple_counter (&sm->counters.fastpath.in2out_ed.udp,
1131                                          thread_index, sw_if_index0, 1);
1132         }
1133       else
1134         {
1135           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1136             {
1137               new_port0 = udp0->src_port = s0->out2in.port;
1138               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1139                 {
1140                   udp0->dst_port = s0->ext_host_port;
1141                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1142                 }
1143               vlib_increment_simple_counter (&sm->counters.fastpath.
1144                                              in2out_ed.udp, thread_index,
1145                                              sw_if_index0, 1);
1146             }
1147         }
1148
1149       /* Accounting */
1150       nat44_session_update_counters (s0, now,
1151                                      vlib_buffer_length_in_chain (vm, b0),
1152                                      thread_index);
1153       /* Per-user LRU list maintenance */
1154       nat44_session_update_lru (sm, s0, thread_index);
1155
1156     trace0:
1157       if (PREDICT_FALSE
1158           ((node->flags & VLIB_NODE_FLAG_TRACE)
1159            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1160         {
1161           nat_in2out_ed_trace_t *t =
1162             vlib_add_trace (vm, node, b0, sizeof (*t));
1163           t->sw_if_index = sw_if_index0;
1164           t->next_index = next[0];
1165           t->is_slow_path = 0;
1166
1167           if (s0)
1168             t->session_index = s0 - tsm->sessions;
1169           else
1170             t->session_index = ~0;
1171         }
1172
1173       if (next[0] == NAT_NEXT_DROP)
1174         {
1175           vlib_increment_simple_counter (&sm->counters.fastpath.
1176                                          in2out_ed.drops, thread_index,
1177                                          sw_if_index0, 1);
1178         }
1179
1180       n_left_from--;
1181       next++;
1182     }
1183
1184   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1185                                frame->n_vectors);
1186   return frame->n_vectors;
1187 }
1188
1189 static inline uword
1190 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1191                                           vlib_node_runtime_t * node,
1192                                           vlib_frame_t * frame,
1193                                           int is_output_feature)
1194 {
1195   u32 n_left_from, *from;
1196   snat_main_t *sm = &snat_main;
1197   f64 now = vlib_time_now (vm);
1198   u32 thread_index = vm->thread_index;
1199   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1200
1201   from = vlib_frame_vector_args (frame);
1202   n_left_from = frame->n_vectors;
1203
1204   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1205   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1206   vlib_get_buffers (vm, from, b, n_left_from);
1207
1208   while (n_left_from > 0)
1209     {
1210       vlib_buffer_t *b0;
1211       u32 sw_if_index0, rx_fib_index0, proto0, new_addr0, old_addr0,
1212         iph_offset0 = 0;
1213       u16 old_port0, new_port0;
1214       ip4_header_t *ip0;
1215       udp_header_t *udp0;
1216       tcp_header_t *tcp0;
1217       icmp46_header_t *icmp0;
1218       snat_session_t *s0 = 0;
1219       clib_bihash_kv_16_8_t kv0, value0;
1220       ip_csum_t sum0;
1221
1222       b0 = *b;
1223
1224       if (is_output_feature)
1225         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1226
1227       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1228
1229       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1230                               iph_offset0);
1231
1232       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1233       rx_fib_index0 =
1234         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1235
1236       if (PREDICT_FALSE (ip0->ttl == 1))
1237         {
1238           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1239           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1240                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1241                                        0);
1242           next[0] = NAT_NEXT_ICMP_ERROR;
1243           goto trace0;
1244         }
1245
1246       udp0 = ip4_next_header (ip0);
1247       tcp0 = (tcp_header_t *) udp0;
1248       icmp0 = (icmp46_header_t *) udp0;
1249       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1250
1251       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1252         {
1253           s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1254                                               rx_fib_index0,
1255                                               thread_index, now, vm, node);
1256           if (!s0)
1257             next[0] = NAT_NEXT_DROP;
1258
1259           vlib_increment_simple_counter (&sm->counters.slowpath.
1260                                          in2out_ed.other, thread_index,
1261                                          sw_if_index0, 1);
1262           goto trace0;
1263         }
1264
1265       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1266         {
1267           next[0] =
1268             icmp_in2out_ed_slow_path (sm, b0, ip0, icmp0, sw_if_index0,
1269                                       rx_fib_index0, node, next[0], now,
1270                                       thread_index, &s0);
1271           vlib_increment_simple_counter (&sm->counters.slowpath.
1272                                          in2out_ed.icmp, thread_index,
1273                                          sw_if_index0, 1);
1274           goto trace0;
1275         }
1276
1277       init_ed_k (&kv0, ip0->src_address,
1278                  vnet_buffer (b0)->ip.reass.l4_src_port, ip0->dst_address,
1279                  vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1280                  ip0->protocol);
1281       if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1282         {
1283           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1284           s0 =
1285             pool_elt_at_index (tsm->sessions,
1286                                ed_value_get_session_index (&value0));
1287
1288           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1289             {
1290               nat_free_session_data (sm, s0, thread_index, 0);
1291               nat_ed_session_delete (sm, s0, thread_index, 1);
1292               s0 = NULL;
1293             }
1294         }
1295
1296       if (!s0)
1297         {
1298           if (is_output_feature)
1299             {
1300               if (PREDICT_FALSE
1301                   (nat44_ed_not_translate_output_feature
1302                    (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1303                     vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1304                     sw_if_index0, vnet_buffer (b0)->sw_if_index[VLIB_TX],
1305                     now)))
1306                 goto trace0;
1307
1308               /*
1309                * Send DHCP packets to the ipv4 stack, or we won't
1310                * be able to use dhcp client on the outside interface
1311                */
1312               if (PREDICT_FALSE
1313                   (proto0 == NAT_PROTOCOL_UDP
1314                    && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1315                        clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1316                    && ip0->dst_address.as_u32 == 0xffffffff))
1317                 goto trace0;
1318             }
1319           else
1320             {
1321               if (PREDICT_FALSE
1322                   (nat44_ed_not_translate
1323                    (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1324                     thread_index)))
1325                 goto trace0;
1326             }
1327
1328           next[0] =
1329             slow_path_ed (sm, b0, ip0->src_address, ip0->dst_address,
1330                           vnet_buffer (b0)->ip.reass.l4_src_port,
1331                           vnet_buffer (b0)->ip.reass.l4_dst_port,
1332                           ip0->protocol, rx_fib_index0, &s0, node, next[0],
1333                           thread_index, now);
1334
1335           if (PREDICT_FALSE (next[0] == NAT_NEXT_DROP))
1336             goto trace0;
1337
1338           if (PREDICT_FALSE (!s0))
1339             goto trace0;
1340
1341         }
1342
1343       b0->flags |= VNET_BUFFER_F_IS_NATED;
1344
1345       if (!is_output_feature)
1346         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1347
1348       old_addr0 = ip0->src_address.as_u32;
1349       new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1350       sum0 = ip0->checksum;
1351       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1352                              src_address);
1353       if (PREDICT_FALSE (is_twice_nat_session (s0)))
1354         sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1355                                s0->ext_host_addr.as_u32, ip4_header_t,
1356                                dst_address);
1357       ip0->checksum = ip_csum_fold (sum0);
1358
1359       old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1360
1361       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1362         {
1363           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1364             {
1365               new_port0 = udp0->src_port = s0->out2in.port;
1366               sum0 = tcp0->checksum;
1367               sum0 =
1368                 ip_csum_update (sum0, old_addr0, new_addr0,
1369                                 ip4_header_t, dst_address);
1370               sum0 =
1371                 ip_csum_update (sum0, old_port0, new_port0,
1372                                 ip4_header_t, length);
1373               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1374                 {
1375                   sum0 =
1376                     ip_csum_update (sum0, ip0->dst_address.as_u32,
1377                                     s0->ext_host_addr.as_u32,
1378                                     ip4_header_t, dst_address);
1379                   sum0 =
1380                     ip_csum_update (sum0,
1381                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1382                                     s0->ext_host_port, ip4_header_t, length);
1383                   tcp0->dst_port = s0->ext_host_port;
1384                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1385                 }
1386               mss_clamping (sm->mss_clamping, tcp0, &sum0);
1387               tcp0->checksum = ip_csum_fold (sum0);
1388             }
1389           vlib_increment_simple_counter (&sm->counters.slowpath.in2out_ed.tcp,
1390                                          thread_index, sw_if_index0, 1);
1391           nat44_set_tcp_session_state_i2o (sm, now, s0, b0, thread_index);
1392         }
1393       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1394                && udp0->checksum)
1395         {
1396           new_port0 = udp0->src_port = s0->out2in.port;
1397           sum0 = udp0->checksum;
1398           sum0 =
1399             ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1400                             dst_address);
1401           sum0 =
1402             ip_csum_update (sum0, old_port0, new_port0, ip4_header_t, length);
1403           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1404             {
1405               sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1406                                      s0->ext_host_addr.as_u32,
1407                                      ip4_header_t, dst_address);
1408               sum0 =
1409                 ip_csum_update (sum0,
1410                                 vnet_buffer (b0)->ip.reass.l4_dst_port,
1411                                 s0->ext_host_port, ip4_header_t, length);
1412               udp0->dst_port = s0->ext_host_port;
1413               ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1414             }
1415           udp0->checksum = ip_csum_fold (sum0);
1416           vlib_increment_simple_counter (&sm->counters.slowpath.in2out_ed.udp,
1417                                          thread_index, sw_if_index0, 1);
1418         }
1419       else
1420         {
1421           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1422             {
1423               new_port0 = udp0->src_port = s0->out2in.port;
1424               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1425                 {
1426                   udp0->dst_port = s0->ext_host_port;
1427                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1428                 }
1429               vlib_increment_simple_counter (&sm->counters.slowpath.
1430                                              in2out_ed.udp, thread_index,
1431                                              sw_if_index0, 1);
1432             }
1433         }
1434
1435       /* Accounting */
1436       nat44_session_update_counters (s0, now,
1437                                      vlib_buffer_length_in_chain
1438                                      (vm, b0), thread_index);
1439       /* Per-user LRU list maintenance */
1440       nat44_session_update_lru (sm, s0, thread_index);
1441
1442     trace0:
1443       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1444                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1445         {
1446           nat_in2out_ed_trace_t *t =
1447             vlib_add_trace (vm, node, b0, sizeof (*t));
1448           t->sw_if_index = sw_if_index0;
1449           t->next_index = next[0];
1450           t->is_slow_path = 1;
1451
1452           if (s0)
1453             t->session_index = s0 - tsm->sessions;
1454           else
1455             t->session_index = ~0;
1456         }
1457
1458       if (next[0] == NAT_NEXT_DROP)
1459         {
1460           vlib_increment_simple_counter (&sm->counters.slowpath.
1461                                          in2out_ed.drops, thread_index,
1462                                          sw_if_index0, 1);
1463         }
1464
1465       n_left_from--;
1466       next++;
1467       b++;
1468     }
1469
1470   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1471                                frame->n_vectors);
1472
1473   return frame->n_vectors;
1474 }
1475
1476 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1477                                      vlib_node_runtime_t * node,
1478                                      vlib_frame_t * frame)
1479 {
1480   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1481 }
1482
1483 /* *INDENT-OFF* */
1484 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1485   .name = "nat44-ed-in2out",
1486   .vector_size = sizeof (u32),
1487   .sibling_of = "nat-default",
1488   .format_trace = format_nat_in2out_ed_trace,
1489   .type = VLIB_NODE_TYPE_INTERNAL,
1490   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1491   .error_strings = nat_in2out_ed_error_strings,
1492   .runtime_data_bytes = sizeof (snat_runtime_t),
1493 };
1494 /* *INDENT-ON* */
1495
1496 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1497                                             vlib_node_runtime_t * node,
1498                                             vlib_frame_t * frame)
1499 {
1500   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1501 }
1502
1503 /* *INDENT-OFF* */
1504 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1505   .name = "nat44-ed-in2out-output",
1506   .vector_size = sizeof (u32),
1507   .sibling_of = "nat-default",
1508   .format_trace = format_nat_in2out_ed_trace,
1509   .type = VLIB_NODE_TYPE_INTERNAL,
1510   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1511   .error_strings = nat_in2out_ed_error_strings,
1512   .runtime_data_bytes = sizeof (snat_runtime_t),
1513 };
1514 /* *INDENT-ON* */
1515
1516 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1517                                               vlib_node_runtime_t *
1518                                               node, vlib_frame_t * frame)
1519 {
1520   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1521 }
1522
1523 /* *INDENT-OFF* */
1524 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1525   .name = "nat44-ed-in2out-slowpath",
1526   .vector_size = sizeof (u32),
1527   .sibling_of = "nat-default",
1528   .format_trace = format_nat_in2out_ed_trace,
1529   .type = VLIB_NODE_TYPE_INTERNAL,
1530   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1531   .error_strings = nat_in2out_ed_error_strings,
1532   .runtime_data_bytes = sizeof (snat_runtime_t),
1533 };
1534 /* *INDENT-ON* */
1535
1536 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1537                                                      vlib_node_runtime_t
1538                                                      * node,
1539                                                      vlib_frame_t * frame)
1540 {
1541   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1542 }
1543
1544 /* *INDENT-OFF* */
1545 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1546   .name = "nat44-ed-in2out-output-slowpath",
1547   .vector_size = sizeof (u32),
1548   .sibling_of = "nat-default",
1549   .format_trace = format_nat_in2out_ed_trace,
1550   .type = VLIB_NODE_TYPE_INTERNAL,
1551   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1552   .error_strings = nat_in2out_ed_error_strings,
1553   .runtime_data_bytes = sizeof (snat_runtime_t),
1554 };
1555 /* *INDENT-ON* */
1556
1557 static u8 *
1558 format_nat_pre_trace (u8 * s, va_list * args)
1559 {
1560   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1561   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1562   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1563   return format (s, "in2out next_index %d arc_next_index %d", t->next_index,
1564                  t->arc_next_index);
1565 }
1566
1567 VLIB_NODE_FN (nat_pre_in2out_node)
1568   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1569 {
1570   return nat_pre_node_fn_inline (vm, node, frame,
1571                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1572 }
1573
1574 VLIB_NODE_FN (nat_pre_in2out_output_node)
1575   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1576 {
1577   return nat_pre_node_fn_inline (vm, node, frame,
1578                                  NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH);
1579 }
1580
1581 /* *INDENT-OFF* */
1582 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1583   .name = "nat-pre-in2out",
1584   .vector_size = sizeof (u32),
1585   .sibling_of = "nat-default",
1586   .format_trace = format_nat_pre_trace,
1587   .type = VLIB_NODE_TYPE_INTERNAL,
1588   .n_errors = 0,
1589 };
1590
1591 VLIB_REGISTER_NODE (nat_pre_in2out_output_node) = {
1592   .name = "nat-pre-in2out-output",
1593   .vector_size = sizeof (u32),
1594   .sibling_of = "nat-default",
1595   .format_trace = format_nat_pre_trace,
1596   .type = VLIB_NODE_TYPE_INTERNAL,
1597   .n_errors = 0,
1598 };
1599 /* *INDENT-ON* */
1600
1601 /*
1602  * fd.io coding-style-patch-verification: ON
1603  *
1604  * Local Variables:
1605  * eval: (c-set-style "gnu")
1606  * End:
1607  */