src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <nat/nat44/ed_inlines.h>
  35
  36 static char *nat_in2out_ed_error_strings[] = {
  37 #define _(sym,string) string,
  38   foreach_nat_in2out_ed_error
  39 #undef _
  40 };
  41
  42 typedef struct
  43 {
  44   u32 sw_if_index;
  45   u32 next_index;
  46   u32 session_index;
  47   u32 is_slow_path;
  48 } nat_in2out_ed_trace_t;
  49
  50 static u8 *
  51 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  56   char *tag;
  57
  58   tag =
  59     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  60     "NAT44_IN2OUT_ED_FAST_PATH";
  61
  62   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  63               t->sw_if_index, t->next_index, t->session_index);
  64
  65   return s;
  66 }
  67
  68 #ifndef CLIB_MARCH_VARIANT
  69 int
  70 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  71 {
  72   snat_main_t *sm = &snat_main;
  73   nat44_is_idle_session_ctx_t *ctx = arg;
  74   snat_session_t *s;
  75   u64 sess_timeout_time;
  76   u8 proto;
  77   u16 r_port, l_port;
  78   ip4_address_t *l_addr, *r_addr;
  79   u32 fib_index;
  80   clib_bihash_kv_16_8_t ed_kv;
  81   int i;
  82   snat_address_t *a;
  83   snat_session_key_t key;
  84   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  85                                                        ctx->thread_index);
  86
  87   s = pool_elt_at_index (tsm->sessions, kv->value);
  88   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  89   if (ctx->now >= sess_timeout_time)
  90     {
  91       if (is_fwd_bypass_session (s))
  92         goto delete;
  93
  94       l_addr = &s->out2in.addr;
  95       r_addr = &s->ext_host_addr;
  96       fib_index = s->out2in.fib_index;
  97       if (snat_is_unk_proto_session (s))
  98         {
  99           proto = s->in2out.port;
 100           r_port = 0;
 101           l_port = 0;
 102         }
 103       else
 104         {
 105           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 106           l_port = s->out2in.port;
 107           r_port = s->ext_host_port;
 108         }
 109       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 110                   &ed_kv);
 111       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 112         nat_elog_warn ("out2in_ed key del failed");
 113
 114       if (snat_is_unk_proto_session (s))
 115         goto delete;
 116
 117       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 118                                            s->in2out.addr.as_u32,
 119                                            s->out2in.addr.as_u32,
 120                                            s->in2out.protocol,
 121                                            s->in2out.port,
 122                                            s->out2in.port,
 123                                            s->in2out.fib_index);
 124
 125       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 126                              &s->in2out.addr, s->in2out.port,
 127                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 128                              &s->out2in.addr, s->out2in.port,
 129                              &s->ext_host_addr, s->ext_host_port,
 130                              s->in2out.protocol, is_twice_nat_session (s));
 131
 132       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 133                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 134                    ctx->thread_index);
 135
 136       if (is_twice_nat_session (s))
 137         {
 138           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 139             {
 140               key.protocol = s->in2out.protocol;
 141               key.port = s->ext_host_nat_port;
 142               a = sm->twice_nat_addresses + i;
 143               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 144                 {
 145                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 146                                                       ctx->thread_index,
 147                                                       &key);
 148                   break;
 149                 }
 150             }
 151         }
 152
 153       if (snat_is_session_static (s))
 154         goto delete;
 155
 156       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 157                                           &s->out2in);
 158     delete:
 159       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 160       return 1;
 161     }
 162
 163   return 0;
 164 }
 165 #endif
 166
 167 static inline u32
 168 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 169                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 170                           u32 sw_if_index0, u32 rx_fib_index0,
 171                           vlib_node_runtime_t * node, u32 next0, f64 now,
 172                           u32 thread_index, snat_session_t ** p_s0)
 173 {
 174   vlib_main_t *vm = vlib_get_main ();
 175
 176   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 177                        next0, thread_index, p_s0, 0);
 178   snat_session_t *s0 = *p_s0;
 179   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 180     {
 181       /* Accounting */
 182       nat44_session_update_counters (s0, now,
 183                                      vlib_buffer_length_in_chain
 184                                      (vm, b0), thread_index);
 185       /* Per-user LRU list maintenance */
 186       nat44_session_update_lru (sm, s0, thread_index);
 187     }
 188   return next0;
 189 }
 190
 191 static_always_inline u16
 192 snat_random_port (u16 min, u16 max)
 193 {
 194   snat_main_t *sm = &snat_main;
 195   return min + random_u32 (&sm->random_seed) /
 196     (random_u32_max () / (max - min + 1) + 1);
 197 }
 198
 199 static int
 200 nat_ed_alloc_addr_and_port (snat_main_t * sm, u32 rx_fib_index,
 201                             u32 nat_proto, u32 thread_index,
 202                             ip4_address_t r_addr, u16 r_port, u8 proto,
 203                             u16 port_per_thread, u32 snat_thread_index,
 204                             snat_session_t * s,
 205                             ip4_address_t * allocated_addr,
 206                             u16 * allocated_port,
 207                             clib_bihash_kv_16_8_t * out2in_ed_kv)
 208 {
 209   int i;
 210   snat_address_t *a, *ga = 0;
 211   u32 portnum;
 212   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 213
 214   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 215
 216   for (i = 0; i < vec_len (sm->addresses); i++)
 217     {
 218       a = sm->addresses + i;
 219       switch (nat_proto)
 220         {
 221 #define _(N, j, n, unused)                                                    \
 222   case NAT_PROTOCOL_##N:                                                     \
 223     if (a->fib_index == rx_fib_index)                                         \
 224       {                                                                       \
 225         u16 port = snat_random_port (1, port_per_thread);                     \
 226         u16 attempts = port_per_thread;                                       \
 227         while (attempts > 0)                                                  \
 228           {                                                                   \
 229             --attempts;                                                       \
 230             portnum = port_thread_offset + port;                              \
 231             make_ed_kv (&a->addr, &r_addr, proto, s->out2in.fib_index,        \
 232                         clib_host_to_net_u16 (portnum), r_port,               \
 233                         s - tsm->sessions, out2in_ed_kv);                     \
 234             int rv = clib_bihash_add_del_16_8 (&tsm->out2in_ed, out2in_ed_kv, \
 235                                                2 /* is_add */);               \
 236             if (0 == rv)                                                      \
 237               {                                                               \
 238                 ++a->busy_##n##_port_refcounts[portnum];                      \
 239                 a->busy_##n##_ports_per_thread[thread_index]++;               \
 240                 a->busy_##n##_ports++;                                        \
 241                 *allocated_addr = a->addr;                                    \
 242                 *allocated_port = clib_host_to_net_u16 (portnum);             \
 243                 return 0;                                                     \
 244               }                                                               \
 245             port = (port + 1) % port_per_thread;                              \
 246           }                                                                   \
 247       }                                                                       \
 248     else if (a->fib_index == ~0)                                              \
 249       {                                                                       \
 250         ga = a;                                                               \
 251       }                                                                       \
 252     break;
 253
 254           foreach_nat_protocol;
 255         default:
 256           nat_elog_info ("unknown protocol");
 257           return 1;
 258         }
 259     }
 260
 261   if (ga)
 262     {
 263       /* fake fib_index to reuse macro */
 264       rx_fib_index = ~0;
 265       a = ga;
 266       switch (nat_proto)
 267         {
 268           foreach_nat_protocol;
 269         default:
 270           nat_elog_info ("unknown protocol");
 271           return 1;
 272         }
 273     }
 274
 275 #undef _
 276
 277   /* Totally out of translations to use... */
 278   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 279   return 1;
 280 }
 281
 282 static u32
 283 slow_path_ed (snat_main_t * sm,
 284               vlib_buffer_t * b,
 285               ip4_address_t l_addr,
 286               ip4_address_t r_addr,
 287               u16 l_port,
 288               u16 r_port,
 289               u8 proto,
 290               u32 rx_fib_index,
 291               snat_session_t ** sessionp,
 292               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 293 {
 294   snat_session_t *s = NULL;
 295   snat_session_key_t key0, key1;
 296   lb_nat_type_t lb = 0;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   u32 nat_proto = ip_proto_to_nat_proto (proto);
 299   nat_outside_fib_t *outside_fib;
 300   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 301   clib_bihash_kv_16_8_t out2in_ed_kv;
 302   ip4_address_t allocated_addr;
 303   u16 allocated_port;
 304   u8 identity_nat;
 305   fib_prefix_t pfx = {
 306     .fp_proto = FIB_PROTOCOL_IP4,
 307     .fp_len = 32,
 308     .fp_addr = {.ip4.as_u32 = r_addr.as_u32,},
 309   };
 310   nat44_is_idle_session_ctx_t ctx;
 311
 312   if (PREDICT_TRUE (nat_proto == NAT_PROTOCOL_TCP))
 313     {
 314       if (PREDICT_FALSE
 315           (!tcp_flags_is_init
 316            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 317         {
 318           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 319           return NAT_NEXT_DROP;
 320         }
 321     }
 322
 323   // TODO: based on fib index do a lookup
 324   if (PREDICT_FALSE
 325       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 326     {
 327       if (!nat_lru_free_one (sm, thread_index, now))
 328         {
 329           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 330           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 331           nat_elog_notice ("maximum sessions exceeded");
 332           return NAT_NEXT_DROP;
 333         }
 334     }
 335
 336   key0.addr = l_addr;
 337   key0.port = l_port;
 338   key1.protocol = key0.protocol = nat_proto;
 339   key0.fib_index = rx_fib_index;
 340   key1.fib_index = sm->outside_fib_index;
 341
 342   /* First try to match static mapping by local address and port */
 343   if (snat_static_mapping_match
 344       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 345     {
 346       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 347       if (!s)
 348         {
 349           nat_elog_warn ("create NAT session failed");
 350           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 351           return NAT_NEXT_DROP;
 352         }
 353       switch (vec_len (sm->outside_fibs))
 354         {
 355         case 0:
 356           s->out2in.fib_index = sm->outside_fib_index;
 357           break;
 358         case 1:
 359           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 360           break;
 361         default:
 362           /* *INDENT-OFF* */
 363           vec_foreach (outside_fib, sm->outside_fibs)
 364           {
 365             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 366             if (FIB_NODE_INDEX_INVALID != fei)
 367               {
 368                 if (fib_entry_get_resolving_interface (fei) != ~0)
 369                   {
 370                     s->out2in.fib_index = outside_fib->fib_index;
 371                     break;
 372                   }
 373               }
 374           }
 375           /* *INDENT-ON* */
 376           break;
 377         }
 378
 379       /* Try to create dynamic translation */
 380       if (nat_ed_alloc_addr_and_port (sm, rx_fib_index, nat_proto,
 381                                       thread_index, r_addr, r_port, proto,
 382                                       sm->port_per_thread,
 383                                       tsm->snat_thread_index, s,
 384                                       &allocated_addr,
 385                                       &allocated_port, &out2in_ed_kv))
 386         {
 387           nat_elog_notice ("addresses exhausted");
 388           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 389           nat_free_session_data (sm, s, thread_index, 0);
 390           nat_ed_session_delete (sm, s, thread_index, 1);
 391           return NAT_NEXT_DROP;
 392         }
 393       key1.addr = allocated_addr;
 394       key1.port = allocated_port;
 395     }
 396   else
 397     {
 398       if (PREDICT_FALSE (identity_nat))
 399         {
 400           *sessionp = s;
 401           return next;
 402         }
 403       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 404       if (!s)
 405         {
 406           nat_elog_warn ("create NAT session failed");
 407           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 408           return NAT_NEXT_DROP;
 409         }
 410       switch (vec_len (sm->outside_fibs))
 411         {
 412         case 0:
 413           s->out2in.fib_index = sm->outside_fib_index;
 414           break;
 415         case 1:
 416           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 417           break;
 418         default:
 419           /* *INDENT-OFF* */
 420           vec_foreach (outside_fib, sm->outside_fibs)
 421           {
 422             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 423             if (FIB_NODE_INDEX_INVALID != fei)
 424               {
 425                 if (fib_entry_get_resolving_interface (fei) != ~0)
 426                   {
 427                     s->out2in.fib_index = outside_fib->fib_index;
 428                     break;
 429                   }
 430               }
 431           }
 432           /* *INDENT-ON* */
 433           break;
 434         }
 435
 436       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 437
 438
 439       make_ed_kv (&key1.addr, &r_addr, proto,
 440                   s->out2in.fib_index, key1.port, r_port, s - tsm->sessions,
 441                   &out2in_ed_kv);
 442       if (clib_bihash_add_or_overwrite_stale_16_8
 443           (&tsm->out2in_ed, &out2in_ed_kv, nat44_o2i_ed_is_idle_session_cb,
 444            &ctx))
 445         nat_elog_notice ("out2in-ed key add failed");
 446     }
 447
 448   if (lb)
 449     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 450   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 451   s->ext_host_addr = r_addr;
 452   s->ext_host_port = r_port;
 453   s->in2out = key0;
 454   s->out2in = key1;
 455   s->out2in.protocol = key0.protocol;
 456
 457   clib_bihash_kv_16_8_t in2out_ed_kv;
 458   make_ed_kv (&l_addr, &r_addr, proto, rx_fib_index, l_port, r_port,
 459               s - tsm->sessions, &in2out_ed_kv);
 460   ctx.now = now;
 461   ctx.thread_index = thread_index;
 462   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &in2out_ed_kv,
 463                                                nat44_i2o_ed_is_idle_session_cb,
 464                                                &ctx))
 465     nat_elog_notice ("in2out-ed key add failed");
 466
 467   *sessionp = s;
 468
 469   /* log NAT event */
 470   snat_ipfix_logging_nat44_ses_create (thread_index,
 471                                        s->in2out.addr.as_u32,
 472                                        s->out2in.addr.as_u32,
 473                                        s->in2out.protocol,
 474                                        s->in2out.port,
 475                                        s->out2in.port, s->in2out.fib_index);
 476
 477   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 478                          &s->in2out.addr, s->in2out.port,
 479                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 480                          &s->out2in.addr, s->out2in.port,
 481                          &s->ext_host_addr, s->ext_host_port,
 482                          s->in2out.protocol, 0);
 483
 484   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 485                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 486                &s->ext_host_nat_addr, s->ext_host_nat_port,
 487                s->in2out.protocol, s->in2out.fib_index, s->flags,
 488                thread_index, 0);
 489
 490   return next;
 491 }
 492
 493 static_always_inline int
 494 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 495                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 496                         u32 rx_fib_index, u32 thread_index)
 497 {
 498   udp_header_t *udp = ip4_next_header (ip);
 499   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 500   clib_bihash_kv_16_8_t kv, value;
 501   snat_session_key_t key0, key1;
 502
 503   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 504               sm->outside_fib_index, udp->dst_port, udp->src_port, ~0ULL,
 505               &kv);
 506
 507   /* NAT packet aimed at external address if has active sessions */
 508   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 509     {
 510       key0.addr = ip->dst_address;
 511       key0.port = udp->dst_port;
 512       key0.protocol = proto;
 513       key0.fib_index = sm->outside_fib_index;
 514       /* or is static mappings */
 515       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 516         return 0;
 517     }
 518   else
 519     return 0;
 520
 521   if (sm->forwarding_enabled)
 522     return 1;
 523
 524   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 525                                   rx_fib_index);
 526 }
 527
 528 static_always_inline int
 529 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 530                                       u32 thread_index, f64 now,
 531                                       vlib_main_t * vm, vlib_buffer_t * b)
 532 {
 533   clib_bihash_kv_16_8_t kv, value;
 534   snat_session_t *s = 0;
 535   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 536
 537   if (!sm->forwarding_enabled)
 538     return 0;
 539
 540   if (ip->protocol == IP_PROTOCOL_ICMP)
 541     {
 542       if (get_icmp_i2o_ed_key (b, ip, 0, ~0ULL, 0, 0, 0, &kv))
 543         return 0;
 544     }
 545   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 546     {
 547       make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol, 0,
 548                   vnet_buffer (b)->ip.reass.l4_src_port,
 549                   vnet_buffer (b)->ip.reass.l4_dst_port, ~0ULL, &kv);
 550     }
 551   else
 552     {
 553       make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 554                   0, ~0ULL, &kv);
 555     }
 556
 557   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 558     {
 559       s = pool_elt_at_index (tsm->sessions, value.value);
 560       if (is_fwd_bypass_session (s))
 561         {
 562           if (ip->protocol == IP_PROTOCOL_TCP)
 563             {
 564               if (nat44_set_tcp_session_state_i2o
 565                   (sm, now, s, b, thread_index))
 566                 return 1;
 567             }
 568           /* Accounting */
 569           nat44_session_update_counters (s, now,
 570                                          vlib_buffer_length_in_chain (vm, b),
 571                                          thread_index);
 572           /* Per-user LRU list maintenance */
 573           nat44_session_update_lru (sm, s, thread_index);
 574           return 1;
 575         }
 576       else
 577         return 0;
 578     }
 579
 580   return 0;
 581 }
 582
 583 static_always_inline int
 584 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 585                                        u16 src_port, u16 dst_port,
 586                                        u32 thread_index, u32 rx_sw_if_index,
 587                                        u32 tx_sw_if_index)
 588 {
 589   clib_bihash_kv_16_8_t kv, value;
 590   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 591   snat_interface_t *i;
 592   snat_session_t *s;
 593   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 594   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 595
 596   /* src NAT check */
 597   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 598               tx_fib_index, src_port, dst_port, ~0ULL, &kv);
 599   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 600     {
 601       s = pool_elt_at_index (tsm->sessions, value.value);
 602       if (nat44_is_ses_closed (s))
 603         {
 604           nat_free_session_data (sm, s, thread_index, 0);
 605           nat_ed_session_delete (sm, s, thread_index, 1);
 606         }
 607       else
 608         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 609       return 1;
 610     }
 611
 612   /* dst NAT check */
 613   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 614               rx_fib_index, dst_port, src_port, ~0ULL, &kv);
 615   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 616     {
 617       s = pool_elt_at_index (tsm->sessions, value.value);
 618       if (is_fwd_bypass_session (s))
 619         return 0;
 620
 621       /* hairpinning */
 622       /* *INDENT-OFF* */
 623       pool_foreach (i, sm->output_feature_interfaces,
 624       ({
 625         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 626            return 0;
 627       }));
 628       /* *INDENT-ON* */
 629       return 1;
 630     }
 631
 632   return 0;
 633 }
 634
 635 #ifndef CLIB_MARCH_VARIANT
 636 u32
 637 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 638                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 639                       u8 * p_proto, snat_session_key_t * p_value,
 640                       u8 * p_dont_translate, void *d, void *e)
 641 {
 642   u32 sw_if_index;
 643   u32 rx_fib_index;
 644   snat_session_t *s = 0;
 645   u8 dont_translate = 0;
 646   clib_bihash_kv_16_8_t kv, value;
 647   u32 next = ~0;
 648   int err;
 649   u16 l_port = 0, r_port = 0;   // initialize to workaround gcc warning
 650   vlib_main_t *vm = vlib_get_main ();
 651   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 652
 653   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 654   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 655
 656   err =
 657     get_icmp_i2o_ed_key (b, ip, rx_fib_index, ~0ULL, p_proto, &l_port,
 658                          &r_port, &kv);
 659   if (err != 0)
 660     {
 661       b->error = node->errors[err];
 662       next = NAT_NEXT_DROP;
 663       goto out;
 664     }
 665
 666   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 667     {
 668       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 669         {
 670           if (PREDICT_FALSE
 671               (nat44_ed_not_translate_output_feature
 672                (sm, ip, l_port, r_port, thread_index,
 673                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX])))
 674             {
 675               dont_translate = 1;
 676               goto out;
 677             }
 678         }
 679       else
 680         {
 681           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 682                                                      ip, NAT_PROTOCOL_ICMP,
 683                                                      rx_fib_index,
 684                                                      thread_index)))
 685             {
 686               dont_translate = 1;
 687               goto out;
 688             }
 689         }
 690
 691       if (PREDICT_FALSE
 692           (icmp_type_is_error_message
 693            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 694         {
 695           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 696           next = NAT_NEXT_DROP;
 697           goto out;
 698         }
 699
 700       next =
 701         slow_path_ed (sm, b, ip->src_address, ip->dst_address, l_port, r_port,
 702                       ip->protocol, rx_fib_index, &s, node, next,
 703                       thread_index, vlib_time_now (vm));
 704
 705       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 706         goto out;
 707
 708       if (!s)
 709         {
 710           dont_translate = 1;
 711           goto out;
 712         }
 713     }
 714   else
 715     {
 716       if (PREDICT_FALSE
 717           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 718            ICMP4_echo_request
 719            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 720            ICMP4_echo_reply
 721            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 722                                            reass.icmp_type_or_tcp_flags)))
 723         {
 724           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 725           next = NAT_NEXT_DROP;
 726           goto out;
 727         }
 728
 729       s = pool_elt_at_index (tsm->sessions, value.value);
 730     }
 731 out:
 732   if (s)
 733     *p_value = s->out2in;
 734   *p_dont_translate = dont_translate;
 735   if (d)
 736     *(snat_session_t **) d = s;
 737   return next;
 738 }
 739 #endif
 740
 741 static snat_session_t *
 742 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 743                                vlib_buffer_t * b,
 744                                ip4_header_t * ip,
 745                                u32 rx_fib_index,
 746                                u32 thread_index,
 747                                f64 now,
 748                                vlib_main_t * vm, vlib_node_runtime_t * node)
 749 {
 750   clib_bihash_kv_8_8_t kv, value;
 751   clib_bihash_kv_16_8_t s_kv, s_value;
 752   snat_static_mapping_t *m;
 753   u32 old_addr, new_addr = 0;
 754   ip_csum_t sum;
 755   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 756   snat_session_t *s;
 757   u32 outside_fib_index = sm->outside_fib_index;
 758   int i;
 759   u8 is_sm = 0;
 760   nat_outside_fib_t *outside_fib;
 761   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 762   fib_prefix_t pfx = {
 763     .fp_proto = FIB_PROTOCOL_IP4,
 764     .fp_len = 32,
 765     .fp_addr = {
 766                 .ip4.as_u32 = ip->dst_address.as_u32,
 767                 },
 768   };
 769
 770   switch (vec_len (sm->outside_fibs))
 771     {
 772     case 0:
 773       outside_fib_index = sm->outside_fib_index;
 774       break;
 775     case 1:
 776       outside_fib_index = sm->outside_fibs[0].fib_index;
 777       break;
 778     default:
 779       /* *INDENT-OFF* */
 780       vec_foreach (outside_fib, sm->outside_fibs)
 781         {
 782           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 783           if (FIB_NODE_INDEX_INVALID != fei)
 784             {
 785               if (fib_entry_get_resolving_interface (fei) != ~0)
 786                 {
 787                   outside_fib_index = outside_fib->fib_index;
 788                   break;
 789                 }
 790             }
 791         }
 792       /* *INDENT-ON* */
 793       break;
 794     }
 795   old_addr = ip->src_address.as_u32;
 796
 797   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 798               rx_fib_index, 0, 0, ~0ULL, &s_kv);
 799
 800   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 801     {
 802       s = pool_elt_at_index (tsm->sessions, s_value.value);
 803       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 804     }
 805   else
 806     {
 807       if (PREDICT_FALSE
 808           (nat44_ed_maximum_sessions_exceeded
 809            (sm, rx_fib_index, thread_index)))
 810         {
 811           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 812           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 813           nat_elog_notice ("maximum sessions exceeded");
 814           return 0;
 815         }
 816
 817       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 818
 819       /* Try to find static mapping first */
 820       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 821         {
 822           m = pool_elt_at_index (sm->static_mappings, value.value);
 823           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 824           is_sm = 1;
 825           goto create_ses;
 826         }
 827       else
 828         {
 829           /* *INDENT-OFF* */
 830           pool_foreach (s, tsm->sessions, {
 831             if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 832               {
 833                 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 834
 835                 make_ed_kv (&s->out2in.addr, &ip->dst_address, ip->protocol,
 836                             outside_fib_index, 0, 0, ~0ULL, &s_kv);
 837                 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 838                   goto create_ses;
 839
 840                 break;
 841               }
 842           });
 843           /* *INDENT-ON* */
 844
 845           for (i = 0; i < vec_len (sm->addresses); i++)
 846             {
 847               make_ed_kv (&sm->addresses[i].addr, &ip->dst_address,
 848                           ip->protocol, outside_fib_index, 0, 0, ~0ULL,
 849                           &s_kv);
 850               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 851                 {
 852                   new_addr = ip->src_address.as_u32 =
 853                     sm->addresses[i].addr.as_u32;
 854                   goto create_ses;
 855                 }
 856             }
 857           return 0;
 858         }
 859
 860     create_ses:
 861       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 862       if (!s)
 863         {
 864           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 865           nat_elog_warn ("create NAT session failed");
 866           return 0;
 867         }
 868
 869       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 870       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 871       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 872       s->out2in.addr.as_u32 = new_addr;
 873       s->out2in.fib_index = outside_fib_index;
 874       s->in2out.addr.as_u32 = old_addr;
 875       s->in2out.fib_index = rx_fib_index;
 876       s->in2out.port = s->out2in.port = ip->protocol;
 877       if (is_sm)
 878         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 879
 880       /* Add to lookup tables */
 881       make_ed_kv (&s->in2out.addr, &ip->dst_address, ip->protocol,
 882                   rx_fib_index, 0, 0, s - tsm->sessions, &s_kv);
 883       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 884         nat_elog_notice ("in2out key add failed");
 885
 886       make_ed_kv (&s->out2in.addr, &ip->dst_address, ip->protocol,
 887                   outside_fib_index, 0, 0, s - tsm->sessions, &s_kv);
 888       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 889         nat_elog_notice ("out2in key add failed");
 890     }
 891
 892   /* Update IP checksum */
 893   sum = ip->checksum;
 894   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 895   ip->checksum = ip_csum_fold (sum);
 896
 897   /* Accounting */
 898   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 899                                  thread_index);
 900   /* Per-user LRU list maintenance */
 901   nat44_session_update_lru (sm, s, thread_index);
 902
 903   /* Hairpinning */
 904   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 905     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 906
 907   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 908     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 909
 910   return s;
 911 }
 912
 913 static inline uword
 914 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 915                                           vlib_node_runtime_t * node,
 916                                           vlib_frame_t * frame,
 917                                           int is_output_feature)
 918 {
 919   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 920   nat_next_t next_index;
 921   snat_main_t *sm = &snat_main;
 922   f64 now = vlib_time_now (vm);
 923   u32 thread_index = vm->thread_index;
 924   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 925   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 926     0, def_slow;
 927
 928   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 929     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 930
 931   stats_node_index = sm->ed_in2out_node_index;
 932
 933   from = vlib_frame_vector_args (frame);
 934   n_left_from = frame->n_vectors;
 935   next_index = node->cached_next_index;
 936
 937   while (n_left_from > 0)
 938     {
 939       u32 n_left_to_next;
 940
 941       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 942
 943       while (n_left_from > 0 && n_left_to_next > 0)
 944         {
 945           u32 bi0;
 946           vlib_buffer_t *b0;
 947           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 948             new_addr0, old_addr0;
 949           u16 old_port0, new_port0;
 950           ip4_header_t *ip0;
 951           udp_header_t *udp0;
 952           tcp_header_t *tcp0;
 953           snat_session_t *s0 = 0;
 954           clib_bihash_kv_16_8_t kv0, value0;
 955           ip_csum_t sum0;
 956
 957           /* speculatively enqueue b0 to the current next frame */
 958           bi0 = from[0];
 959           to_next[0] = bi0;
 960           from += 1;
 961           to_next += 1;
 962           n_left_from -= 1;
 963           n_left_to_next -= 1;
 964
 965           b0 = vlib_get_buffer (vm, bi0);
 966
 967           if (is_output_feature)
 968             {
 969               vnet_feature_next (&vnet_buffer2 (b0)->nat.arc_next, b0);
 970               iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 971             }
 972
 973           next0 = vnet_buffer2 (b0)->nat.arc_next;
 974
 975           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 976                                   iph_offset0);
 977
 978           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 979           rx_fib_index0 =
 980             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 981                                                  sw_if_index0);
 982
 983           if (PREDICT_FALSE (ip0->ttl == 1))
 984             {
 985               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 986               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 987                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 988                                            0);
 989               next0 = NAT_NEXT_ICMP_ERROR;
 990               goto trace0;
 991             }
 992
 993           udp0 = ip4_next_header (ip0);
 994           tcp0 = (tcp_header_t *) udp0;
 995           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 996
 997           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 998             {
 999               next0 = def_slow;
1000               goto trace0;
1001             }
1002
1003           if (is_output_feature)
1004             {
1005               if (PREDICT_FALSE (nat_not_translate_output_feature_fwd
1006                                  (sm, ip0, thread_index, now, vm, b0)))
1007                 goto trace0;
1008             }
1009
1010           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1011             {
1012               next0 = def_slow;
1013               goto trace0;
1014             }
1015
1016           make_ed_kv (&ip0->src_address, &ip0->dst_address,
1017                       ip0->protocol, rx_fib_index0,
1018                       vnet_buffer (b0)->ip.reass.l4_src_port,
1019                       vnet_buffer (b0)->ip.reass.l4_dst_port, ~0ULL, &kv0);
1020
1021           // lookup for session
1022           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1023             {
1024               // session does not exist go slow path
1025               next0 = def_slow;
1026               goto trace0;
1027             }
1028           s0 = pool_elt_at_index (tsm->sessions, value0.value);
1029
1030           if (s0->tcp_closed_timestamp)
1031             {
1032               if (now >= s0->tcp_closed_timestamp)
1033                 {
1034                   // session is closed, go slow path
1035                   next0 = def_slow;
1036                 }
1037               else
1038                 {
1039                   // session in transitory timeout, drop
1040                   b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1041                   next0 = NAT_NEXT_DROP;
1042                 }
1043               goto trace0;
1044             }
1045
1046           // drop if session expired
1047           u64 sess_timeout_time;
1048           sess_timeout_time = s0->last_heard +
1049             (f64) nat44_session_get_timeout (sm, s0);
1050           if (now >= sess_timeout_time)
1051             {
1052               nat_free_session_data (sm, s0, thread_index, 0);
1053               nat_ed_session_delete (sm, s0, thread_index, 1);
1054               // session is closed, go slow path
1055               next0 = def_slow;
1056               goto trace0;
1057             }
1058
1059           b0->flags |= VNET_BUFFER_F_IS_NATED;
1060
1061           if (!is_output_feature)
1062             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1063
1064           old_addr0 = ip0->src_address.as_u32;
1065           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1066           sum0 = ip0->checksum;
1067           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1068                                  src_address);
1069           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1070             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1071                                    s0->ext_host_addr.as_u32, ip4_header_t,
1072                                    dst_address);
1073           ip0->checksum = ip_csum_fold (sum0);
1074
1075           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1076
1077           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1078             {
1079               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1080                 {
1081                   new_port0 = udp0->src_port = s0->out2in.port;
1082                   sum0 = tcp0->checksum;
1083                   sum0 =
1084                     ip_csum_update (sum0, old_addr0, new_addr0,
1085                                     ip4_header_t, dst_address);
1086                   sum0 =
1087                     ip_csum_update (sum0, old_port0, new_port0,
1088                                     ip4_header_t, length);
1089                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1090                     {
1091                       sum0 =
1092                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1093                                         s0->ext_host_addr.as_u32,
1094                                         ip4_header_t, dst_address);
1095                       sum0 =
1096                         ip_csum_update (sum0,
1097                                         vnet_buffer (b0)->ip.
1098                                         reass.l4_dst_port, s0->ext_host_port,
1099                                         ip4_header_t, length);
1100                       tcp0->dst_port = s0->ext_host_port;
1101                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1102                     }
1103                   mss_clamping (sm, tcp0, &sum0);
1104                   tcp0->checksum = ip_csum_fold (sum0);
1105                 }
1106               tcp_packets++;
1107               if (nat44_set_tcp_session_state_i2o
1108                   (sm, now, s0, b0, thread_index))
1109                 goto trace0;
1110             }
1111           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1112                    && udp0->checksum)
1113             {
1114               new_port0 = udp0->src_port = s0->out2in.port;
1115               sum0 = udp0->checksum;
1116               sum0 =
1117                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1118                                 dst_address);
1119               sum0 =
1120                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1121                                 length);
1122               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1123                 {
1124                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1125                                          s0->ext_host_addr.as_u32,
1126                                          ip4_header_t, dst_address);
1127                   sum0 =
1128                     ip_csum_update (sum0,
1129                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1130                                     s0->ext_host_port, ip4_header_t, length);
1131                   udp0->dst_port = s0->ext_host_port;
1132                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1133                 }
1134               udp0->checksum = ip_csum_fold (sum0);
1135               udp_packets++;
1136             }
1137           else
1138             {
1139               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1140                 {
1141                   new_port0 = udp0->src_port = s0->out2in.port;
1142                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1143                     {
1144                       udp0->dst_port = s0->ext_host_port;
1145                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1146                     }
1147                   udp_packets++;
1148                 }
1149             }
1150
1151           /* Accounting */
1152           nat44_session_update_counters (s0, now,
1153                                          vlib_buffer_length_in_chain
1154                                          (vm, b0), thread_index);
1155           /* Per-user LRU list maintenance */
1156           nat44_session_update_lru (sm, s0, thread_index);
1157
1158         trace0:
1159           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1160                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1161             {
1162               nat_in2out_ed_trace_t *t =
1163                 vlib_add_trace (vm, node, b0, sizeof (*t));
1164               t->sw_if_index = sw_if_index0;
1165               t->next_index = next0;
1166               t->is_slow_path = 0;
1167
1168               if (s0)
1169                 t->session_index = s0 - tsm->sessions;
1170               else
1171                 t->session_index = ~0;
1172             }
1173
1174           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1175           /* verify speculative enqueue, maybe switch current next frame */
1176           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1177                                            to_next, n_left_to_next,
1178                                            bi0, next0);
1179         }
1180
1181       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1182     }
1183
1184   vlib_node_increment_counter (vm, stats_node_index,
1185                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1186                                pkts_processed);
1187   vlib_node_increment_counter (vm, stats_node_index,
1188                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1189   vlib_node_increment_counter (vm, stats_node_index,
1190                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1191   vlib_node_increment_counter (vm, stats_node_index,
1192                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1193                                icmp_packets);
1194   vlib_node_increment_counter (vm, stats_node_index,
1195                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1196                                other_packets);
1197   return frame->n_vectors;
1198 }
1199
1200 static inline uword
1201 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1202                                           vlib_node_runtime_t * node,
1203                                           vlib_frame_t * frame,
1204                                           int is_output_feature)
1205 {
1206   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
1207   nat_next_t next_index;
1208   snat_main_t *sm = &snat_main;
1209   f64 now = vlib_time_now (vm);
1210   u32 thread_index = vm->thread_index;
1211   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1212   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = 0;
1213
1214   stats_node_index = sm->ed_in2out_slowpath_node_index;
1215
1216   from = vlib_frame_vector_args (frame);
1217   n_left_from = frame->n_vectors;
1218   next_index = node->cached_next_index;
1219
1220   while (n_left_from > 0)
1221     {
1222       u32 n_left_to_next;
1223
1224       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1225
1226       while (n_left_from > 0 && n_left_to_next > 0)
1227         {
1228           u32 bi0;
1229           vlib_buffer_t *b0;
1230           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1231             new_addr0, old_addr0;
1232           u16 old_port0, new_port0;
1233           ip4_header_t *ip0;
1234           udp_header_t *udp0;
1235           tcp_header_t *tcp0;
1236           icmp46_header_t *icmp0;
1237           snat_session_t *s0 = 0;
1238           clib_bihash_kv_16_8_t kv0, value0;
1239           ip_csum_t sum0;
1240
1241           /* speculatively enqueue b0 to the current next frame */
1242           bi0 = from[0];
1243           to_next[0] = bi0;
1244           from += 1;
1245           to_next += 1;
1246           n_left_from -= 1;
1247           n_left_to_next -= 1;
1248
1249           b0 = vlib_get_buffer (vm, bi0);
1250
1251           if (is_output_feature)
1252             iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1253
1254           next0 = vnet_buffer2 (b0)->nat.arc_next;
1255
1256           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1257                                   iph_offset0);
1258
1259           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1260           rx_fib_index0 =
1261             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1262                                                  sw_if_index0);
1263
1264           if (PREDICT_FALSE (ip0->ttl == 1))
1265             {
1266               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1267               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1268                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1269                                            0);
1270               next0 = NAT_NEXT_ICMP_ERROR;
1271               goto trace0;
1272             }
1273
1274           udp0 = ip4_next_header (ip0);
1275           tcp0 = (tcp_header_t *) udp0;
1276           icmp0 = (icmp46_header_t *) udp0;
1277           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1278
1279           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1280             {
1281               s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1282                                                   rx_fib_index0,
1283                                                   thread_index, now,
1284                                                   vm, node);
1285               if (!s0)
1286                 next0 = NAT_NEXT_DROP;
1287
1288               other_packets++;
1289               goto trace0;
1290             }
1291
1292           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1293             {
1294               next0 = icmp_in2out_ed_slow_path
1295                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1296                  node, next0, now, thread_index, &s0);
1297               icmp_packets++;
1298               goto trace0;
1299             }
1300
1301           // move down
1302           make_ed_kv (&ip0->src_address, &ip0->dst_address,
1303                       ip0->protocol, rx_fib_index0,
1304                       vnet_buffer (b0)->ip.reass.l4_src_port,
1305                       vnet_buffer (b0)->ip.reass.l4_dst_port, ~0ULL, &kv0);
1306
1307           if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1308             {
1309               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1310
1311               if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1312                 {
1313                   nat_free_session_data (sm, s0, thread_index, 0);
1314                   nat_ed_session_delete (sm, s0, thread_index, 1);
1315                   s0 = NULL;
1316                 }
1317             }
1318
1319           if (!s0)
1320             {
1321               if (is_output_feature)
1322                 {
1323                   if (PREDICT_FALSE
1324                       (nat44_ed_not_translate_output_feature
1325                        (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1326                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1327                         sw_if_index0,
1328                         vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1329                     goto trace0;
1330
1331                   /*
1332                    * Send DHCP packets to the ipv4 stack, or we won't
1333                    * be able to use dhcp client on the outside interface
1334                    */
1335                   if (PREDICT_FALSE
1336                       (proto0 == NAT_PROTOCOL_UDP
1337                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1338                            clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1339                        && ip0->dst_address.as_u32 == 0xffffffff))
1340                     goto trace0;
1341                 }
1342               else
1343                 {
1344                   if (PREDICT_FALSE
1345                       (nat44_ed_not_translate
1346                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1347                         thread_index)))
1348                     goto trace0;
1349                 }
1350
1351               next0 =
1352                 slow_path_ed (sm, b0, ip0->src_address, ip0->dst_address,
1353                               vnet_buffer (b0)->ip.reass.l4_src_port,
1354                               vnet_buffer (b0)->ip.reass.l4_dst_port,
1355                               ip0->protocol, rx_fib_index0, &s0, node, next0,
1356                               thread_index, now);
1357
1358               if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1359                 goto trace0;
1360
1361               if (PREDICT_FALSE (!s0))
1362                 goto trace0;
1363
1364             }
1365
1366           b0->flags |= VNET_BUFFER_F_IS_NATED;
1367
1368           if (!is_output_feature)
1369             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1370
1371           old_addr0 = ip0->src_address.as_u32;
1372           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1373           sum0 = ip0->checksum;
1374           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1375                                  src_address);
1376           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1377             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1378                                    s0->ext_host_addr.as_u32, ip4_header_t,
1379                                    dst_address);
1380           ip0->checksum = ip_csum_fold (sum0);
1381
1382           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1383
1384           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1385             {
1386               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1387                 {
1388                   new_port0 = udp0->src_port = s0->out2in.port;
1389                   sum0 = tcp0->checksum;
1390                   sum0 =
1391                     ip_csum_update (sum0, old_addr0, new_addr0,
1392                                     ip4_header_t, dst_address);
1393                   sum0 =
1394                     ip_csum_update (sum0, old_port0, new_port0,
1395                                     ip4_header_t, length);
1396                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1397                     {
1398                       sum0 =
1399                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1400                                         s0->ext_host_addr.as_u32,
1401                                         ip4_header_t, dst_address);
1402                       sum0 =
1403                         ip_csum_update (sum0,
1404                                         vnet_buffer (b0)->ip.
1405                                         reass.l4_dst_port, s0->ext_host_port,
1406                                         ip4_header_t, length);
1407                       tcp0->dst_port = s0->ext_host_port;
1408                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1409                     }
1410                   mss_clamping (sm, tcp0, &sum0);
1411                   tcp0->checksum = ip_csum_fold (sum0);
1412                 }
1413               tcp_packets++;
1414               if (nat44_set_tcp_session_state_i2o
1415                   (sm, now, s0, b0, thread_index))
1416                 goto trace0;
1417             }
1418           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1419                    && udp0->checksum)
1420             {
1421               new_port0 = udp0->src_port = s0->out2in.port;
1422               sum0 = udp0->checksum;
1423               sum0 =
1424                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1425                                 dst_address);
1426               sum0 =
1427                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1428                                 length);
1429               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1430                 {
1431                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1432                                          s0->ext_host_addr.as_u32,
1433                                          ip4_header_t, dst_address);
1434                   sum0 =
1435                     ip_csum_update (sum0,
1436                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1437                                     s0->ext_host_port, ip4_header_t, length);
1438                   udp0->dst_port = s0->ext_host_port;
1439                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1440                 }
1441               udp0->checksum = ip_csum_fold (sum0);
1442               udp_packets++;
1443             }
1444           else
1445             {
1446               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1447                 {
1448                   new_port0 = udp0->src_port = s0->out2in.port;
1449                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1450                     {
1451                       udp0->dst_port = s0->ext_host_port;
1452                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1453                     }
1454                   udp_packets++;
1455                 }
1456             }
1457
1458           /* Accounting */
1459           nat44_session_update_counters (s0, now,
1460                                          vlib_buffer_length_in_chain
1461                                          (vm, b0), thread_index);
1462           /* Per-user LRU list maintenance */
1463           nat44_session_update_lru (sm, s0, thread_index);
1464
1465         trace0:
1466           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1467                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1468             {
1469               nat_in2out_ed_trace_t *t =
1470                 vlib_add_trace (vm, node, b0, sizeof (*t));
1471               t->sw_if_index = sw_if_index0;
1472               t->next_index = next0;
1473               t->is_slow_path = 1;
1474
1475               if (s0)
1476                 t->session_index = s0 - tsm->sessions;
1477               else
1478                 t->session_index = ~0;
1479             }
1480
1481           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1482
1483           /* verify speculative enqueue, maybe switch current next frame */
1484           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1485                                            to_next, n_left_to_next,
1486                                            bi0, next0);
1487         }
1488
1489       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1490     }
1491
1492   vlib_node_increment_counter (vm, stats_node_index,
1493                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1494                                pkts_processed);
1495   vlib_node_increment_counter (vm, stats_node_index,
1496                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1497   vlib_node_increment_counter (vm, stats_node_index,
1498                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1499   vlib_node_increment_counter (vm, stats_node_index,
1500                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1501                                icmp_packets);
1502   vlib_node_increment_counter (vm, stats_node_index,
1503                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1504                                other_packets);
1505   return frame->n_vectors;
1506 }
1507
1508 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1509                                      vlib_node_runtime_t * node,
1510                                      vlib_frame_t * frame)
1511 {
1512   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1513 }
1514
1515 /* *INDENT-OFF* */
1516 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1517   .name = "nat44-ed-in2out",
1518   .vector_size = sizeof (u32),
1519   .sibling_of = "nat-default",
1520   .format_trace = format_nat_in2out_ed_trace,
1521   .type = VLIB_NODE_TYPE_INTERNAL,
1522   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1523   .error_strings = nat_in2out_ed_error_strings,
1524   .runtime_data_bytes = sizeof (snat_runtime_t),
1525 };
1526 /* *INDENT-ON* */
1527
1528 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1529                                             vlib_node_runtime_t * node,
1530                                             vlib_frame_t * frame)
1531 {
1532   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1533 }
1534
1535 /* *INDENT-OFF* */
1536 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1537   .name = "nat44-ed-in2out-output",
1538   .vector_size = sizeof (u32),
1539   .sibling_of = "nat-default",
1540   .format_trace = format_nat_in2out_ed_trace,
1541   .type = VLIB_NODE_TYPE_INTERNAL,
1542   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1543   .error_strings = nat_in2out_ed_error_strings,
1544   .runtime_data_bytes = sizeof (snat_runtime_t),
1545 };
1546 /* *INDENT-ON* */
1547
1548 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1549                                               vlib_node_runtime_t *
1550                                               node, vlib_frame_t * frame)
1551 {
1552   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1553 }
1554
1555 /* *INDENT-OFF* */
1556 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1557   .name = "nat44-ed-in2out-slowpath",
1558   .vector_size = sizeof (u32),
1559   .sibling_of = "nat-default",
1560   .format_trace = format_nat_in2out_ed_trace,
1561   .type = VLIB_NODE_TYPE_INTERNAL,
1562   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1563   .error_strings = nat_in2out_ed_error_strings,
1564   .runtime_data_bytes = sizeof (snat_runtime_t),
1565 };
1566 /* *INDENT-ON* */
1567
1568 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1569                                                      vlib_node_runtime_t
1570                                                      * node,
1571                                                      vlib_frame_t * frame)
1572 {
1573   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1574 }
1575
1576 /* *INDENT-OFF* */
1577 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1578   .name = "nat44-ed-in2out-output-slowpath",
1579   .vector_size = sizeof (u32),
1580   .sibling_of = "nat-default",
1581   .format_trace = format_nat_in2out_ed_trace,
1582   .type = VLIB_NODE_TYPE_INTERNAL,
1583   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1584   .error_strings = nat_in2out_ed_error_strings,
1585   .runtime_data_bytes = sizeof (snat_runtime_t),
1586 };
1587 /* *INDENT-ON* */
1588
1589 static u8 *
1590 format_nat_pre_trace (u8 * s, va_list * args)
1591 {
1592   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1593   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1594   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1595   return format (s, "in2out next_index %d arc_next_index %d", t->next_index,
1596                  t->arc_next_index);
1597 }
1598
1599 VLIB_NODE_FN (nat_pre_in2out_node)
1600   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1601 {
1602   return nat_pre_node_fn_inline (vm, node, frame,
1603                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1604 }
1605
1606 /* *INDENT-OFF* */
1607 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1608   .name = "nat-pre-in2out",
1609   .vector_size = sizeof (u32),
1610   .sibling_of = "nat-default",
1611   .format_trace = format_nat_pre_trace,
1612   .type = VLIB_NODE_TYPE_INTERNAL,
1613   .n_errors = 0,
1614 };
1615 /* *INDENT-ON* */
1616
1617 /*
1618  * fd.io coding-style-patch-verification: ON
1619  *
1620  * Local Variables:
1621  * eval: (c-set-style "gnu")
1622  * End:
1623  */