src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <nat/nat44/ed_inlines.h>
  35
  36 static char *nat_in2out_ed_error_strings[] = {
  37 #define _(sym,string) string,
  38   foreach_nat_in2out_ed_error
  39 #undef _
  40 };
  41
  42 typedef struct
  43 {
  44   u32 sw_if_index;
  45   u32 next_index;
  46   u32 session_index;
  47   u32 is_slow_path;
  48 } nat_in2out_ed_trace_t;
  49
  50 static u8 *
  51 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  56   char *tag;
  57
  58   tag =
  59     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  60     "NAT44_IN2OUT_ED_FAST_PATH";
  61
  62   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  63               t->sw_if_index, t->next_index, t->session_index);
  64
  65   return s;
  66 }
  67
  68 #ifndef CLIB_MARCH_VARIANT
  69 int
  70 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  71 {
  72   snat_main_t *sm = &snat_main;
  73   nat44_is_idle_session_ctx_t *ctx = arg;
  74   snat_session_t *s;
  75   u64 sess_timeout_time;
  76   u8 proto;
  77   u16 r_port, l_port;
  78   ip4_address_t *l_addr, *r_addr;
  79   u32 fib_index;
  80   clib_bihash_kv_16_8_t ed_kv;
  81   int i;
  82   snat_address_t *a;
  83   snat_session_key_t key;
  84   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  85                                                        ctx->thread_index);
  86
  87   s = pool_elt_at_index (tsm->sessions, kv->value);
  88   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  89   if (ctx->now >= sess_timeout_time)
  90     {
  91       if (is_fwd_bypass_session (s))
  92         goto delete;
  93
  94       l_addr = &s->out2in.addr;
  95       r_addr = &s->ext_host_addr;
  96       fib_index = s->out2in.fib_index;
  97       if (snat_is_unk_proto_session (s))
  98         {
  99           proto = s->in2out.port;
 100           r_port = 0;
 101           l_port = 0;
 102         }
 103       else
 104         {
 105           proto = nat_proto_to_ip_proto (s->in2out.protocol);
 106           l_port = s->out2in.port;
 107           r_port = s->ext_host_port;
 108         }
 109       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 110                   &ed_kv);
 111       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 112         nat_elog_warn ("out2in_ed key del failed");
 113
 114       if (snat_is_unk_proto_session (s))
 115         goto delete;
 116
 117       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 118                                            s->in2out.addr.as_u32,
 119                                            s->out2in.addr.as_u32,
 120                                            s->in2out.protocol,
 121                                            s->in2out.port,
 122                                            s->out2in.port,
 123                                            s->in2out.fib_index);
 124
 125       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 126                              &s->in2out.addr, s->in2out.port,
 127                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 128                              &s->out2in.addr, s->out2in.port,
 129                              &s->ext_host_addr, s->ext_host_port,
 130                              s->in2out.protocol, is_twice_nat_session (s));
 131
 132       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 133                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 134                    ctx->thread_index);
 135
 136       if (is_twice_nat_session (s))
 137         {
 138           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 139             {
 140               key.protocol = s->in2out.protocol;
 141               key.port = s->ext_host_nat_port;
 142               a = sm->twice_nat_addresses + i;
 143               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 144                 {
 145                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 146                                                       ctx->thread_index,
 147                                                       &key);
 148                   break;
 149                 }
 150             }
 151         }
 152
 153       if (snat_is_session_static (s))
 154         goto delete;
 155
 156       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 157                                           &s->out2in);
 158     delete:
 159       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 160       return 1;
 161     }
 162
 163   return 0;
 164 }
 165 #endif
 166
 167 static inline u32
 168 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 169                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 170                           u32 sw_if_index0, u32 rx_fib_index0,
 171                           vlib_node_runtime_t * node, u32 next0, f64 now,
 172                           u32 thread_index, snat_session_t ** p_s0)
 173 {
 174   vlib_main_t *vm = vlib_get_main ();
 175
 176   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 177                        next0, thread_index, p_s0, 0);
 178   snat_session_t *s0 = *p_s0;
 179   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 180     {
 181       /* Accounting */
 182       nat44_session_update_counters (s0, now,
 183                                      vlib_buffer_length_in_chain
 184                                      (vm, b0), thread_index);
 185       /* Per-user LRU list maintenance */
 186       nat44_session_update_lru (sm, s0, thread_index);
 187     }
 188   return next0;
 189 }
 190
 191 static_always_inline u16
 192 snat_random_port (u16 min, u16 max)
 193 {
 194   snat_main_t *sm = &snat_main;
 195   return min + random_u32 (&sm->random_seed) /
 196     (random_u32_max () / (max - min + 1) + 1);
 197 }
 198
 199 static int
 200 nat_ed_alloc_addr_and_port (snat_main_t * sm, u32 rx_fib_index,
 201                             u32 nat_proto, u32 thread_index,
 202                             ip4_address_t r_addr, u16 r_port, u8 proto,
 203                             u16 port_per_thread, u32 snat_thread_index,
 204                             snat_session_t * s,
 205                             ip4_address_t * allocated_addr,
 206                             u16 * allocated_port,
 207                             clib_bihash_kv_16_8_t * out2in_ed_kv)
 208 {
 209   int i;
 210   snat_address_t *a, *ga = 0;
 211   u32 portnum;
 212   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 213
 214   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 215
 216   for (i = 0; i < vec_len (sm->addresses); i++)
 217     {
 218       a = sm->addresses + i;
 219       switch (nat_proto)
 220         {
 221 #define _(N, j, n, unused)                                                    \
 222   case NAT_PROTOCOL_##N:                                                     \
 223     if (a->fib_index == rx_fib_index)                                         \
 224       {                                                                       \
 225         u16 port = snat_random_port (1, port_per_thread);                     \
 226         u16 attempts = port_per_thread;                                       \
 227         while (attempts > 0)                                                  \
 228           {                                                                   \
 229             --attempts;                                                       \
 230             portnum = port_thread_offset + port;                              \
 231             make_ed_kv (&a->addr, &r_addr, proto, s->out2in.fib_index,        \
 232                         clib_host_to_net_u16 (portnum), r_port,               \
 233                         s - tsm->sessions, out2in_ed_kv);                     \
 234             int rv = clib_bihash_add_del_16_8 (&tsm->out2in_ed, out2in_ed_kv, \
 235                                                2 /* is_add */);               \
 236             if (0 == rv)                                                      \
 237               {                                                               \
 238                 ++a->busy_##n##_port_refcounts[portnum];                      \
 239                 a->busy_##n##_ports_per_thread[thread_index]++;               \
 240                 a->busy_##n##_ports++;                                        \
 241                 *allocated_addr = a->addr;                                    \
 242                 *allocated_port = clib_host_to_net_u16 (portnum);             \
 243                 return 0;                                                     \
 244               }                                                               \
 245             port = (port + 1) % port_per_thread;                              \
 246           }                                                                   \
 247       }                                                                       \
 248     else if (a->fib_index == ~0)                                              \
 249       {                                                                       \
 250         ga = a;                                                               \
 251       }                                                                       \
 252     break;
 253
 254           foreach_nat_protocol;
 255         default:
 256           nat_elog_info ("unknown protocol");
 257           return 1;
 258         }
 259     }
 260
 261   if (ga)
 262     {
 263       /* fake fib_index to reuse macro */
 264       rx_fib_index = ~0;
 265       a = ga;
 266       switch (nat_proto)
 267         {
 268           foreach_nat_protocol;
 269         default:
 270           nat_elog_info ("unknown protocol");
 271           return 1;
 272         }
 273     }
 274
 275 #undef _
 276
 277   /* Totally out of translations to use... */
 278   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 279   return 1;
 280 }
 281
 282 static u32
 283 slow_path_ed (snat_main_t * sm,
 284               vlib_buffer_t * b,
 285               ip4_address_t l_addr,
 286               ip4_address_t r_addr,
 287               u16 l_port,
 288               u16 r_port,
 289               u8 proto,
 290               u32 rx_fib_index,
 291               snat_session_t ** sessionp,
 292               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 293 {
 294   snat_session_t *s = NULL;
 295   snat_session_key_t key0, key1;
 296   lb_nat_type_t lb = 0;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   u32 nat_proto = ip_proto_to_nat_proto (proto);
 299   nat_outside_fib_t *outside_fib;
 300   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 301   clib_bihash_kv_16_8_t out2in_ed_kv;
 302   ip4_address_t allocated_addr;
 303   u16 allocated_port;
 304   u8 identity_nat;
 305   fib_prefix_t pfx = {
 306     .fp_proto = FIB_PROTOCOL_IP4,
 307     .fp_len = 32,
 308     .fp_addr = {.ip4.as_u32 = r_addr.as_u32,},
 309   };
 310   nat44_is_idle_session_ctx_t ctx;
 311
 312   if (PREDICT_TRUE (nat_proto == NAT_PROTOCOL_TCP))
 313     {
 314       if (PREDICT_FALSE
 315           (!tcp_flags_is_init
 316            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 317         {
 318           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 319           return NAT_NEXT_DROP;
 320         }
 321     }
 322
 323   // TODO: based on fib index do a lookup
 324   if (PREDICT_FALSE
 325       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 326     {
 327       if (!nat_lru_free_one (sm, thread_index, now))
 328         {
 329           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 330           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 331           nat_elog_notice ("maximum sessions exceeded");
 332           return NAT_NEXT_DROP;
 333         }
 334     }
 335
 336   key0.addr = l_addr;
 337   key0.port = l_port;
 338   key1.protocol = key0.protocol = nat_proto;
 339   key0.fib_index = rx_fib_index;
 340   key1.fib_index = sm->outside_fib_index;
 341
 342   /* First try to match static mapping by local address and port */
 343   if (snat_static_mapping_match
 344       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 345     {
 346       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 347       if (!s)
 348         {
 349           nat_elog_warn ("create NAT session failed");
 350           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 351           return NAT_NEXT_DROP;
 352         }
 353       switch (vec_len (sm->outside_fibs))
 354         {
 355         case 0:
 356           s->out2in.fib_index = sm->outside_fib_index;
 357           break;
 358         case 1:
 359           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 360           break;
 361         default:
 362           /* *INDENT-OFF* */
 363           vec_foreach (outside_fib, sm->outside_fibs)
 364           {
 365             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 366             if (FIB_NODE_INDEX_INVALID != fei)
 367               {
 368                 if (fib_entry_get_resolving_interface (fei) != ~0)
 369                   {
 370                     s->out2in.fib_index = outside_fib->fib_index;
 371                     break;
 372                   }
 373               }
 374           }
 375           /* *INDENT-ON* */
 376           break;
 377         }
 378
 379       /* Try to create dynamic translation */
 380       if (nat_ed_alloc_addr_and_port (sm, rx_fib_index, nat_proto,
 381                                       thread_index, r_addr, r_port, proto,
 382                                       sm->port_per_thread,
 383                                       tsm->snat_thread_index, s,
 384                                       &allocated_addr,
 385                                       &allocated_port, &out2in_ed_kv))
 386         {
 387           nat_elog_notice ("addresses exhausted");
 388           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 389           nat_ed_session_delete (sm, s, thread_index, 1);
 390           return NAT_NEXT_DROP;
 391         }
 392       key1.addr = allocated_addr;
 393       key1.port = allocated_port;
 394     }
 395   else
 396     {
 397       if (PREDICT_FALSE (identity_nat))
 398         {
 399           *sessionp = s;
 400           return next;
 401         }
 402       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 403       if (!s)
 404         {
 405           nat_elog_warn ("create NAT session failed");
 406           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 407           return NAT_NEXT_DROP;
 408         }
 409       switch (vec_len (sm->outside_fibs))
 410         {
 411         case 0:
 412           s->out2in.fib_index = sm->outside_fib_index;
 413           break;
 414         case 1:
 415           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 416           break;
 417         default:
 418           /* *INDENT-OFF* */
 419           vec_foreach (outside_fib, sm->outside_fibs)
 420           {
 421             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 422             if (FIB_NODE_INDEX_INVALID != fei)
 423               {
 424                 if (fib_entry_get_resolving_interface (fei) != ~0)
 425                   {
 426                     s->out2in.fib_index = outside_fib->fib_index;
 427                     break;
 428                   }
 429               }
 430           }
 431           /* *INDENT-ON* */
 432           break;
 433         }
 434
 435       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 436
 437
 438       make_ed_kv (&key1.addr, &r_addr, proto,
 439                   s->out2in.fib_index, key1.port, r_port, s - tsm->sessions,
 440                   &out2in_ed_kv);
 441       if (clib_bihash_add_or_overwrite_stale_16_8
 442           (&tsm->out2in_ed, &out2in_ed_kv, nat44_o2i_ed_is_idle_session_cb,
 443            &ctx))
 444         nat_elog_notice ("out2in-ed key add failed");
 445     }
 446
 447   if (lb)
 448     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 449   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 450   s->ext_host_addr = r_addr;
 451   s->ext_host_port = r_port;
 452   s->in2out = key0;
 453   s->out2in = key1;
 454   s->out2in.protocol = key0.protocol;
 455
 456   clib_bihash_kv_16_8_t in2out_ed_kv;
 457   make_ed_kv (&l_addr, &r_addr, proto, rx_fib_index, l_port, r_port,
 458               s - tsm->sessions, &in2out_ed_kv);
 459   ctx.now = now;
 460   ctx.thread_index = thread_index;
 461   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &in2out_ed_kv,
 462                                                nat44_i2o_ed_is_idle_session_cb,
 463                                                &ctx))
 464     nat_elog_notice ("in2out-ed key add failed");
 465
 466   *sessionp = s;
 467
 468   /* log NAT event */
 469   snat_ipfix_logging_nat44_ses_create (thread_index,
 470                                        s->in2out.addr.as_u32,
 471                                        s->out2in.addr.as_u32,
 472                                        s->in2out.protocol,
 473                                        s->in2out.port,
 474                                        s->out2in.port, s->in2out.fib_index);
 475
 476   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 477                          &s->in2out.addr, s->in2out.port,
 478                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 479                          &s->out2in.addr, s->out2in.port,
 480                          &s->ext_host_addr, s->ext_host_port,
 481                          s->in2out.protocol, 0);
 482
 483   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 484                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 485                &s->ext_host_nat_addr, s->ext_host_nat_port,
 486                s->in2out.protocol, s->in2out.fib_index, s->flags,
 487                thread_index, 0);
 488
 489   return next;
 490 }
 491
 492 static_always_inline int
 493 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 494                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 495                         u32 rx_fib_index, u32 thread_index)
 496 {
 497   udp_header_t *udp = ip4_next_header (ip);
 498   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 499   clib_bihash_kv_16_8_t kv, value;
 500   snat_session_key_t key0, key1;
 501
 502   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 503               sm->outside_fib_index, udp->dst_port, udp->src_port, ~0ULL,
 504               &kv);
 505
 506   /* NAT packet aimed at external address if has active sessions */
 507   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 508     {
 509       key0.addr = ip->dst_address;
 510       key0.port = udp->dst_port;
 511       key0.protocol = proto;
 512       key0.fib_index = sm->outside_fib_index;
 513       /* or is static mappings */
 514       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 515         return 0;
 516     }
 517   else
 518     return 0;
 519
 520   if (sm->forwarding_enabled)
 521     return 1;
 522
 523   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 524                                   rx_fib_index);
 525 }
 526
 527 static_always_inline int
 528 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 529                                       u32 thread_index, f64 now,
 530                                       vlib_main_t * vm, vlib_buffer_t * b)
 531 {
 532   clib_bihash_kv_16_8_t kv, value;
 533   snat_session_t *s = 0;
 534   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 535
 536   if (!sm->forwarding_enabled)
 537     return 0;
 538
 539   if (ip->protocol == IP_PROTOCOL_ICMP)
 540     {
 541       if (get_icmp_i2o_ed_key (b, ip, 0, ~0ULL, 0, 0, 0, &kv))
 542         return 0;
 543     }
 544   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 545     {
 546       make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol, 0,
 547                   vnet_buffer (b)->ip.reass.l4_src_port,
 548                   vnet_buffer (b)->ip.reass.l4_dst_port, ~0ULL, &kv);
 549     }
 550   else
 551     {
 552       make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 553                   0, ~0ULL, &kv);
 554     }
 555
 556   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 557     {
 558       s = pool_elt_at_index (tsm->sessions, value.value);
 559       if (is_fwd_bypass_session (s))
 560         {
 561           if (ip->protocol == IP_PROTOCOL_TCP)
 562             {
 563               if (nat44_set_tcp_session_state_i2o
 564                   (sm, now, s, b, thread_index))
 565                 return 1;
 566             }
 567           /* Accounting */
 568           nat44_session_update_counters (s, now,
 569                                          vlib_buffer_length_in_chain (vm, b),
 570                                          thread_index);
 571           /* Per-user LRU list maintenance */
 572           nat44_session_update_lru (sm, s, thread_index);
 573           return 1;
 574         }
 575       else
 576         return 0;
 577     }
 578
 579   return 0;
 580 }
 581
 582 static_always_inline int
 583 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 584                                        u16 src_port, u16 dst_port,
 585                                        u32 thread_index, u32 rx_sw_if_index,
 586                                        u32 tx_sw_if_index)
 587 {
 588   clib_bihash_kv_16_8_t kv, value;
 589   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 590   snat_interface_t *i;
 591   snat_session_t *s;
 592   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 593   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 594
 595   /* src NAT check */
 596   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 597               tx_fib_index, src_port, dst_port, ~0ULL, &kv);
 598   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 599     {
 600       s = pool_elt_at_index (tsm->sessions, value.value);
 601       if (nat44_is_ses_closed (s))
 602         {
 603           nat_free_session_data (sm, s, thread_index, 0);
 604           nat_ed_session_delete (sm, s, thread_index, 1);
 605         }
 606       else
 607         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 608       return 1;
 609     }
 610
 611   /* dst NAT check */
 612   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 613               rx_fib_index, dst_port, src_port, ~0ULL, &kv);
 614   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 615     {
 616       s = pool_elt_at_index (tsm->sessions, value.value);
 617       if (is_fwd_bypass_session (s))
 618         return 0;
 619
 620       /* hairpinning */
 621       /* *INDENT-OFF* */
 622       pool_foreach (i, sm->output_feature_interfaces,
 623       ({
 624         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 625            return 0;
 626       }));
 627       /* *INDENT-ON* */
 628       return 1;
 629     }
 630
 631   return 0;
 632 }
 633
 634 #ifndef CLIB_MARCH_VARIANT
 635 u32
 636 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 637                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 638                       u8 * p_proto, snat_session_key_t * p_value,
 639                       u8 * p_dont_translate, void *d, void *e)
 640 {
 641   u32 sw_if_index;
 642   u32 rx_fib_index;
 643   snat_session_t *s = 0;
 644   u8 dont_translate = 0;
 645   clib_bihash_kv_16_8_t kv, value;
 646   u32 next = ~0;
 647   int err;
 648   u16 l_port = 0, r_port = 0;   // initialize to workaround gcc warning
 649   vlib_main_t *vm = vlib_get_main ();
 650   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 651
 652   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 653   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 654
 655   err =
 656     get_icmp_i2o_ed_key (b, ip, rx_fib_index, ~0ULL, p_proto, &l_port,
 657                          &r_port, &kv);
 658   if (err != 0)
 659     {
 660       b->error = node->errors[err];
 661       next = NAT_NEXT_DROP;
 662       goto out;
 663     }
 664
 665   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 666     {
 667       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 668         {
 669           if (PREDICT_FALSE
 670               (nat44_ed_not_translate_output_feature
 671                (sm, ip, l_port, r_port, thread_index,
 672                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX])))
 673             {
 674               dont_translate = 1;
 675               goto out;
 676             }
 677         }
 678       else
 679         {
 680           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 681                                                      ip, NAT_PROTOCOL_ICMP,
 682                                                      rx_fib_index,
 683                                                      thread_index)))
 684             {
 685               dont_translate = 1;
 686               goto out;
 687             }
 688         }
 689
 690       if (PREDICT_FALSE
 691           (icmp_type_is_error_message
 692            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 693         {
 694           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 695           next = NAT_NEXT_DROP;
 696           goto out;
 697         }
 698
 699       next =
 700         slow_path_ed (sm, b, ip->src_address, ip->dst_address, l_port, r_port,
 701                       ip->protocol, rx_fib_index, &s, node, next,
 702                       thread_index, vlib_time_now (vm));
 703
 704       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 705         goto out;
 706
 707       if (!s)
 708         {
 709           dont_translate = 1;
 710           goto out;
 711         }
 712     }
 713   else
 714     {
 715       if (PREDICT_FALSE
 716           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 717            ICMP4_echo_request
 718            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 719            ICMP4_echo_reply
 720            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 721                                            reass.icmp_type_or_tcp_flags)))
 722         {
 723           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 724           next = NAT_NEXT_DROP;
 725           goto out;
 726         }
 727
 728       s = pool_elt_at_index (tsm->sessions, value.value);
 729     }
 730 out:
 731   if (s)
 732     *p_value = s->out2in;
 733   *p_dont_translate = dont_translate;
 734   if (d)
 735     *(snat_session_t **) d = s;
 736   return next;
 737 }
 738 #endif
 739
 740 static snat_session_t *
 741 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 742                                vlib_buffer_t * b,
 743                                ip4_header_t * ip,
 744                                u32 rx_fib_index,
 745                                u32 thread_index,
 746                                f64 now,
 747                                vlib_main_t * vm, vlib_node_runtime_t * node)
 748 {
 749   clib_bihash_kv_8_8_t kv, value;
 750   clib_bihash_kv_16_8_t s_kv, s_value;
 751   snat_static_mapping_t *m;
 752   u32 old_addr, new_addr = 0;
 753   ip_csum_t sum;
 754   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 755   snat_session_t *s;
 756   u32 outside_fib_index = sm->outside_fib_index;
 757   int i;
 758   u8 is_sm = 0;
 759   nat_outside_fib_t *outside_fib;
 760   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 761   fib_prefix_t pfx = {
 762     .fp_proto = FIB_PROTOCOL_IP4,
 763     .fp_len = 32,
 764     .fp_addr = {
 765                 .ip4.as_u32 = ip->dst_address.as_u32,
 766                 },
 767   };
 768
 769   switch (vec_len (sm->outside_fibs))
 770     {
 771     case 0:
 772       outside_fib_index = sm->outside_fib_index;
 773       break;
 774     case 1:
 775       outside_fib_index = sm->outside_fibs[0].fib_index;
 776       break;
 777     default:
 778       /* *INDENT-OFF* */
 779       vec_foreach (outside_fib, sm->outside_fibs)
 780         {
 781           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 782           if (FIB_NODE_INDEX_INVALID != fei)
 783             {
 784               if (fib_entry_get_resolving_interface (fei) != ~0)
 785                 {
 786                   outside_fib_index = outside_fib->fib_index;
 787                   break;
 788                 }
 789             }
 790         }
 791       /* *INDENT-ON* */
 792       break;
 793     }
 794   old_addr = ip->src_address.as_u32;
 795
 796   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 797               rx_fib_index, 0, 0, ~0ULL, &s_kv);
 798
 799   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 800     {
 801       s = pool_elt_at_index (tsm->sessions, s_value.value);
 802       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 803     }
 804   else
 805     {
 806       if (PREDICT_FALSE
 807           (nat44_ed_maximum_sessions_exceeded
 808            (sm, rx_fib_index, thread_index)))
 809         {
 810           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 811           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 812           nat_elog_notice ("maximum sessions exceeded");
 813           return 0;
 814         }
 815
 816       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 817
 818       /* Try to find static mapping first */
 819       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 820         {
 821           m = pool_elt_at_index (sm->static_mappings, value.value);
 822           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 823           is_sm = 1;
 824           goto create_ses;
 825         }
 826       else
 827         {
 828           /* *INDENT-OFF* */
 829           pool_foreach (s, tsm->sessions, {
 830             if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 831               {
 832                 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 833
 834                 make_ed_kv (&s->out2in.addr, &ip->dst_address, ip->protocol,
 835                             outside_fib_index, 0, 0, ~0ULL, &s_kv);
 836                 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 837                   goto create_ses;
 838
 839                 break;
 840               }
 841           });
 842           /* *INDENT-ON* */
 843
 844           for (i = 0; i < vec_len (sm->addresses); i++)
 845             {
 846               make_ed_kv (&sm->addresses[i].addr, &ip->dst_address,
 847                           ip->protocol, outside_fib_index, 0, 0, ~0ULL,
 848                           &s_kv);
 849               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 850                 {
 851                   new_addr = ip->src_address.as_u32 =
 852                     sm->addresses[i].addr.as_u32;
 853                   goto create_ses;
 854                 }
 855             }
 856           return 0;
 857         }
 858
 859     create_ses:
 860       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 861       if (!s)
 862         {
 863           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 864           nat_elog_warn ("create NAT session failed");
 865           return 0;
 866         }
 867
 868       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 869       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 870       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 871       s->out2in.addr.as_u32 = new_addr;
 872       s->out2in.fib_index = outside_fib_index;
 873       s->in2out.addr.as_u32 = old_addr;
 874       s->in2out.fib_index = rx_fib_index;
 875       s->in2out.port = s->out2in.port = ip->protocol;
 876       if (is_sm)
 877         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 878
 879       /* Add to lookup tables */
 880       make_ed_kv (&s->in2out.addr, &ip->dst_address, ip->protocol,
 881                   rx_fib_index, 0, 0, s - tsm->sessions, &s_kv);
 882       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 883         nat_elog_notice ("in2out key add failed");
 884
 885       make_ed_kv (&s->out2in.addr, &ip->dst_address, ip->protocol,
 886                   outside_fib_index, 0, 0, s - tsm->sessions, &s_kv);
 887       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 888         nat_elog_notice ("out2in key add failed");
 889     }
 890
 891   /* Update IP checksum */
 892   sum = ip->checksum;
 893   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 894   ip->checksum = ip_csum_fold (sum);
 895
 896   /* Accounting */
 897   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 898                                  thread_index);
 899   /* Per-user LRU list maintenance */
 900   nat44_session_update_lru (sm, s, thread_index);
 901
 902   /* Hairpinning */
 903   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 904     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 905
 906   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 907     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 908
 909   return s;
 910 }
 911
 912 static inline uword
 913 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 914                                           vlib_node_runtime_t * node,
 915                                           vlib_frame_t * frame,
 916                                           int is_output_feature)
 917 {
 918   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 919   nat_next_t next_index;
 920   snat_main_t *sm = &snat_main;
 921   f64 now = vlib_time_now (vm);
 922   u32 thread_index = vm->thread_index;
 923   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 924   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 925     0, def_slow;
 926
 927   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 928     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 929
 930   stats_node_index = sm->ed_in2out_node_index;
 931
 932   from = vlib_frame_vector_args (frame);
 933   n_left_from = frame->n_vectors;
 934   next_index = node->cached_next_index;
 935
 936   while (n_left_from > 0)
 937     {
 938       u32 n_left_to_next;
 939
 940       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 941
 942       while (n_left_from > 0 && n_left_to_next > 0)
 943         {
 944           u32 bi0;
 945           vlib_buffer_t *b0;
 946           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 947             new_addr0, old_addr0;
 948           u16 old_port0, new_port0;
 949           ip4_header_t *ip0;
 950           udp_header_t *udp0;
 951           tcp_header_t *tcp0;
 952           snat_session_t *s0 = 0;
 953           clib_bihash_kv_16_8_t kv0, value0;
 954           ip_csum_t sum0;
 955
 956           /* speculatively enqueue b0 to the current next frame */
 957           bi0 = from[0];
 958           to_next[0] = bi0;
 959           from += 1;
 960           to_next += 1;
 961           n_left_from -= 1;
 962           n_left_to_next -= 1;
 963
 964           b0 = vlib_get_buffer (vm, bi0);
 965
 966           if (is_output_feature)
 967             {
 968               vnet_feature_next (&vnet_buffer2 (b0)->nat.arc_next, b0);
 969               iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 970             }
 971
 972           next0 = vnet_buffer2 (b0)->nat.arc_next;
 973
 974           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 975                                   iph_offset0);
 976
 977           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 978           rx_fib_index0 =
 979             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 980                                                  sw_if_index0);
 981
 982           if (PREDICT_FALSE (ip0->ttl == 1))
 983             {
 984               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 985               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 986                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 987                                            0);
 988               next0 = NAT_NEXT_ICMP_ERROR;
 989               goto trace0;
 990             }
 991
 992           udp0 = ip4_next_header (ip0);
 993           tcp0 = (tcp_header_t *) udp0;
 994           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 995
 996           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 997             {
 998               next0 = def_slow;
 999               goto trace0;
1000             }
1001
1002           if (is_output_feature)
1003             {
1004               if (PREDICT_FALSE (nat_not_translate_output_feature_fwd
1005                                  (sm, ip0, thread_index, now, vm, b0)))
1006                 goto trace0;
1007             }
1008
1009           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1010             {
1011               next0 = def_slow;
1012               goto trace0;
1013             }
1014
1015           make_ed_kv (&ip0->src_address, &ip0->dst_address,
1016                       ip0->protocol, rx_fib_index0,
1017                       vnet_buffer (b0)->ip.reass.l4_src_port,
1018                       vnet_buffer (b0)->ip.reass.l4_dst_port, ~0ULL, &kv0);
1019
1020           // lookup for session
1021           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1022             {
1023               // session does not exist go slow path
1024               next0 = def_slow;
1025               goto trace0;
1026             }
1027           s0 = pool_elt_at_index (tsm->sessions, value0.value);
1028
1029           if (s0->tcp_closed_timestamp)
1030             {
1031               if (now >= s0->tcp_closed_timestamp)
1032                 {
1033                   // session is closed, go slow path
1034                   next0 = def_slow;
1035                 }
1036               else
1037                 {
1038                   // session in transitory timeout, drop
1039                   b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1040                   next0 = NAT_NEXT_DROP;
1041                 }
1042               goto trace0;
1043             }
1044
1045           // drop if session expired
1046           u64 sess_timeout_time;
1047           sess_timeout_time = s0->last_heard +
1048             (f64) nat44_session_get_timeout (sm, s0);
1049           if (now >= sess_timeout_time)
1050             {
1051               nat_free_session_data (sm, s0, thread_index, 0);
1052               nat_ed_session_delete (sm, s0, thread_index, 1);
1053               // session is closed, go slow path
1054               next0 = def_slow;
1055               goto trace0;
1056             }
1057
1058           b0->flags |= VNET_BUFFER_F_IS_NATED;
1059
1060           if (!is_output_feature)
1061             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1062
1063           old_addr0 = ip0->src_address.as_u32;
1064           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1065           sum0 = ip0->checksum;
1066           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1067                                  src_address);
1068           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1069             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1070                                    s0->ext_host_addr.as_u32, ip4_header_t,
1071                                    dst_address);
1072           ip0->checksum = ip_csum_fold (sum0);
1073
1074           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1075
1076           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1077             {
1078               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1079                 {
1080                   new_port0 = udp0->src_port = s0->out2in.port;
1081                   sum0 = tcp0->checksum;
1082                   sum0 =
1083                     ip_csum_update (sum0, old_addr0, new_addr0,
1084                                     ip4_header_t, dst_address);
1085                   sum0 =
1086                     ip_csum_update (sum0, old_port0, new_port0,
1087                                     ip4_header_t, length);
1088                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1089                     {
1090                       sum0 =
1091                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1092                                         s0->ext_host_addr.as_u32,
1093                                         ip4_header_t, dst_address);
1094                       sum0 =
1095                         ip_csum_update (sum0,
1096                                         vnet_buffer (b0)->ip.
1097                                         reass.l4_dst_port, s0->ext_host_port,
1098                                         ip4_header_t, length);
1099                       tcp0->dst_port = s0->ext_host_port;
1100                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1101                     }
1102                   mss_clamping (sm, tcp0, &sum0);
1103                   tcp0->checksum = ip_csum_fold (sum0);
1104                 }
1105               tcp_packets++;
1106               if (nat44_set_tcp_session_state_i2o
1107                   (sm, now, s0, b0, thread_index))
1108                 goto trace0;
1109             }
1110           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1111                    && udp0->checksum)
1112             {
1113               new_port0 = udp0->src_port = s0->out2in.port;
1114               sum0 = udp0->checksum;
1115               sum0 =
1116                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1117                                 dst_address);
1118               sum0 =
1119                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1120                                 length);
1121               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1122                 {
1123                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1124                                          s0->ext_host_addr.as_u32,
1125                                          ip4_header_t, dst_address);
1126                   sum0 =
1127                     ip_csum_update (sum0,
1128                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1129                                     s0->ext_host_port, ip4_header_t, length);
1130                   udp0->dst_port = s0->ext_host_port;
1131                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1132                 }
1133               udp0->checksum = ip_csum_fold (sum0);
1134               udp_packets++;
1135             }
1136           else
1137             {
1138               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1139                 {
1140                   new_port0 = udp0->src_port = s0->out2in.port;
1141                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1142                     {
1143                       udp0->dst_port = s0->ext_host_port;
1144                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1145                     }
1146                   udp_packets++;
1147                 }
1148             }
1149
1150           /* Accounting */
1151           nat44_session_update_counters (s0, now,
1152                                          vlib_buffer_length_in_chain
1153                                          (vm, b0), thread_index);
1154           /* Per-user LRU list maintenance */
1155           nat44_session_update_lru (sm, s0, thread_index);
1156
1157         trace0:
1158           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1159                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1160             {
1161               nat_in2out_ed_trace_t *t =
1162                 vlib_add_trace (vm, node, b0, sizeof (*t));
1163               t->sw_if_index = sw_if_index0;
1164               t->next_index = next0;
1165               t->is_slow_path = 0;
1166
1167               if (s0)
1168                 t->session_index = s0 - tsm->sessions;
1169               else
1170                 t->session_index = ~0;
1171             }
1172
1173           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1174           /* verify speculative enqueue, maybe switch current next frame */
1175           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1176                                            to_next, n_left_to_next,
1177                                            bi0, next0);
1178         }
1179
1180       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1181     }
1182
1183   vlib_node_increment_counter (vm, stats_node_index,
1184                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1185                                pkts_processed);
1186   vlib_node_increment_counter (vm, stats_node_index,
1187                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1188   vlib_node_increment_counter (vm, stats_node_index,
1189                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1190   vlib_node_increment_counter (vm, stats_node_index,
1191                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1192                                icmp_packets);
1193   vlib_node_increment_counter (vm, stats_node_index,
1194                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1195                                other_packets);
1196   return frame->n_vectors;
1197 }
1198
1199 static inline uword
1200 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1201                                           vlib_node_runtime_t * node,
1202                                           vlib_frame_t * frame,
1203                                           int is_output_feature)
1204 {
1205   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
1206   nat_next_t next_index;
1207   snat_main_t *sm = &snat_main;
1208   f64 now = vlib_time_now (vm);
1209   u32 thread_index = vm->thread_index;
1210   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1211   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = 0;
1212
1213   stats_node_index = sm->ed_in2out_slowpath_node_index;
1214
1215   from = vlib_frame_vector_args (frame);
1216   n_left_from = frame->n_vectors;
1217   next_index = node->cached_next_index;
1218
1219   while (n_left_from > 0)
1220     {
1221       u32 n_left_to_next;
1222
1223       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1224
1225       while (n_left_from > 0 && n_left_to_next > 0)
1226         {
1227           u32 bi0;
1228           vlib_buffer_t *b0;
1229           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1230             new_addr0, old_addr0;
1231           u16 old_port0, new_port0;
1232           ip4_header_t *ip0;
1233           udp_header_t *udp0;
1234           tcp_header_t *tcp0;
1235           icmp46_header_t *icmp0;
1236           snat_session_t *s0 = 0;
1237           clib_bihash_kv_16_8_t kv0, value0;
1238           ip_csum_t sum0;
1239
1240           /* speculatively enqueue b0 to the current next frame */
1241           bi0 = from[0];
1242           to_next[0] = bi0;
1243           from += 1;
1244           to_next += 1;
1245           n_left_from -= 1;
1246           n_left_to_next -= 1;
1247
1248           b0 = vlib_get_buffer (vm, bi0);
1249
1250           if (is_output_feature)
1251             iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1252
1253           next0 = vnet_buffer2 (b0)->nat.arc_next;
1254
1255           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1256                                   iph_offset0);
1257
1258           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1259           rx_fib_index0 =
1260             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1261                                                  sw_if_index0);
1262
1263           if (PREDICT_FALSE (ip0->ttl == 1))
1264             {
1265               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1266               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1267                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1268                                            0);
1269               next0 = NAT_NEXT_ICMP_ERROR;
1270               goto trace0;
1271             }
1272
1273           udp0 = ip4_next_header (ip0);
1274           tcp0 = (tcp_header_t *) udp0;
1275           icmp0 = (icmp46_header_t *) udp0;
1276           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1277
1278           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1279             {
1280               s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1281                                                   rx_fib_index0,
1282                                                   thread_index, now,
1283                                                   vm, node);
1284               if (!s0)
1285                 next0 = NAT_NEXT_DROP;
1286
1287               other_packets++;
1288               goto trace0;
1289             }
1290
1291           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1292             {
1293               next0 = icmp_in2out_ed_slow_path
1294                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1295                  node, next0, now, thread_index, &s0);
1296               icmp_packets++;
1297               goto trace0;
1298             }
1299
1300           // move down
1301           make_ed_kv (&ip0->src_address, &ip0->dst_address,
1302                       ip0->protocol, rx_fib_index0,
1303                       vnet_buffer (b0)->ip.reass.l4_src_port,
1304                       vnet_buffer (b0)->ip.reass.l4_dst_port, ~0ULL, &kv0);
1305
1306           if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1307             {
1308               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1309
1310               if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1311                 {
1312                   nat_free_session_data (sm, s0, thread_index, 0);
1313                   nat_ed_session_delete (sm, s0, thread_index, 1);
1314                   s0 = NULL;
1315                 }
1316             }
1317
1318           if (!s0)
1319             {
1320               if (is_output_feature)
1321                 {
1322                   if (PREDICT_FALSE
1323                       (nat44_ed_not_translate_output_feature
1324                        (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1325                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1326                         sw_if_index0,
1327                         vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1328                     goto trace0;
1329
1330                   /*
1331                    * Send DHCP packets to the ipv4 stack, or we won't
1332                    * be able to use dhcp client on the outside interface
1333                    */
1334                   if (PREDICT_FALSE
1335                       (proto0 == NAT_PROTOCOL_UDP
1336                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1337                            clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1338                        && ip0->dst_address.as_u32 == 0xffffffff))
1339                     goto trace0;
1340                 }
1341               else
1342                 {
1343                   if (PREDICT_FALSE
1344                       (nat44_ed_not_translate
1345                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1346                         thread_index)))
1347                     goto trace0;
1348                 }
1349
1350               next0 =
1351                 slow_path_ed (sm, b0, ip0->src_address, ip0->dst_address,
1352                               vnet_buffer (b0)->ip.reass.l4_src_port,
1353                               vnet_buffer (b0)->ip.reass.l4_dst_port,
1354                               ip0->protocol, rx_fib_index0, &s0, node, next0,
1355                               thread_index, now);
1356
1357               if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1358                 goto trace0;
1359
1360               if (PREDICT_FALSE (!s0))
1361                 goto trace0;
1362
1363             }
1364
1365           b0->flags |= VNET_BUFFER_F_IS_NATED;
1366
1367           if (!is_output_feature)
1368             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1369
1370           old_addr0 = ip0->src_address.as_u32;
1371           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1372           sum0 = ip0->checksum;
1373           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1374                                  src_address);
1375           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1376             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1377                                    s0->ext_host_addr.as_u32, ip4_header_t,
1378                                    dst_address);
1379           ip0->checksum = ip_csum_fold (sum0);
1380
1381           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1382
1383           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1384             {
1385               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1386                 {
1387                   new_port0 = udp0->src_port = s0->out2in.port;
1388                   sum0 = tcp0->checksum;
1389                   sum0 =
1390                     ip_csum_update (sum0, old_addr0, new_addr0,
1391                                     ip4_header_t, dst_address);
1392                   sum0 =
1393                     ip_csum_update (sum0, old_port0, new_port0,
1394                                     ip4_header_t, length);
1395                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1396                     {
1397                       sum0 =
1398                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1399                                         s0->ext_host_addr.as_u32,
1400                                         ip4_header_t, dst_address);
1401                       sum0 =
1402                         ip_csum_update (sum0,
1403                                         vnet_buffer (b0)->ip.
1404                                         reass.l4_dst_port, s0->ext_host_port,
1405                                         ip4_header_t, length);
1406                       tcp0->dst_port = s0->ext_host_port;
1407                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1408                     }
1409                   mss_clamping (sm, tcp0, &sum0);
1410                   tcp0->checksum = ip_csum_fold (sum0);
1411                 }
1412               tcp_packets++;
1413               if (nat44_set_tcp_session_state_i2o
1414                   (sm, now, s0, b0, thread_index))
1415                 goto trace0;
1416             }
1417           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1418                    && udp0->checksum)
1419             {
1420               new_port0 = udp0->src_port = s0->out2in.port;
1421               sum0 = udp0->checksum;
1422               sum0 =
1423                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1424                                 dst_address);
1425               sum0 =
1426                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1427                                 length);
1428               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1429                 {
1430                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1431                                          s0->ext_host_addr.as_u32,
1432                                          ip4_header_t, dst_address);
1433                   sum0 =
1434                     ip_csum_update (sum0,
1435                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1436                                     s0->ext_host_port, ip4_header_t, length);
1437                   udp0->dst_port = s0->ext_host_port;
1438                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1439                 }
1440               udp0->checksum = ip_csum_fold (sum0);
1441               udp_packets++;
1442             }
1443           else
1444             {
1445               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1446                 {
1447                   new_port0 = udp0->src_port = s0->out2in.port;
1448                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1449                     {
1450                       udp0->dst_port = s0->ext_host_port;
1451                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1452                     }
1453                   udp_packets++;
1454                 }
1455             }
1456
1457           /* Accounting */
1458           nat44_session_update_counters (s0, now,
1459                                          vlib_buffer_length_in_chain
1460                                          (vm, b0), thread_index);
1461           /* Per-user LRU list maintenance */
1462           nat44_session_update_lru (sm, s0, thread_index);
1463
1464         trace0:
1465           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1466                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1467             {
1468               nat_in2out_ed_trace_t *t =
1469                 vlib_add_trace (vm, node, b0, sizeof (*t));
1470               t->sw_if_index = sw_if_index0;
1471               t->next_index = next0;
1472               t->is_slow_path = 1;
1473
1474               if (s0)
1475                 t->session_index = s0 - tsm->sessions;
1476               else
1477                 t->session_index = ~0;
1478             }
1479
1480           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1481
1482           /* verify speculative enqueue, maybe switch current next frame */
1483           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1484                                            to_next, n_left_to_next,
1485                                            bi0, next0);
1486         }
1487
1488       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1489     }
1490
1491   vlib_node_increment_counter (vm, stats_node_index,
1492                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1493                                pkts_processed);
1494   vlib_node_increment_counter (vm, stats_node_index,
1495                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1496   vlib_node_increment_counter (vm, stats_node_index,
1497                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1498   vlib_node_increment_counter (vm, stats_node_index,
1499                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1500                                icmp_packets);
1501   vlib_node_increment_counter (vm, stats_node_index,
1502                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1503                                other_packets);
1504   return frame->n_vectors;
1505 }
1506
1507 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1508                                      vlib_node_runtime_t * node,
1509                                      vlib_frame_t * frame)
1510 {
1511   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1512 }
1513
1514 /* *INDENT-OFF* */
1515 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1516   .name = "nat44-ed-in2out",
1517   .vector_size = sizeof (u32),
1518   .sibling_of = "nat-default",
1519   .format_trace = format_nat_in2out_ed_trace,
1520   .type = VLIB_NODE_TYPE_INTERNAL,
1521   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1522   .error_strings = nat_in2out_ed_error_strings,
1523   .runtime_data_bytes = sizeof (snat_runtime_t),
1524 };
1525 /* *INDENT-ON* */
1526
1527 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1528                                             vlib_node_runtime_t * node,
1529                                             vlib_frame_t * frame)
1530 {
1531   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1532 }
1533
1534 /* *INDENT-OFF* */
1535 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1536   .name = "nat44-ed-in2out-output",
1537   .vector_size = sizeof (u32),
1538   .sibling_of = "nat-default",
1539   .format_trace = format_nat_in2out_ed_trace,
1540   .type = VLIB_NODE_TYPE_INTERNAL,
1541   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1542   .error_strings = nat_in2out_ed_error_strings,
1543   .runtime_data_bytes = sizeof (snat_runtime_t),
1544 };
1545 /* *INDENT-ON* */
1546
1547 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1548                                               vlib_node_runtime_t *
1549                                               node, vlib_frame_t * frame)
1550 {
1551   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1552 }
1553
1554 /* *INDENT-OFF* */
1555 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1556   .name = "nat44-ed-in2out-slowpath",
1557   .vector_size = sizeof (u32),
1558   .sibling_of = "nat-default",
1559   .format_trace = format_nat_in2out_ed_trace,
1560   .type = VLIB_NODE_TYPE_INTERNAL,
1561   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1562   .error_strings = nat_in2out_ed_error_strings,
1563   .runtime_data_bytes = sizeof (snat_runtime_t),
1564 };
1565 /* *INDENT-ON* */
1566
1567 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1568                                                      vlib_node_runtime_t
1569                                                      * node,
1570                                                      vlib_frame_t * frame)
1571 {
1572   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1573 }
1574
1575 /* *INDENT-OFF* */
1576 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1577   .name = "nat44-ed-in2out-output-slowpath",
1578   .vector_size = sizeof (u32),
1579   .sibling_of = "nat-default",
1580   .format_trace = format_nat_in2out_ed_trace,
1581   .type = VLIB_NODE_TYPE_INTERNAL,
1582   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1583   .error_strings = nat_in2out_ed_error_strings,
1584   .runtime_data_bytes = sizeof (snat_runtime_t),
1585 };
1586 /* *INDENT-ON* */
1587
1588 static u8 *
1589 format_nat_pre_trace (u8 * s, va_list * args)
1590 {
1591   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1592   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1593   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1594   return format (s, "in2out next_index %d arc_next_index %d", t->next_index,
1595                  t->arc_next_index);
1596 }
1597
1598 VLIB_NODE_FN (nat_pre_in2out_node)
1599   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1600 {
1601   return nat_pre_node_fn_inline (vm, node, frame,
1602                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1603 }
1604
1605 /* *INDENT-OFF* */
1606 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1607   .name = "nat-pre-in2out",
1608   .vector_size = sizeof (u32),
1609   .sibling_of = "nat-default",
1610   .format_trace = format_nat_pre_trace,
1611   .type = VLIB_NODE_TYPE_INTERNAL,
1612   .n_errors = 0,
1613 };
1614 /* *INDENT-ON* */
1615
1616 /*
1617  * fd.io coding-style-patch-verification: ON
1618  *
1619  * Local Variables:
1620  * eval: (c-set-style "gnu")
1621  * End:
1622  */