src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 static char *nat_in2out_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_in2out_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat_in2out_ed_trace_t;
  48
  49 static u8 *
  50 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  59     "NAT44_IN2OUT_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 #ifndef CLIB_MARCH_VARIANT
  68 int
  69 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  70 {
  71   snat_main_t *sm = &snat_main;
  72   nat44_is_idle_session_ctx_t *ctx = arg;
  73   snat_session_t *s;
  74   u64 sess_timeout_time;
  75   u8 proto;
  76   u16 r_port, l_port;
  77   ip4_address_t *l_addr, *r_addr;
  78   u32 fib_index;
  79   clib_bihash_kv_16_8_t ed_kv;
  80   int i;
  81   snat_address_t *a;
  82   snat_session_key_t key;
  83   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  84                                                        ctx->thread_index);
  85
  86   s = pool_elt_at_index (tsm->sessions, kv->value);
  87   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  88   if (ctx->now >= sess_timeout_time)
  89     {
  90       if (is_fwd_bypass_session (s))
  91         goto delete;
  92
  93       l_addr = &s->out2in.addr;
  94       r_addr = &s->ext_host_addr;
  95       fib_index = s->out2in.fib_index;
  96       if (snat_is_unk_proto_session (s))
  97         {
  98           proto = s->in2out.port;
  99           r_port = 0;
 100           l_port = 0;
 101         }
 102       else
 103         {
 104           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 105           l_port = s->out2in.port;
 106           r_port = s->ext_host_port;
 107         }
 108       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 109                   &ed_kv);
 110       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 111         nat_elog_warn ("out2in_ed key del failed");
 112
 113       if (snat_is_unk_proto_session (s))
 114         goto delete;
 115
 116       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 117                                            s->in2out.addr.as_u32,
 118                                            s->out2in.addr.as_u32,
 119                                            s->in2out.protocol,
 120                                            s->in2out.port,
 121                                            s->out2in.port,
 122                                            s->in2out.fib_index);
 123
 124       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 125                              &s->in2out.addr, s->in2out.port,
 126                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 127                              &s->out2in.addr, s->out2in.port,
 128                              &s->ext_host_addr, s->ext_host_port,
 129                              s->in2out.protocol, is_twice_nat_session (s));
 130
 131       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 132                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 133                    ctx->thread_index);
 134
 135       if (is_twice_nat_session (s))
 136         {
 137           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 138             {
 139               key.protocol = s->in2out.protocol;
 140               key.port = s->ext_host_nat_port;
 141               a = sm->twice_nat_addresses + i;
 142               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 143                 {
 144                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 145                                                       ctx->thread_index,
 146                                                       &key);
 147                   break;
 148                 }
 149             }
 150         }
 151
 152       if (snat_is_session_static (s))
 153         goto delete;
 154
 155       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 156                                           &s->out2in);
 157     delete:
 158       nat44_ed_delete_session (sm, s, ctx->thread_index, 1);
 159       return 1;
 160     }
 161
 162   return 0;
 163 }
 164 #endif
 165
 166 static inline u32
 167 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 168                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 169                           u32 sw_if_index0, u32 rx_fib_index0,
 170                           vlib_node_runtime_t * node, u32 next0, f64 now,
 171                           u32 thread_index, snat_session_t ** p_s0)
 172 {
 173   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 174                        next0, thread_index, p_s0, 0);
 175   snat_session_t *s0 = *p_s0;
 176   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 177     {
 178       /* Accounting */
 179       nat44_session_update_counters (s0, now,
 180                                      vlib_buffer_length_in_chain
 181                                      (sm->vlib_main, b0), thread_index);
 182       /* Per-user LRU list maintenance */
 183       nat44_session_update_lru (sm, s0, thread_index);
 184     }
 185   return next0;
 186 }
 187
 188 static_always_inline u16
 189 snat_random_port (u16 min, u16 max)
 190 {
 191   snat_main_t *sm = &snat_main;
 192   return min + random_u32 (&sm->random_seed) /
 193     (random_u32_max () / (max - min + 1) + 1);
 194 }
 195
 196 static int
 197 nat_ed_alloc_addr_and_port (snat_main_t * sm, u32 rx_fib_index,
 198                             u32 snat_proto, u32 thread_index,
 199                             ip4_address_t r_addr, u16 r_port, u8 proto,
 200                             u16 port_per_thread, u32 snat_thread_index,
 201                             snat_session_t * s,
 202                             ip4_address_t * allocated_addr,
 203                             u16 * allocated_port,
 204                             clib_bihash_kv_16_8_t * out2in_ed_kv)
 205 {
 206   int i;
 207   snat_address_t *a, *ga = 0;
 208   u32 portnum;
 209   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 210
 211   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 212
 213   for (i = 0; i < vec_len (sm->addresses); i++)
 214     {
 215       a = sm->addresses + i;
 216       switch (snat_proto)
 217         {
 218 #define _(N, j, n, unused)                                                    \
 219   case SNAT_PROTOCOL_##N:                                                     \
 220     if (a->fib_index == rx_fib_index)                                         \
 221       {                                                                       \
 222         u16 port = snat_random_port (1, port_per_thread);                     \
 223         u16 attempts = port_per_thread;                                       \
 224         while (attempts > 0)                                                  \
 225           {                                                                   \
 226             --attempts;                                                       \
 227             portnum = port_thread_offset + port;                              \
 228             make_ed_kv (&a->addr, &r_addr, proto, s->out2in.fib_index,        \
 229                         clib_host_to_net_u16 (portnum), r_port,               \
 230                         s - tsm->sessions, out2in_ed_kv);                     \
 231             int rv = clib_bihash_add_del_16_8 (&tsm->out2in_ed, out2in_ed_kv, \
 232                                                2 /* is_add */);               \
 233             if (0 == rv)                                                      \
 234               {                                                               \
 235                 ++a->busy_##n##_port_refcounts[portnum];                      \
 236                 a->busy_##n##_ports_per_thread[thread_index]++;               \
 237                 a->busy_##n##_ports++;                                        \
 238                 *allocated_addr = a->addr;                                    \
 239                 *allocated_port = clib_host_to_net_u16 (portnum);             \
 240                 return 0;                                                     \
 241               }                                                               \
 242             port = (port + 1) % port_per_thread;                              \
 243           }                                                                   \
 244       }                                                                       \
 245     else if (a->fib_index == ~0)                                              \
 246       {                                                                       \
 247         ga = a;                                                               \
 248       }                                                                       \
 249     break;
 250
 251           foreach_snat_protocol;
 252         default:
 253           nat_elog_info ("unknown protocol");
 254           return 1;
 255         }
 256     }
 257
 258   if (ga)
 259     {
 260       /* fake fib_index to reuse macro */
 261       rx_fib_index = ~0;
 262       a = ga;
 263       switch (snat_proto)
 264         {
 265           foreach_snat_protocol;
 266         default:
 267           nat_elog_info ("unknown protocol");
 268           return 1;
 269         }
 270     }
 271
 272 #undef _
 273
 274   /* Totally out of translations to use... */
 275   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 276   return 1;
 277 }
 278
 279 static u32
 280 slow_path_ed (snat_main_t * sm,
 281               vlib_buffer_t * b,
 282               ip4_address_t l_addr,
 283               ip4_address_t r_addr,
 284               u16 l_port,
 285               u16 r_port,
 286               u8 proto,
 287               u32 rx_fib_index,
 288               snat_session_t ** sessionp,
 289               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 290 {
 291   snat_session_t *s = NULL;
 292   snat_session_key_t key0, key1;
 293   lb_nat_type_t lb = 0;
 294   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 295   u32 snat_proto = ip_proto_to_snat_proto (proto);
 296   nat_outside_fib_t *outside_fib;
 297   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 298   clib_bihash_kv_16_8_t out2in_ed_kv;
 299   bool out2in_ed_inserted = false;
 300   ip4_address_t allocated_addr;
 301   u16 allocated_port;
 302   u8 identity_nat;
 303   fib_prefix_t pfx = {
 304     .fp_proto = FIB_PROTOCOL_IP4,
 305     .fp_len = 32,
 306     .fp_addr = {.ip4.as_u32 = r_addr.as_u32,},
 307   };
 308   nat44_is_idle_session_ctx_t ctx;
 309
 310   if (PREDICT_TRUE (snat_proto == SNAT_PROTOCOL_TCP))
 311     {
 312       if (PREDICT_FALSE
 313           (!tcp_flags_is_init
 314            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 315         {
 316           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 317           return NAT_NEXT_DROP;
 318         }
 319     }
 320
 321   if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 322     {
 323       if (!nat_global_lru_free_one (sm, thread_index, now))
 324         {
 325           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 326           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 327           nat_elog_notice ("maximum sessions exceeded");
 328           return NAT_NEXT_DROP;
 329         }
 330     }
 331
 332   key0.addr = l_addr;
 333   key0.port = l_port;
 334   key1.protocol = key0.protocol = snat_proto;
 335   key0.fib_index = rx_fib_index;
 336   key1.fib_index = sm->outside_fib_index;
 337
 338   /* First try to match static mapping by local address and port */
 339   if (snat_static_mapping_match
 340       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 341     {
 342       s = nat_ed_session_alloc (sm, thread_index, now);
 343       if (!s)
 344         {
 345           nat_elog_warn ("create NAT session failed");
 346           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 347           goto drop;
 348         }
 349       switch (vec_len (sm->outside_fibs))
 350         {
 351         case 0:
 352           s->out2in.fib_index = sm->outside_fib_index;
 353           break;
 354         case 1:
 355           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 356           break;
 357         default:
 358           /* *INDENT-OFF* */
 359           vec_foreach (outside_fib, sm->outside_fibs)
 360           {
 361             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 362             if (FIB_NODE_INDEX_INVALID != fei)
 363               {
 364                 if (fib_entry_get_resolving_interface (fei) != ~0)
 365                   {
 366                     s->out2in.fib_index = outside_fib->fib_index;
 367                     break;
 368                   }
 369               }
 370           }
 371           /* *INDENT-ON* */
 372           break;
 373         }
 374
 375       /* Try to create dynamic translation */
 376       if (nat_ed_alloc_addr_and_port (sm, rx_fib_index, snat_proto,
 377                                       thread_index, r_addr, r_port, proto,
 378                                       sm->port_per_thread,
 379                                       tsm->snat_thread_index, s,
 380                                       &allocated_addr,
 381                                       &allocated_port, &out2in_ed_kv))
 382         {
 383           nat_elog_notice ("addresses exhausted");
 384           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 385           goto drop;
 386         }
 387
 388       out2in_ed_inserted = true;
 389       key1.addr = allocated_addr;
 390       key1.port = allocated_port;
 391     }
 392   else
 393     {
 394       if (PREDICT_FALSE (identity_nat))
 395         {
 396           *sessionp = s;
 397           return next;
 398         }
 399       s = nat_ed_session_alloc (sm, thread_index, now);
 400       if (!s)
 401         {
 402           nat_elog_warn ("create NAT session failed");
 403           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 404           goto drop;
 405         }
 406       switch (vec_len (sm->outside_fibs))
 407         {
 408         case 0:
 409           s->out2in.fib_index = sm->outside_fib_index;
 410           break;
 411         case 1:
 412           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 413           break;
 414         default:
 415           /* *INDENT-OFF* */
 416           vec_foreach (outside_fib, sm->outside_fibs)
 417           {
 418             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 419             if (FIB_NODE_INDEX_INVALID != fei)
 420               {
 421                 if (fib_entry_get_resolving_interface (fei) != ~0)
 422                   {
 423                     s->out2in.fib_index = outside_fib->fib_index;
 424                     break;
 425                   }
 426               }
 427           }
 428           /* *INDENT-ON* */
 429           break;
 430         }
 431
 432       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 433
 434
 435       make_ed_kv (&key1.addr, &r_addr, proto,
 436                   s->out2in.fib_index, key1.port, r_port, s - tsm->sessions,
 437                   &out2in_ed_kv);
 438       if (clib_bihash_add_or_overwrite_stale_16_8
 439           (&tsm->out2in_ed, &out2in_ed_kv, nat44_o2i_ed_is_idle_session_cb,
 440            &ctx))
 441         nat_elog_notice ("out2in-ed key add failed");
 442       out2in_ed_inserted = true;
 443     }
 444
 445   if (lb)
 446     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 447   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 448   s->ext_host_addr = r_addr;
 449   s->ext_host_port = r_port;
 450   s->in2out = key0;
 451   s->out2in = key1;
 452   s->out2in.protocol = key0.protocol;
 453
 454   clib_bihash_kv_16_8_t in2out_ed_kv;
 455   make_ed_kv (&l_addr, &r_addr, proto, rx_fib_index, l_port, r_port,
 456               s - tsm->sessions, &in2out_ed_kv);
 457   ctx.now = now;
 458   ctx.thread_index = thread_index;
 459   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &in2out_ed_kv,
 460                                                nat44_i2o_ed_is_idle_session_cb,
 461                                                &ctx))
 462     nat_elog_notice ("in2out-ed key add failed");
 463
 464   *sessionp = s;
 465
 466   /* log NAT event */
 467   snat_ipfix_logging_nat44_ses_create (thread_index,
 468                                        s->in2out.addr.as_u32,
 469                                        s->out2in.addr.as_u32,
 470                                        s->in2out.protocol,
 471                                        s->in2out.port,
 472                                        s->out2in.port, s->in2out.fib_index);
 473
 474   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 475                          &s->in2out.addr, s->in2out.port,
 476                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 477                          &s->out2in.addr, s->out2in.port,
 478                          &s->ext_host_addr, s->ext_host_port,
 479                          s->in2out.protocol, 0);
 480
 481   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 482                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 483                &s->ext_host_nat_addr, s->ext_host_nat_port,
 484                s->in2out.protocol, s->in2out.fib_index, s->flags,
 485                thread_index, 0);
 486
 487   return next;
 488 drop:
 489   if (out2in_ed_inserted)
 490     {
 491       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &out2in_ed_kv, 0))
 492         nat_elog_notice ("out2in-ed key del failed");
 493     }
 494   if (s)
 495     {
 496       nat_free_session_data (sm, s, thread_index, 0);
 497       nat44_ed_delete_session (sm, s, thread_index, 1);
 498     }
 499   return NAT_NEXT_DROP;
 500 }
 501
 502 static_always_inline int
 503 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 504                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 505                         u32 rx_fib_index, u32 thread_index)
 506 {
 507   udp_header_t *udp = ip4_next_header (ip);
 508   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 509   clib_bihash_kv_16_8_t kv, value;
 510   snat_session_key_t key0, key1;
 511
 512   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 513               sm->outside_fib_index, udp->dst_port, udp->src_port, ~0ULL,
 514               &kv);
 515
 516   /* NAT packet aimed at external address if has active sessions */
 517   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 518     {
 519       key0.addr = ip->dst_address;
 520       key0.port = udp->dst_port;
 521       key0.protocol = proto;
 522       key0.fib_index = sm->outside_fib_index;
 523       /* or is static mappings */
 524       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 525         return 0;
 526     }
 527   else
 528     return 0;
 529
 530   if (sm->forwarding_enabled)
 531     return 1;
 532
 533   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 534                                   rx_fib_index);
 535 }
 536
 537 static_always_inline int
 538 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 539                                       u32 thread_index, f64 now,
 540                                       vlib_main_t * vm, vlib_buffer_t * b)
 541 {
 542   clib_bihash_kv_16_8_t kv, value;
 543   snat_session_t *s = 0;
 544   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 545
 546   if (!sm->forwarding_enabled)
 547     return 0;
 548
 549   if (ip->protocol == IP_PROTOCOL_ICMP)
 550     {
 551       if (get_icmp_i2o_ed_key (b, ip, 0, ~0ULL, 0, 0, 0, &kv))
 552         return 0;
 553     }
 554   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 555     {
 556       make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol, 0,
 557                   vnet_buffer (b)->ip.reass.l4_src_port,
 558                   vnet_buffer (b)->ip.reass.l4_dst_port, ~0ULL, &kv);
 559     }
 560   else
 561     {
 562       make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 563                   0, ~0ULL, &kv);
 564     }
 565
 566   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 567     {
 568       s = pool_elt_at_index (tsm->sessions, value.value);
 569       if (is_fwd_bypass_session (s))
 570         {
 571           if (ip->protocol == IP_PROTOCOL_TCP)
 572             {
 573               if (nat44_set_tcp_session_state_i2o
 574                   (sm, now, s, b, thread_index))
 575                 return 1;
 576             }
 577           /* Accounting */
 578           nat44_session_update_counters (s, now,
 579                                          vlib_buffer_length_in_chain (vm, b),
 580                                          thread_index);
 581           /* Per-user LRU list maintenance */
 582           nat44_session_update_lru (sm, s, thread_index);
 583           return 1;
 584         }
 585       else
 586         return 0;
 587     }
 588
 589   return 0;
 590 }
 591
 592 static_always_inline int
 593 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 594                                        u16 src_port, u16 dst_port,
 595                                        u32 thread_index, u32 rx_sw_if_index,
 596                                        u32 tx_sw_if_index)
 597 {
 598   clib_bihash_kv_16_8_t kv, value;
 599   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 600   snat_interface_t *i;
 601   snat_session_t *s;
 602   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 603   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 604
 605   /* src NAT check */
 606   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 607               tx_fib_index, src_port, dst_port, ~0ULL, &kv);
 608   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 609     {
 610       s = pool_elt_at_index (tsm->sessions, value.value);
 611       if (nat44_is_ses_closed (s))
 612         {
 613           nat_free_session_data (sm, s, thread_index, 0);
 614           nat44_ed_delete_session (sm, s, thread_index, 1);
 615         }
 616       else
 617         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 618       return 1;
 619     }
 620
 621   /* dst NAT check */
 622   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 623               rx_fib_index, dst_port, src_port, ~0ULL, &kv);
 624   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 625     {
 626       s = pool_elt_at_index (tsm->sessions, value.value);
 627       if (is_fwd_bypass_session (s))
 628         return 0;
 629
 630       /* hairpinning */
 631       /* *INDENT-OFF* */
 632       pool_foreach (i, sm->output_feature_interfaces,
 633       ({
 634         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 635            return 0;
 636       }));
 637       /* *INDENT-ON* */
 638       return 1;
 639     }
 640
 641   return 0;
 642 }
 643
 644 #ifndef CLIB_MARCH_VARIANT
 645 u32
 646 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 647                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 648                       u8 * p_proto, snat_session_key_t * p_value,
 649                       u8 * p_dont_translate, void *d, void *e)
 650 {
 651   u32 sw_if_index;
 652   u32 rx_fib_index;
 653   snat_session_t *s = 0;
 654   u8 dont_translate = 0;
 655   clib_bihash_kv_16_8_t kv, value;
 656   u32 next = ~0;
 657   int err;
 658   u16 l_port = 0, r_port = 0;   // initialize to workaround gcc warning
 659   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 660
 661   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 662   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 663
 664   err =
 665     get_icmp_i2o_ed_key (b, ip, rx_fib_index, ~0ULL, p_proto, &l_port,
 666                          &r_port, &kv);
 667   if (err != 0)
 668     {
 669       b->error = node->errors[err];
 670       next = NAT_NEXT_DROP;
 671       goto out;
 672     }
 673
 674   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 675     {
 676       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 677         {
 678           if (PREDICT_FALSE
 679               (nat44_ed_not_translate_output_feature
 680                (sm, ip, l_port, r_port, thread_index,
 681                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX])))
 682             {
 683               dont_translate = 1;
 684               goto out;
 685             }
 686         }
 687       else
 688         {
 689           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 690                                                      ip, SNAT_PROTOCOL_ICMP,
 691                                                      rx_fib_index,
 692                                                      thread_index)))
 693             {
 694               dont_translate = 1;
 695               goto out;
 696             }
 697         }
 698
 699       if (PREDICT_FALSE
 700           (icmp_type_is_error_message
 701            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 702         {
 703           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 704           next = NAT_NEXT_DROP;
 705           goto out;
 706         }
 707
 708       next =
 709         slow_path_ed (sm, b, ip->src_address, ip->dst_address, l_port, r_port,
 710                       ip->protocol, rx_fib_index, &s, node, next,
 711                       thread_index, vlib_time_now (sm->vlib_main));
 712
 713       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 714         goto out;
 715
 716       if (!s)
 717         {
 718           dont_translate = 1;
 719           goto out;
 720         }
 721     }
 722   else
 723     {
 724       if (PREDICT_FALSE
 725           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 726            ICMP4_echo_request
 727            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 728            ICMP4_echo_reply
 729            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 730                                            reass.icmp_type_or_tcp_flags)))
 731         {
 732           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 733           next = NAT_NEXT_DROP;
 734           goto out;
 735         }
 736
 737       s = pool_elt_at_index (tsm->sessions, value.value);
 738     }
 739 out:
 740   if (s)
 741     *p_value = s->out2in;
 742   *p_dont_translate = dont_translate;
 743   if (d)
 744     *(snat_session_t **) d = s;
 745   return next;
 746 }
 747 #endif
 748
 749 static snat_session_t *
 750 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 751                                vlib_buffer_t * b,
 752                                ip4_header_t * ip,
 753                                u32 rx_fib_index,
 754                                u32 thread_index,
 755                                f64 now,
 756                                vlib_main_t * vm, vlib_node_runtime_t * node)
 757 {
 758   clib_bihash_kv_8_8_t kv, value;
 759   clib_bihash_kv_16_8_t s_kv, s_value;
 760   snat_static_mapping_t *m;
 761   u32 old_addr, new_addr = 0;
 762   ip_csum_t sum;
 763   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 764   snat_session_t *s;
 765   u32 outside_fib_index = sm->outside_fib_index;
 766   int i;
 767   u8 is_sm = 0;
 768   nat_outside_fib_t *outside_fib;
 769   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 770   fib_prefix_t pfx = {
 771     .fp_proto = FIB_PROTOCOL_IP4,
 772     .fp_len = 32,
 773     .fp_addr = {
 774                 .ip4.as_u32 = ip->dst_address.as_u32,
 775                 },
 776   };
 777
 778   switch (vec_len (sm->outside_fibs))
 779     {
 780     case 0:
 781       outside_fib_index = sm->outside_fib_index;
 782       break;
 783     case 1:
 784       outside_fib_index = sm->outside_fibs[0].fib_index;
 785       break;
 786     default:
 787       /* *INDENT-OFF* */
 788       vec_foreach (outside_fib, sm->outside_fibs)
 789         {
 790           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 791           if (FIB_NODE_INDEX_INVALID != fei)
 792             {
 793               if (fib_entry_get_resolving_interface (fei) != ~0)
 794                 {
 795                   outside_fib_index = outside_fib->fib_index;
 796                   break;
 797                 }
 798             }
 799         }
 800       /* *INDENT-ON* */
 801       break;
 802     }
 803   old_addr = ip->src_address.as_u32;
 804
 805   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 806               rx_fib_index, 0, 0, ~0ULL, &s_kv);
 807
 808   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 809     {
 810       s = pool_elt_at_index (tsm->sessions, s_value.value);
 811       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 812     }
 813   else
 814     {
 815       if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 816         {
 817           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 818           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 819           nat_elog_notice ("maximum sessions exceeded");
 820           return 0;
 821         }
 822
 823       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 824
 825       /* Try to find static mapping first */
 826       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 827         {
 828           m = pool_elt_at_index (sm->static_mappings, value.value);
 829           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 830           is_sm = 1;
 831           goto create_ses;
 832         }
 833       else
 834         {
 835           /* *INDENT-OFF* */
 836           pool_foreach (s, tsm->sessions, {
 837             if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 838               {
 839                 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 840
 841                 make_ed_kv (&s->out2in.addr, &ip->dst_address, ip->protocol,
 842                             outside_fib_index, 0, 0, ~0ULL, &s_kv);
 843                 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 844                   goto create_ses;
 845
 846                 break;
 847               }
 848           });
 849           /* *INDENT-ON* */
 850
 851           for (i = 0; i < vec_len (sm->addresses); i++)
 852             {
 853               make_ed_kv (&sm->addresses[i].addr, &ip->dst_address,
 854                           ip->protocol, outside_fib_index, 0, 0, ~0ULL,
 855                           &s_kv);
 856               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 857                 {
 858                   new_addr = ip->src_address.as_u32 =
 859                     sm->addresses[i].addr.as_u32;
 860                   goto create_ses;
 861                 }
 862             }
 863           return 0;
 864         }
 865
 866     create_ses:
 867       s = nat_ed_session_alloc (sm, thread_index, now);
 868       if (!s)
 869         {
 870           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 871           nat_elog_warn ("create NAT session failed");
 872           return 0;
 873         }
 874
 875       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 876       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 877       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 878       s->out2in.addr.as_u32 = new_addr;
 879       s->out2in.fib_index = outside_fib_index;
 880       s->in2out.addr.as_u32 = old_addr;
 881       s->in2out.fib_index = rx_fib_index;
 882       s->in2out.port = s->out2in.port = ip->protocol;
 883       if (is_sm)
 884         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 885
 886       /* Add to lookup tables */
 887       make_ed_kv (&s->in2out.addr, &ip->dst_address, ip->protocol,
 888                   rx_fib_index, 0, 0, s - tsm->sessions, &s_kv);
 889       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 890         nat_elog_notice ("in2out key add failed");
 891
 892       make_ed_kv (&s->out2in.addr, &ip->dst_address, ip->protocol,
 893                   outside_fib_index, 0, 0, s - tsm->sessions, &s_kv);
 894       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 895         nat_elog_notice ("out2in key add failed");
 896     }
 897
 898   /* Update IP checksum */
 899   sum = ip->checksum;
 900   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 901   ip->checksum = ip_csum_fold (sum);
 902
 903   /* Accounting */
 904   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 905                                  thread_index);
 906   /* Per-user LRU list maintenance */
 907   nat44_session_update_lru (sm, s, thread_index);
 908
 909   /* Hairpinning */
 910   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 911     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 912
 913   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 914     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 915
 916   return s;
 917 }
 918
 919 static inline uword
 920 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 921                                           vlib_node_runtime_t * node,
 922                                           vlib_frame_t * frame,
 923                                           int is_output_feature)
 924 {
 925   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 926   nat_next_t next_index;
 927   snat_main_t *sm = &snat_main;
 928   f64 now = vlib_time_now (vm);
 929   u32 thread_index = vm->thread_index;
 930   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 931   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 932     0, def_slow;
 933
 934   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 935     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 936
 937   stats_node_index = sm->ed_in2out_node_index;
 938
 939   from = vlib_frame_vector_args (frame);
 940   n_left_from = frame->n_vectors;
 941   next_index = node->cached_next_index;
 942
 943   while (n_left_from > 0)
 944     {
 945       u32 n_left_to_next;
 946
 947       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 948
 949       while (n_left_from > 0 && n_left_to_next > 0)
 950         {
 951           u32 bi0;
 952           vlib_buffer_t *b0;
 953           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 954             new_addr0, old_addr0;
 955           u16 old_port0, new_port0;
 956           ip4_header_t *ip0;
 957           udp_header_t *udp0;
 958           tcp_header_t *tcp0;
 959           snat_session_t *s0 = 0;
 960           clib_bihash_kv_16_8_t kv0, value0;
 961           ip_csum_t sum0;
 962
 963           /* speculatively enqueue b0 to the current next frame */
 964           bi0 = from[0];
 965           to_next[0] = bi0;
 966           from += 1;
 967           to_next += 1;
 968           n_left_from -= 1;
 969           n_left_to_next -= 1;
 970
 971           b0 = vlib_get_buffer (vm, bi0);
 972
 973           if (is_output_feature)
 974             {
 975               vnet_feature_next (&vnet_buffer2 (b0)->nat.arc_next, b0);
 976               iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 977             }
 978
 979           next0 = vnet_buffer2 (b0)->nat.arc_next;
 980
 981           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 982                                   iph_offset0);
 983
 984           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 985           rx_fib_index0 =
 986             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 987                                                  sw_if_index0);
 988
 989           if (PREDICT_FALSE (ip0->ttl == 1))
 990             {
 991               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 992               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 993                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 994                                            0);
 995               next0 = NAT_NEXT_ICMP_ERROR;
 996               goto trace0;
 997             }
 998
 999           udp0 = ip4_next_header (ip0);
1000           tcp0 = (tcp_header_t *) udp0;
1001           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1002
1003           if (PREDICT_FALSE (proto0 == ~0))
1004             {
1005               next0 = def_slow;
1006               goto trace0;
1007             }
1008
1009           if (is_output_feature)
1010             {
1011               if (PREDICT_FALSE (nat_not_translate_output_feature_fwd
1012                                  (sm, ip0, thread_index, now, vm, b0)))
1013                 goto trace0;
1014             }
1015
1016           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1017             {
1018               next0 = def_slow;
1019               goto trace0;
1020             }
1021
1022           make_ed_kv (&ip0->src_address, &ip0->dst_address,
1023                       ip0->protocol, rx_fib_index0,
1024                       vnet_buffer (b0)->ip.reass.l4_src_port,
1025                       vnet_buffer (b0)->ip.reass.l4_dst_port, ~0ULL, &kv0);
1026
1027           // lookup for session
1028           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1029             {
1030               // session does not exist go slow path
1031               next0 = def_slow;
1032               goto trace0;
1033             }
1034           s0 = pool_elt_at_index (tsm->sessions, value0.value);
1035
1036           if (s0->tcp_close_timestamp)
1037             {
1038               if (now >= s0->tcp_close_timestamp)
1039                 {
1040                   // session is closed, go slow path
1041                   next0 = def_slow;
1042                 }
1043               else
1044                 {
1045                   // session in transitory timeout, drop
1046                   b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1047                   next0 = NAT_NEXT_DROP;
1048                 }
1049               goto trace0;
1050             }
1051
1052           // drop if session expired
1053           u64 sess_timeout_time;
1054           sess_timeout_time = s0->last_heard +
1055             (f64) nat44_session_get_timeout (sm, s0);
1056           if (now >= sess_timeout_time)
1057             {
1058               nat_free_session_data (sm, s0, thread_index, 0);
1059               nat44_ed_delete_session (sm, s0, thread_index, 1);
1060               // session is closed, go slow path
1061               next0 = def_slow;
1062               goto trace0;
1063             }
1064
1065           b0->flags |= VNET_BUFFER_F_IS_NATED;
1066
1067           if (!is_output_feature)
1068             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1069
1070           old_addr0 = ip0->src_address.as_u32;
1071           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1072           sum0 = ip0->checksum;
1073           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1074                                  src_address);
1075           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1076             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1077                                    s0->ext_host_addr.as_u32, ip4_header_t,
1078                                    dst_address);
1079           ip0->checksum = ip_csum_fold (sum0);
1080
1081           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1082
1083           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1084             {
1085               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1086                 {
1087                   new_port0 = udp0->src_port = s0->out2in.port;
1088                   sum0 = tcp0->checksum;
1089                   sum0 =
1090                     ip_csum_update (sum0, old_addr0, new_addr0,
1091                                     ip4_header_t, dst_address);
1092                   sum0 =
1093                     ip_csum_update (sum0, old_port0, new_port0,
1094                                     ip4_header_t, length);
1095                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1096                     {
1097                       sum0 =
1098                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1099                                         s0->ext_host_addr.as_u32,
1100                                         ip4_header_t, dst_address);
1101                       sum0 =
1102                         ip_csum_update (sum0,
1103                                         vnet_buffer (b0)->ip.
1104                                         reass.l4_dst_port, s0->ext_host_port,
1105                                         ip4_header_t, length);
1106                       tcp0->dst_port = s0->ext_host_port;
1107                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1108                     }
1109                   mss_clamping (sm, tcp0, &sum0);
1110                   tcp0->checksum = ip_csum_fold (sum0);
1111                 }
1112               tcp_packets++;
1113               if (nat44_set_tcp_session_state_i2o
1114                   (sm, now, s0, b0, thread_index))
1115                 goto trace0;
1116             }
1117           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1118                    && udp0->checksum)
1119             {
1120               new_port0 = udp0->src_port = s0->out2in.port;
1121               sum0 = udp0->checksum;
1122               sum0 =
1123                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1124                                 dst_address);
1125               sum0 =
1126                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1127                                 length);
1128               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1129                 {
1130                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1131                                          s0->ext_host_addr.as_u32,
1132                                          ip4_header_t, dst_address);
1133                   sum0 =
1134                     ip_csum_update (sum0,
1135                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1136                                     s0->ext_host_port, ip4_header_t, length);
1137                   udp0->dst_port = s0->ext_host_port;
1138                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1139                 }
1140               udp0->checksum = ip_csum_fold (sum0);
1141               udp_packets++;
1142             }
1143           else
1144             {
1145               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1146                 {
1147                   new_port0 = udp0->src_port = s0->out2in.port;
1148                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1149                     {
1150                       udp0->dst_port = s0->ext_host_port;
1151                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1152                     }
1153                   udp_packets++;
1154                 }
1155             }
1156
1157           /* Accounting */
1158           nat44_session_update_counters (s0, now,
1159                                          vlib_buffer_length_in_chain
1160                                          (vm, b0), thread_index);
1161           /* Per-user LRU list maintenance */
1162           nat44_session_update_lru (sm, s0, thread_index);
1163
1164         trace0:
1165           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1166                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1167             {
1168               nat_in2out_ed_trace_t *t =
1169                 vlib_add_trace (vm, node, b0, sizeof (*t));
1170               t->sw_if_index = sw_if_index0;
1171               t->next_index = next0;
1172               t->is_slow_path = 0;
1173
1174               if (s0)
1175                 t->session_index = s0 - tsm->sessions;
1176               else
1177                 t->session_index = ~0;
1178             }
1179
1180           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1181           /* verify speculative enqueue, maybe switch current next frame */
1182           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1183                                            to_next, n_left_to_next,
1184                                            bi0, next0);
1185         }
1186
1187       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1188     }
1189
1190   vlib_node_increment_counter (vm, stats_node_index,
1191                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1192                                pkts_processed);
1193   vlib_node_increment_counter (vm, stats_node_index,
1194                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1195   vlib_node_increment_counter (vm, stats_node_index,
1196                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1197   vlib_node_increment_counter (vm, stats_node_index,
1198                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1199                                icmp_packets);
1200   vlib_node_increment_counter (vm, stats_node_index,
1201                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1202                                other_packets);
1203   return frame->n_vectors;
1204 }
1205
1206 static inline uword
1207 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1208                                           vlib_node_runtime_t * node,
1209                                           vlib_frame_t * frame,
1210                                           int is_output_feature)
1211 {
1212   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
1213   nat_next_t next_index;
1214   snat_main_t *sm = &snat_main;
1215   f64 now = vlib_time_now (vm);
1216   u32 thread_index = vm->thread_index;
1217   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1218   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = 0;
1219
1220   stats_node_index = sm->ed_in2out_slowpath_node_index;
1221
1222   from = vlib_frame_vector_args (frame);
1223   n_left_from = frame->n_vectors;
1224   next_index = node->cached_next_index;
1225
1226   while (n_left_from > 0)
1227     {
1228       u32 n_left_to_next;
1229
1230       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1231
1232       while (n_left_from > 0 && n_left_to_next > 0)
1233         {
1234           u32 bi0;
1235           vlib_buffer_t *b0;
1236           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1237             new_addr0, old_addr0;
1238           u16 old_port0, new_port0;
1239           ip4_header_t *ip0;
1240           udp_header_t *udp0;
1241           tcp_header_t *tcp0;
1242           icmp46_header_t *icmp0;
1243           snat_session_t *s0 = 0;
1244           clib_bihash_kv_16_8_t kv0, value0;
1245           ip_csum_t sum0;
1246
1247           /* speculatively enqueue b0 to the current next frame */
1248           bi0 = from[0];
1249           to_next[0] = bi0;
1250           from += 1;
1251           to_next += 1;
1252           n_left_from -= 1;
1253           n_left_to_next -= 1;
1254
1255           b0 = vlib_get_buffer (vm, bi0);
1256
1257           if (is_output_feature)
1258             iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1259
1260           next0 = vnet_buffer2 (b0)->nat.arc_next;
1261
1262           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1263                                   iph_offset0);
1264
1265           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1266           rx_fib_index0 =
1267             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1268                                                  sw_if_index0);
1269
1270           if (PREDICT_FALSE (ip0->ttl == 1))
1271             {
1272               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1273               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1274                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1275                                            0);
1276               next0 = NAT_NEXT_ICMP_ERROR;
1277               goto trace0;
1278             }
1279
1280           udp0 = ip4_next_header (ip0);
1281           tcp0 = (tcp_header_t *) udp0;
1282           icmp0 = (icmp46_header_t *) udp0;
1283           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1284
1285           if (PREDICT_FALSE (proto0 == ~0))
1286             {
1287               s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1288                                                   rx_fib_index0,
1289                                                   thread_index, now,
1290                                                   vm, node);
1291               if (!s0)
1292                 next0 = NAT_NEXT_DROP;
1293
1294               other_packets++;
1295               goto trace0;
1296             }
1297
1298           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1299             {
1300               next0 = icmp_in2out_ed_slow_path
1301                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1302                  node, next0, now, thread_index, &s0);
1303               icmp_packets++;
1304               goto trace0;
1305             }
1306
1307           // move down
1308           make_ed_kv (&ip0->src_address, &ip0->dst_address,
1309                       ip0->protocol, rx_fib_index0,
1310                       vnet_buffer (b0)->ip.reass.l4_src_port,
1311                       vnet_buffer (b0)->ip.reass.l4_dst_port, ~0ULL, &kv0);
1312
1313           if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1314             {
1315               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1316
1317               if (s0->tcp_close_timestamp && now >= s0->tcp_close_timestamp)
1318                 {
1319                   nat_free_session_data (sm, s0, thread_index, 0);
1320                   nat44_ed_delete_session (sm, s0, thread_index, 1);
1321                   s0 = NULL;
1322                 }
1323             }
1324
1325           if (!s0)
1326             {
1327               if (is_output_feature)
1328                 {
1329                   if (PREDICT_FALSE
1330                       (nat44_ed_not_translate_output_feature
1331                        (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1332                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1333                         sw_if_index0,
1334                         vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1335                     goto trace0;
1336
1337                   /*
1338                    * Send DHCP packets to the ipv4 stack, or we won't
1339                    * be able to use dhcp client on the outside interface
1340                    */
1341                   if (PREDICT_FALSE
1342                       (proto0 == SNAT_PROTOCOL_UDP
1343                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1344                            clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1345                        && ip0->dst_address.as_u32 == 0xffffffff))
1346                     goto trace0;
1347                 }
1348               else
1349                 {
1350                   if (PREDICT_FALSE
1351                       (nat44_ed_not_translate
1352                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1353                         thread_index)))
1354                     goto trace0;
1355                 }
1356
1357               next0 =
1358                 slow_path_ed (sm, b0, ip0->src_address, ip0->dst_address,
1359                               vnet_buffer (b0)->ip.reass.l4_src_port,
1360                               vnet_buffer (b0)->ip.reass.l4_dst_port,
1361                               ip0->protocol, rx_fib_index0, &s0, node, next0,
1362                               thread_index, now);
1363
1364               if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1365                 goto trace0;
1366
1367               if (PREDICT_FALSE (!s0))
1368                 goto trace0;
1369
1370             }
1371
1372           b0->flags |= VNET_BUFFER_F_IS_NATED;
1373
1374           if (!is_output_feature)
1375             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1376
1377           old_addr0 = ip0->src_address.as_u32;
1378           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1379           sum0 = ip0->checksum;
1380           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1381                                  src_address);
1382           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1383             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1384                                    s0->ext_host_addr.as_u32, ip4_header_t,
1385                                    dst_address);
1386           ip0->checksum = ip_csum_fold (sum0);
1387
1388           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1389
1390           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1391             {
1392               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1393                 {
1394                   new_port0 = udp0->src_port = s0->out2in.port;
1395                   sum0 = tcp0->checksum;
1396                   sum0 =
1397                     ip_csum_update (sum0, old_addr0, new_addr0,
1398                                     ip4_header_t, dst_address);
1399                   sum0 =
1400                     ip_csum_update (sum0, old_port0, new_port0,
1401                                     ip4_header_t, length);
1402                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1403                     {
1404                       sum0 =
1405                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1406                                         s0->ext_host_addr.as_u32,
1407                                         ip4_header_t, dst_address);
1408                       sum0 =
1409                         ip_csum_update (sum0,
1410                                         vnet_buffer (b0)->ip.
1411                                         reass.l4_dst_port, s0->ext_host_port,
1412                                         ip4_header_t, length);
1413                       tcp0->dst_port = s0->ext_host_port;
1414                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1415                     }
1416                   mss_clamping (sm, tcp0, &sum0);
1417                   tcp0->checksum = ip_csum_fold (sum0);
1418                 }
1419               tcp_packets++;
1420               if (nat44_set_tcp_session_state_i2o
1421                   (sm, now, s0, b0, thread_index))
1422                 goto trace0;
1423             }
1424           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1425                    && udp0->checksum)
1426             {
1427               new_port0 = udp0->src_port = s0->out2in.port;
1428               sum0 = udp0->checksum;
1429               sum0 =
1430                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1431                                 dst_address);
1432               sum0 =
1433                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1434                                 length);
1435               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1436                 {
1437                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1438                                          s0->ext_host_addr.as_u32,
1439                                          ip4_header_t, dst_address);
1440                   sum0 =
1441                     ip_csum_update (sum0,
1442                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1443                                     s0->ext_host_port, ip4_header_t, length);
1444                   udp0->dst_port = s0->ext_host_port;
1445                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1446                 }
1447               udp0->checksum = ip_csum_fold (sum0);
1448               udp_packets++;
1449             }
1450           else
1451             {
1452               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1453                 {
1454                   new_port0 = udp0->src_port = s0->out2in.port;
1455                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1456                     {
1457                       udp0->dst_port = s0->ext_host_port;
1458                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1459                     }
1460                   udp_packets++;
1461                 }
1462             }
1463
1464           /* Accounting */
1465           nat44_session_update_counters (s0, now,
1466                                          vlib_buffer_length_in_chain
1467                                          (vm, b0), thread_index);
1468           /* Per-user LRU list maintenance */
1469           nat44_session_update_lru (sm, s0, thread_index);
1470
1471         trace0:
1472           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1473                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1474             {
1475               nat_in2out_ed_trace_t *t =
1476                 vlib_add_trace (vm, node, b0, sizeof (*t));
1477               t->sw_if_index = sw_if_index0;
1478               t->next_index = next0;
1479               t->is_slow_path = 1;
1480
1481               if (s0)
1482                 t->session_index = s0 - tsm->sessions;
1483               else
1484                 t->session_index = ~0;
1485             }
1486
1487           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1488
1489           /* verify speculative enqueue, maybe switch current next frame */
1490           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1491                                            to_next, n_left_to_next,
1492                                            bi0, next0);
1493         }
1494
1495       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1496     }
1497
1498   vlib_node_increment_counter (vm, stats_node_index,
1499                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1500                                pkts_processed);
1501   vlib_node_increment_counter (vm, stats_node_index,
1502                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1503   vlib_node_increment_counter (vm, stats_node_index,
1504                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1505   vlib_node_increment_counter (vm, stats_node_index,
1506                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1507                                icmp_packets);
1508   vlib_node_increment_counter (vm, stats_node_index,
1509                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1510                                other_packets);
1511   return frame->n_vectors;
1512 }
1513
1514 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1515                                      vlib_node_runtime_t * node,
1516                                      vlib_frame_t * frame)
1517 {
1518   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1519 }
1520
1521 /* *INDENT-OFF* */
1522 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1523   .name = "nat44-ed-in2out",
1524   .vector_size = sizeof (u32),
1525   .sibling_of = "nat-default",
1526   .format_trace = format_nat_in2out_ed_trace,
1527   .type = VLIB_NODE_TYPE_INTERNAL,
1528   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1529   .error_strings = nat_in2out_ed_error_strings,
1530   .runtime_data_bytes = sizeof (snat_runtime_t),
1531 };
1532 /* *INDENT-ON* */
1533
1534 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1535                                             vlib_node_runtime_t * node,
1536                                             vlib_frame_t * frame)
1537 {
1538   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1539 }
1540
1541 /* *INDENT-OFF* */
1542 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1543   .name = "nat44-ed-in2out-output",
1544   .vector_size = sizeof (u32),
1545   .sibling_of = "nat-default",
1546   .format_trace = format_nat_in2out_ed_trace,
1547   .type = VLIB_NODE_TYPE_INTERNAL,
1548   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1549   .error_strings = nat_in2out_ed_error_strings,
1550   .runtime_data_bytes = sizeof (snat_runtime_t),
1551 };
1552 /* *INDENT-ON* */
1553
1554 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1555                                               vlib_node_runtime_t *
1556                                               node, vlib_frame_t * frame)
1557 {
1558   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1559 }
1560
1561 /* *INDENT-OFF* */
1562 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1563   .name = "nat44-ed-in2out-slowpath",
1564   .vector_size = sizeof (u32),
1565   .sibling_of = "nat-default",
1566   .format_trace = format_nat_in2out_ed_trace,
1567   .type = VLIB_NODE_TYPE_INTERNAL,
1568   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1569   .error_strings = nat_in2out_ed_error_strings,
1570   .runtime_data_bytes = sizeof (snat_runtime_t),
1571 };
1572 /* *INDENT-ON* */
1573
1574 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1575                                                      vlib_node_runtime_t
1576                                                      * node,
1577                                                      vlib_frame_t * frame)
1578 {
1579   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1580 }
1581
1582 /* *INDENT-OFF* */
1583 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1584   .name = "nat44-ed-in2out-output-slowpath",
1585   .vector_size = sizeof (u32),
1586   .sibling_of = "nat-default",
1587   .format_trace = format_nat_in2out_ed_trace,
1588   .type = VLIB_NODE_TYPE_INTERNAL,
1589   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1590   .error_strings = nat_in2out_ed_error_strings,
1591   .runtime_data_bytes = sizeof (snat_runtime_t),
1592 };
1593 /* *INDENT-ON* */
1594
1595 static u8 *
1596 format_nat_pre_trace (u8 * s, va_list * args)
1597 {
1598   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1599   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1600   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1601   return format (s, "in2out next_index %d", t->next_index);
1602 }
1603
1604 VLIB_NODE_FN (nat_pre_in2out_node)
1605   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1606 {
1607   return nat_pre_node_fn_inline (vm, node, frame,
1608                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1609 }
1610
1611 /* *INDENT-OFF* */
1612 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1613   .name = "nat-pre-in2out",
1614   .vector_size = sizeof (u32),
1615   .sibling_of = "nat-default",
1616   .format_trace = format_nat_pre_trace,
1617   .type = VLIB_NODE_TYPE_INTERNAL,
1618   .n_errors = 0,
1619 };
1620 /* *INDENT-ON* */
1621
1622 /*
1623  * fd.io coding-style-patch-verification: ON
1624  *
1625  * Local Variables:
1626  * eval: (c-set-style "gnu")
1627  * End:
1628  */