src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 static char *nat_in2out_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_in2out_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat_in2out_ed_trace_t;
  48
  49 static u8 *
  50 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  59     "NAT44_IN2OUT_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 #ifndef CLIB_MARCH_VARIANT
  68 int
  69 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  70 {
  71   snat_main_t *sm = &snat_main;
  72   nat44_is_idle_session_ctx_t *ctx = arg;
  73   snat_session_t *s;
  74   u64 sess_timeout_time;
  75   nat_ed_ses_key_t ed_key;
  76   clib_bihash_kv_16_8_t ed_kv;
  77   int i;
  78   snat_address_t *a;
  79   snat_session_key_t key;
  80   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  81                                                        ctx->thread_index);
  82
  83   s = pool_elt_at_index (tsm->sessions, kv->value);
  84   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  85   if (ctx->now >= sess_timeout_time)
  86     {
  87       if (is_fwd_bypass_session (s))
  88         goto delete;
  89
  90       ed_key.l_addr = s->out2in.addr;
  91       ed_key.r_addr = s->ext_host_addr;
  92       ed_key.fib_index = s->out2in.fib_index;
  93       if (snat_is_unk_proto_session (s))
  94         {
  95           ed_key.proto = s->in2out.port;
  96           ed_key.r_port = 0;
  97           ed_key.l_port = 0;
  98         }
  99       else
 100         {
 101           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 102           ed_key.l_port = s->out2in.port;
 103           ed_key.r_port = s->ext_host_port;
 104         }
 105       ed_kv.key[0] = ed_key.as_u64[0];
 106       ed_kv.key[1] = ed_key.as_u64[1];
 107       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 108         nat_elog_warn ("out2in_ed key del failed");
 109
 110       if (snat_is_unk_proto_session (s))
 111         goto delete;
 112
 113       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 114                                            s->in2out.addr.as_u32,
 115                                            s->out2in.addr.as_u32,
 116                                            s->in2out.protocol,
 117                                            s->in2out.port,
 118                                            s->out2in.port,
 119                                            s->in2out.fib_index);
 120
 121       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 122                              &s->in2out.addr, s->in2out.port,
 123                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 124                              &s->out2in.addr, s->out2in.port,
 125                              &s->ext_host_addr, s->ext_host_port,
 126                              s->in2out.protocol, is_twice_nat_session (s));
 127
 128       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 129                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 130                    ctx->thread_index);
 131
 132       if (is_twice_nat_session (s))
 133         {
 134           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 135             {
 136               key.protocol = s->in2out.protocol;
 137               key.port = s->ext_host_nat_port;
 138               a = sm->twice_nat_addresses + i;
 139               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 140                 {
 141                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 142                                                       ctx->thread_index,
 143                                                       &key);
 144                   break;
 145                 }
 146             }
 147         }
 148
 149       if (snat_is_session_static (s))
 150         goto delete;
 151
 152       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 153                                           &s->out2in);
 154     delete:
 155       nat44_ed_delete_session (sm, s, ctx->thread_index, 1);
 156       return 1;
 157     }
 158
 159   return 0;
 160 }
 161 #endif
 162
 163 static inline u32
 164 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 165                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 166                           u32 sw_if_index0, u32 rx_fib_index0,
 167                           vlib_node_runtime_t * node, u32 next0, f64 now,
 168                           u32 thread_index, snat_session_t ** p_s0)
 169 {
 170   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 171                        next0, thread_index, p_s0, 0);
 172   snat_session_t *s0 = *p_s0;
 173   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 174     {
 175       /* Accounting */
 176       nat44_session_update_counters (s0, now,
 177                                      vlib_buffer_length_in_chain
 178                                      (sm->vlib_main, b0), thread_index);
 179       /* Per-user LRU list maintenance */
 180       nat44_session_update_lru (sm, s0, thread_index);
 181     }
 182   return next0;
 183 }
 184
 185 static_always_inline u16
 186 snat_random_port (u16 min, u16 max)
 187 {
 188   snat_main_t *sm = &snat_main;
 189   return min + random_u32 (&sm->random_seed) /
 190     (random_u32_max () / (max - min + 1) + 1);
 191 }
 192
 193 static int
 194 nat_ed_alloc_addr_and_port (snat_main_t * sm, u32 fib_index,
 195                             u32 thread_index, nat_ed_ses_key_t * key,
 196                             snat_session_key_t * key1, u16 port_per_thread,
 197                             u32 snat_thread_index,
 198                             snat_session_t * s,
 199                             clib_bihash_kv_16_8_t * out2in_ed_kv)
 200 {
 201   int i;
 202   snat_address_t *a, *ga = 0;
 203   u32 portnum;
 204   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 205
 206   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 207
 208   for (i = 0; i < vec_len (sm->addresses); i++)
 209     {
 210       a = sm->addresses + i;
 211       switch (key1->protocol)
 212         {
 213 #define _(N, j, n, unused)                                                    \
 214   case SNAT_PROTOCOL_##N:                                                     \
 215     if (a->fib_index == fib_index)                                            \
 216       {                                                                       \
 217         u16 port = snat_random_port (1, port_per_thread);                     \
 218         u16 attempts = port_per_thread;                                       \
 219         while (attempts > 0)                                                  \
 220           {                                                                   \
 221             --attempts;                                                       \
 222             portnum = port_thread_offset + port;                              \
 223             make_ed_kv (out2in_ed_kv, &a->addr, &key->r_addr, key->proto,     \
 224                         s->out2in.fib_index, clib_host_to_net_u16 (portnum),  \
 225                         key->r_port);                                         \
 226             out2in_ed_kv->value = s - tsm->sessions;                          \
 227             int rv = clib_bihash_add_del_16_8 (&tsm->out2in_ed, out2in_ed_kv, \
 228                                                2 /* is_add */);               \
 229             if (0 == rv)                                                      \
 230               {                                                               \
 231                 ++a->busy_##n##_port_refcounts[portnum];                      \
 232                 a->busy_##n##_ports_per_thread[thread_index]++;               \
 233                 a->busy_##n##_ports++;                                        \
 234                 key1->addr = a->addr;                                         \
 235                 key1->port = clib_host_to_net_u16 (portnum);                  \
 236                 return 0;                                                     \
 237               }                                                               \
 238             port = (port + 1) % port_per_thread;                              \
 239           }                                                                   \
 240       }                                                                       \
 241     else if (a->fib_index == ~0)                                              \
 242       {                                                                       \
 243         ga = a;                                                               \
 244       }                                                                       \
 245     break;
 246
 247           foreach_snat_protocol;
 248         default:
 249           nat_elog_info ("unknown protocol");
 250           return 1;
 251         }
 252     }
 253
 254   if (ga)
 255     {
 256       /* fake fib_index to reuse macro */
 257       fib_index = ~0;
 258       a = ga;
 259       switch (key1->protocol)
 260         {
 261           foreach_snat_protocol;
 262         default:
 263           nat_elog_info ("unknown protocol");
 264           return 1;
 265         }
 266     }
 267
 268 #undef _
 269
 270   /* Totally out of translations to use... */
 271   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 272   return 1;
 273 }
 274
 275 static u32
 276 slow_path_ed (snat_main_t * sm,
 277               vlib_buffer_t * b,
 278               u32 rx_fib_index,
 279               clib_bihash_kv_16_8_t * kv,
 280               snat_session_t ** sessionp,
 281               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 282 {
 283   snat_session_t *s = NULL;
 284   snat_session_key_t key0, key1;
 285   lb_nat_type_t lb = 0;
 286   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 287   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 288   u32 proto = ip_proto_to_snat_proto (key->proto);
 289   nat_outside_fib_t *outside_fib;
 290   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 291   clib_bihash_kv_16_8_t out2in_ed_kv;
 292   bool out2in_ed_inserted = false;
 293   u8 identity_nat;
 294   fib_prefix_t pfx = {
 295     .fp_proto = FIB_PROTOCOL_IP4,
 296     .fp_len = 32,
 297     .fp_addr = {
 298                 .ip4.as_u32 = key->r_addr.as_u32,
 299                 },
 300   };
 301   nat44_is_idle_session_ctx_t ctx;
 302
 303   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_TCP))
 304     {
 305       if (PREDICT_FALSE
 306           (!tcp_flags_is_init
 307            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 308         {
 309           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 310           return NAT_NEXT_DROP;
 311         }
 312     }
 313
 314   if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 315     {
 316       if (!nat_global_lru_free_one (sm, thread_index, now))
 317         {
 318           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 319           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 320           nat_elog_notice ("maximum sessions exceeded");
 321           return NAT_NEXT_DROP;
 322         }
 323     }
 324
 325   key0.addr = key->l_addr;
 326   key0.port = key->l_port;
 327   key1.protocol = key0.protocol = proto;
 328   key0.fib_index = rx_fib_index;
 329   key1.fib_index = sm->outside_fib_index;
 330
 331   /* First try to match static mapping by local address and port */
 332   if (snat_static_mapping_match
 333       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 334     {
 335       s = nat_ed_session_alloc (sm, thread_index, now);
 336       if (!s)
 337         {
 338           nat_elog_warn ("create NAT session failed");
 339           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 340           goto drop;
 341         }
 342       switch (vec_len (sm->outside_fibs))
 343         {
 344         case 0:
 345           s->out2in.fib_index = sm->outside_fib_index;
 346           break;
 347         case 1:
 348           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 349           break;
 350         default:
 351           /* *INDENT-OFF* */
 352           vec_foreach (outside_fib, sm->outside_fibs)
 353           {
 354             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 355             if (FIB_NODE_INDEX_INVALID != fei)
 356               {
 357                 if (fib_entry_get_resolving_interface (fei) != ~0)
 358                   {
 359                     s->out2in.fib_index = outside_fib->fib_index;
 360                     break;
 361                   }
 362               }
 363           }
 364           /* *INDENT-ON* */
 365           break;
 366         }
 367
 368       /* Try to create dynamic translation */
 369       if (nat_ed_alloc_addr_and_port (sm, rx_fib_index,
 370                                       thread_index, key, &key1,
 371                                       sm->port_per_thread,
 372                                       tsm->snat_thread_index, s,
 373                                       &out2in_ed_kv))
 374         {
 375           nat_elog_notice ("addresses exhausted");
 376           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 377           goto drop;
 378         }
 379
 380       out2in_ed_inserted = true;
 381     }
 382   else
 383     {
 384       if (PREDICT_FALSE (identity_nat))
 385         {
 386           *sessionp = s;
 387           return next;
 388         }
 389       s = nat_ed_session_alloc (sm, thread_index, now);
 390       if (!s)
 391         {
 392           nat_elog_warn ("create NAT session failed");
 393           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 394           goto drop;
 395         }
 396       switch (vec_len (sm->outside_fibs))
 397         {
 398         case 0:
 399           s->out2in.fib_index = sm->outside_fib_index;
 400           break;
 401         case 1:
 402           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 403           break;
 404         default:
 405           /* *INDENT-OFF* */
 406           vec_foreach (outside_fib, sm->outside_fibs)
 407           {
 408             fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 409             if (FIB_NODE_INDEX_INVALID != fei)
 410               {
 411                 if (fib_entry_get_resolving_interface (fei) != ~0)
 412                   {
 413                     s->out2in.fib_index = outside_fib->fib_index;
 414                     break;
 415                   }
 416               }
 417           }
 418           /* *INDENT-ON* */
 419           break;
 420         }
 421
 422       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 423
 424
 425       make_ed_kv (&out2in_ed_kv, &key1.addr, &key->r_addr, key->proto,
 426                   s->out2in.fib_index, key1.port, key->r_port);
 427       out2in_ed_kv.value = s - tsm->sessions;
 428       if (clib_bihash_add_or_overwrite_stale_16_8
 429           (&tsm->out2in_ed, &out2in_ed_kv, nat44_o2i_ed_is_idle_session_cb,
 430            &ctx))
 431         nat_elog_notice ("out2in-ed key add failed");
 432       out2in_ed_inserted = true;
 433     }
 434
 435   if (lb)
 436     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 437   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 438   s->ext_host_addr = key->r_addr;
 439   s->ext_host_port = key->r_port;
 440   s->in2out = key0;
 441   s->out2in = key1;
 442   s->out2in.protocol = key0.protocol;
 443
 444   kv->value = s - tsm->sessions;
 445   ctx.now = now;
 446   ctx.thread_index = thread_index;
 447   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 448                                                nat44_i2o_ed_is_idle_session_cb,
 449                                                &ctx))
 450     nat_elog_notice ("in2out-ed key add failed");
 451
 452   *sessionp = s;
 453
 454   /* log NAT event */
 455   snat_ipfix_logging_nat44_ses_create (thread_index,
 456                                        s->in2out.addr.as_u32,
 457                                        s->out2in.addr.as_u32,
 458                                        s->in2out.protocol,
 459                                        s->in2out.port,
 460                                        s->out2in.port, s->in2out.fib_index);
 461
 462   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 463                          &s->in2out.addr, s->in2out.port,
 464                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 465                          &s->out2in.addr, s->out2in.port,
 466                          &s->ext_host_addr, s->ext_host_port,
 467                          s->in2out.protocol, 0);
 468
 469   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 470                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 471                &s->ext_host_nat_addr, s->ext_host_nat_port,
 472                s->in2out.protocol, s->in2out.fib_index, s->flags,
 473                thread_index, 0);
 474
 475   return next;
 476 drop:
 477   if (out2in_ed_inserted)
 478     {
 479       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &out2in_ed_kv, 0))
 480         nat_elog_notice ("out2in-ed key del failed");
 481     }
 482   if (s)
 483     {
 484       nat_free_session_data (sm, s, thread_index, 0);
 485       nat44_ed_delete_session (sm, s, thread_index, 1);
 486     }
 487   return NAT_NEXT_DROP;
 488 }
 489
 490 static_always_inline int
 491 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 492                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 493                         u32 rx_fib_index, u32 thread_index)
 494 {
 495   udp_header_t *udp = ip4_next_header (ip);
 496   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 497   clib_bihash_kv_16_8_t kv, value;
 498   snat_session_key_t key0, key1;
 499
 500   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 501               sm->outside_fib_index, udp->dst_port, udp->src_port);
 502
 503   /* NAT packet aimed at external address if */
 504   /* has active sessions */
 505   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 506     {
 507       key0.addr = ip->dst_address;
 508       key0.port = udp->dst_port;
 509       key0.protocol = proto;
 510       key0.fib_index = sm->outside_fib_index;
 511       /* or is static mappings */
 512       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 513         return 0;
 514     }
 515   else
 516     return 0;
 517
 518   if (sm->forwarding_enabled)
 519     return 1;
 520
 521   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 522                                   rx_fib_index);
 523 }
 524
 525 static_always_inline int
 526 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 527                                       u32 thread_index, f64 now,
 528                                       vlib_main_t * vm, vlib_buffer_t * b)
 529 {
 530   nat_ed_ses_key_t key;
 531   clib_bihash_kv_16_8_t kv, value;
 532   snat_session_t *s = 0;
 533   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 534
 535   if (!sm->forwarding_enabled)
 536     return 0;
 537
 538   if (ip->protocol == IP_PROTOCOL_ICMP)
 539     {
 540       key.as_u64[0] = key.as_u64[1] = 0;
 541       if (get_icmp_i2o_ed_key (b, ip, &key))
 542         return 0;
 543       key.fib_index = 0;
 544       kv.key[0] = key.as_u64[0];
 545       kv.key[1] = key.as_u64[1];
 546     }
 547   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 548     {
 549       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 550                   vnet_buffer (b)->ip.reass.l4_src_port,
 551                   vnet_buffer (b)->ip.reass.l4_dst_port);
 552     }
 553   else
 554     {
 555       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 556                   0);
 557     }
 558
 559   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 560     {
 561       s = pool_elt_at_index (tsm->sessions, value.value);
 562       if (is_fwd_bypass_session (s))
 563         {
 564           if (ip->protocol == IP_PROTOCOL_TCP)
 565             {
 566               if (nat44_set_tcp_session_state_i2o
 567                   (sm, now, s, b, thread_index))
 568                 return 1;
 569             }
 570           /* Accounting */
 571           nat44_session_update_counters (s, now,
 572                                          vlib_buffer_length_in_chain (vm, b),
 573                                          thread_index);
 574           /* Per-user LRU list maintenance */
 575           nat44_session_update_lru (sm, s, thread_index);
 576           return 1;
 577         }
 578       else
 579         return 0;
 580     }
 581
 582   return 0;
 583 }
 584
 585 static_always_inline int
 586 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 587                                        u8 proto, u16 src_port, u16 dst_port,
 588                                        u32 thread_index, u32 rx_sw_if_index,
 589                                        u32 tx_sw_if_index)
 590 {
 591   clib_bihash_kv_16_8_t kv, value;
 592   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 593   snat_interface_t *i;
 594   snat_session_t *s;
 595   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 596   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 597
 598   /* src NAT check */
 599   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 600               src_port, dst_port);
 601   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 602     {
 603       s = pool_elt_at_index (tsm->sessions, value.value);
 604       if (nat44_is_ses_closed (s))
 605         {
 606           nat_free_session_data (sm, s, thread_index, 0);
 607           nat44_ed_delete_session (sm, s, thread_index, 1);
 608         }
 609       else
 610         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 611       return 1;
 612     }
 613
 614   /* dst NAT check */
 615   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 616               dst_port, src_port);
 617   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 618     {
 619       s = pool_elt_at_index (tsm->sessions, value.value);
 620       if (is_fwd_bypass_session (s))
 621         return 0;
 622
 623       /* hairpinning */
 624       /* *INDENT-OFF* */
 625       pool_foreach (i, sm->output_feature_interfaces,
 626       ({
 627         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 628            return 0;
 629       }));
 630       /* *INDENT-ON* */
 631       return 1;
 632     }
 633
 634   return 0;
 635 }
 636
 637 #ifndef CLIB_MARCH_VARIANT
 638 u32
 639 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 640                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 641                       u8 * p_proto, snat_session_key_t * p_value,
 642                       u8 * p_dont_translate, void *d, void *e)
 643 {
 644   u32 sw_if_index;
 645   u32 rx_fib_index;
 646   nat_ed_ses_key_t key;
 647   snat_session_t *s = 0;
 648   u8 dont_translate = 0;
 649   clib_bihash_kv_16_8_t kv, value;
 650   u32 next = ~0;
 651   int err;
 652   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 653
 654   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 655   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 656
 657   key.as_u64[0] = key.as_u64[1] = 0;
 658   err = get_icmp_i2o_ed_key (b, ip, &key);
 659   if (err != 0)
 660     {
 661       b->error = node->errors[err];
 662       next = NAT_NEXT_DROP;
 663       goto out;
 664     }
 665   key.fib_index = rx_fib_index;
 666
 667   kv.key[0] = key.as_u64[0];
 668   kv.key[1] = key.as_u64[1];
 669
 670   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 671     {
 672       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 673         {
 674           if (PREDICT_FALSE
 675               (nat44_ed_not_translate_output_feature
 676                (sm, ip, key.proto, key.l_port, key.r_port, thread_index,
 677                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX])))
 678             {
 679               dont_translate = 1;
 680               goto out;
 681             }
 682         }
 683       else
 684         {
 685           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 686                                                      ip, SNAT_PROTOCOL_ICMP,
 687                                                      rx_fib_index,
 688                                                      thread_index)))
 689             {
 690               dont_translate = 1;
 691               goto out;
 692             }
 693         }
 694
 695       if (PREDICT_FALSE
 696           (icmp_type_is_error_message
 697            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 698         {
 699           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 700           next = NAT_NEXT_DROP;
 701           goto out;
 702         }
 703
 704       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 705                            thread_index, vlib_time_now (sm->vlib_main));
 706
 707       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 708         goto out;
 709
 710       if (!s)
 711         {
 712           dont_translate = 1;
 713           goto out;
 714         }
 715     }
 716   else
 717     {
 718       if (PREDICT_FALSE
 719           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 720            ICMP4_echo_request
 721            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 722            ICMP4_echo_reply
 723            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 724                                            reass.icmp_type_or_tcp_flags)))
 725         {
 726           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 727           next = NAT_NEXT_DROP;
 728           goto out;
 729         }
 730
 731       s = pool_elt_at_index (tsm->sessions, value.value);
 732     }
 733
 734   *p_proto = ip_proto_to_snat_proto (key.proto);
 735 out:
 736   if (s)
 737     *p_value = s->out2in;
 738   *p_dont_translate = dont_translate;
 739   if (d)
 740     *(snat_session_t **) d = s;
 741   return next;
 742 }
 743 #endif
 744
 745 static snat_session_t *
 746 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 747                                vlib_buffer_t * b,
 748                                ip4_header_t * ip,
 749                                u32 rx_fib_index,
 750                                u32 thread_index,
 751                                f64 now,
 752                                vlib_main_t * vm, vlib_node_runtime_t * node)
 753 {
 754   clib_bihash_kv_8_8_t kv, value;
 755   clib_bihash_kv_16_8_t s_kv, s_value;
 756   snat_static_mapping_t *m;
 757   u32 old_addr, new_addr = 0;
 758   ip_csum_t sum;
 759   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 760   snat_session_t *s;
 761   u32 outside_fib_index = sm->outside_fib_index;
 762   int i;
 763   u8 is_sm = 0;
 764   nat_outside_fib_t *outside_fib;
 765   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 766   fib_prefix_t pfx = {
 767     .fp_proto = FIB_PROTOCOL_IP4,
 768     .fp_len = 32,
 769     .fp_addr = {
 770                 .ip4.as_u32 = ip->dst_address.as_u32,
 771                 },
 772   };
 773
 774   switch (vec_len (sm->outside_fibs))
 775     {
 776     case 0:
 777       outside_fib_index = sm->outside_fib_index;
 778       break;
 779     case 1:
 780       outside_fib_index = sm->outside_fibs[0].fib_index;
 781       break;
 782     default:
 783       /* *INDENT-OFF* */
 784       vec_foreach (outside_fib, sm->outside_fibs)
 785         {
 786           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 787           if (FIB_NODE_INDEX_INVALID != fei)
 788             {
 789               if (fib_entry_get_resolving_interface (fei) != ~0)
 790                 {
 791                   outside_fib_index = outside_fib->fib_index;
 792                   break;
 793                 }
 794             }
 795         }
 796       /* *INDENT-ON* */
 797       break;
 798     }
 799   old_addr = ip->src_address.as_u32;
 800
 801   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 802               rx_fib_index, 0, 0);
 803
 804   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 805     {
 806       s = pool_elt_at_index (tsm->sessions, s_value.value);
 807       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 808     }
 809   else
 810     {
 811       if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 812         {
 813           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 814           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 815           nat_elog_notice ("maximum sessions exceeded");
 816           return 0;
 817         }
 818
 819       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 820
 821       /* Try to find static mapping first */
 822       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 823         {
 824           m = pool_elt_at_index (sm->static_mappings, value.value);
 825           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 826           is_sm = 1;
 827           goto create_ses;
 828         }
 829       else
 830         {
 831           /* *INDENT-OFF* */
 832           pool_foreach (s, tsm->sessions, {
 833             if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 834               {
 835                 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 836
 837                 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 838                             outside_fib_index, 0, 0);
 839                 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 840                   goto create_ses;
 841
 842                 break;
 843               }
 844           });
 845           /* *INDENT-ON* */
 846
 847           for (i = 0; i < vec_len (sm->addresses); i++)
 848             {
 849               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 850                           ip->protocol, outside_fib_index, 0, 0);
 851               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 852                 {
 853                   new_addr = ip->src_address.as_u32 =
 854                     sm->addresses[i].addr.as_u32;
 855                   goto create_ses;
 856                 }
 857             }
 858           return 0;
 859         }
 860
 861     create_ses:
 862       s = nat_ed_session_alloc (sm, thread_index, now);
 863       if (!s)
 864         {
 865           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 866           nat_elog_warn ("create NAT session failed");
 867           return 0;
 868         }
 869
 870       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 871       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 872       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 873       s->out2in.addr.as_u32 = new_addr;
 874       s->out2in.fib_index = outside_fib_index;
 875       s->in2out.addr.as_u32 = old_addr;
 876       s->in2out.fib_index = rx_fib_index;
 877       s->in2out.port = s->out2in.port = ip->protocol;
 878       if (is_sm)
 879         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 880
 881       /* Add to lookup tables */
 882       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 883                   rx_fib_index, 0, 0);
 884       s_kv.value = s - tsm->sessions;
 885       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 886         nat_elog_notice ("in2out key add failed");
 887
 888       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 889                   outside_fib_index, 0, 0);
 890       s_kv.value = s - tsm->sessions;
 891       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 892         nat_elog_notice ("out2in key add failed");
 893     }
 894
 895   /* Update IP checksum */
 896   sum = ip->checksum;
 897   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 898   ip->checksum = ip_csum_fold (sum);
 899
 900   /* Accounting */
 901   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 902                                  thread_index);
 903   /* Per-user LRU list maintenance */
 904   nat44_session_update_lru (sm, s, thread_index);
 905
 906   /* Hairpinning */
 907   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 908     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 909
 910   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 911     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 912
 913   return s;
 914 }
 915
 916 static inline uword
 917 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 918                                           vlib_node_runtime_t * node,
 919                                           vlib_frame_t * frame,
 920                                           int is_output_feature)
 921 {
 922   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 923   nat_next_t next_index;
 924   snat_main_t *sm = &snat_main;
 925   f64 now = vlib_time_now (vm);
 926   u32 thread_index = vm->thread_index;
 927   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 928   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 929     0, def_slow;
 930
 931   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 932     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 933
 934   stats_node_index = sm->ed_in2out_node_index;
 935
 936   from = vlib_frame_vector_args (frame);
 937   n_left_from = frame->n_vectors;
 938   next_index = node->cached_next_index;
 939
 940   while (n_left_from > 0)
 941     {
 942       u32 n_left_to_next;
 943
 944       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 945
 946       while (n_left_from > 0 && n_left_to_next > 0)
 947         {
 948           u32 bi0;
 949           vlib_buffer_t *b0;
 950           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 951             new_addr0, old_addr0;
 952           u16 old_port0, new_port0;
 953           ip4_header_t *ip0;
 954           udp_header_t *udp0;
 955           tcp_header_t *tcp0;
 956           snat_session_t *s0 = 0;
 957           clib_bihash_kv_16_8_t kv0, value0;
 958           ip_csum_t sum0;
 959
 960           /* speculatively enqueue b0 to the current next frame */
 961           bi0 = from[0];
 962           to_next[0] = bi0;
 963           from += 1;
 964           to_next += 1;
 965           n_left_from -= 1;
 966           n_left_to_next -= 1;
 967
 968           b0 = vlib_get_buffer (vm, bi0);
 969
 970           if (is_output_feature)
 971             {
 972               vnet_feature_next (&vnet_buffer2 (b0)->nat.arc_next, b0);
 973               iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 974             }
 975
 976           next0 = vnet_buffer2 (b0)->nat.arc_next;
 977
 978           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 979                                   iph_offset0);
 980
 981           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 982           rx_fib_index0 =
 983             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 984                                                  sw_if_index0);
 985
 986           if (PREDICT_FALSE (ip0->ttl == 1))
 987             {
 988               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 989               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 990                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 991                                            0);
 992               next0 = NAT_NEXT_ICMP_ERROR;
 993               goto trace0;
 994             }
 995
 996           udp0 = ip4_next_header (ip0);
 997           tcp0 = (tcp_header_t *) udp0;
 998           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 999
1000           if (PREDICT_FALSE (proto0 == ~0))
1001             {
1002               next0 = def_slow;
1003               goto trace0;
1004             }
1005
1006           if (is_output_feature)
1007             {
1008               if (PREDICT_FALSE (nat_not_translate_output_feature_fwd
1009                                  (sm, ip0, thread_index, now, vm, b0)))
1010                 goto trace0;
1011             }
1012
1013           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1014             {
1015               next0 = def_slow;
1016               goto trace0;
1017             }
1018
1019           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1020                       ip0->protocol, rx_fib_index0,
1021                       vnet_buffer (b0)->ip.reass.l4_src_port,
1022                       vnet_buffer (b0)->ip.reass.l4_dst_port);
1023
1024           // lookup for session
1025           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1026             {
1027               // session does not exist go slow path
1028               next0 = def_slow;
1029               goto trace0;
1030             }
1031           s0 = pool_elt_at_index (tsm->sessions, value0.value);
1032
1033           if (s0->tcp_close_timestamp)
1034             {
1035               if (now >= s0->tcp_close_timestamp)
1036                 {
1037                   // session is closed, go slow path
1038                   next0 = def_slow;
1039                 }
1040               else
1041                 {
1042                   // session in transitory timeout, drop
1043                   b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1044                   next0 = NAT_NEXT_DROP;
1045                 }
1046               goto trace0;
1047             }
1048
1049           // drop if session expired
1050           u64 sess_timeout_time;
1051           sess_timeout_time = s0->last_heard +
1052             (f64) nat44_session_get_timeout (sm, s0);
1053           if (now >= sess_timeout_time)
1054             {
1055               nat_free_session_data (sm, s0, thread_index, 0);
1056               nat44_ed_delete_session (sm, s0, thread_index, 1);
1057               // session is closed, go slow path
1058               next0 = def_slow;
1059               goto trace0;
1060             }
1061
1062           b0->flags |= VNET_BUFFER_F_IS_NATED;
1063
1064           if (!is_output_feature)
1065             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1066
1067           old_addr0 = ip0->src_address.as_u32;
1068           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1069           sum0 = ip0->checksum;
1070           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1071                                  src_address);
1072           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1073             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1074                                    s0->ext_host_addr.as_u32, ip4_header_t,
1075                                    dst_address);
1076           ip0->checksum = ip_csum_fold (sum0);
1077
1078           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1079
1080           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1081             {
1082               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1083                 {
1084                   new_port0 = udp0->src_port = s0->out2in.port;
1085                   sum0 = tcp0->checksum;
1086                   sum0 =
1087                     ip_csum_update (sum0, old_addr0, new_addr0,
1088                                     ip4_header_t, dst_address);
1089                   sum0 =
1090                     ip_csum_update (sum0, old_port0, new_port0,
1091                                     ip4_header_t, length);
1092                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1093                     {
1094                       sum0 =
1095                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1096                                         s0->ext_host_addr.as_u32,
1097                                         ip4_header_t, dst_address);
1098                       sum0 =
1099                         ip_csum_update (sum0,
1100                                         vnet_buffer (b0)->ip.
1101                                         reass.l4_dst_port, s0->ext_host_port,
1102                                         ip4_header_t, length);
1103                       tcp0->dst_port = s0->ext_host_port;
1104                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1105                     }
1106                   mss_clamping (sm, tcp0, &sum0);
1107                   tcp0->checksum = ip_csum_fold (sum0);
1108                 }
1109               tcp_packets++;
1110               if (nat44_set_tcp_session_state_i2o
1111                   (sm, now, s0, b0, thread_index))
1112                 goto trace0;
1113             }
1114           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1115                    && udp0->checksum)
1116             {
1117               new_port0 = udp0->src_port = s0->out2in.port;
1118               sum0 = udp0->checksum;
1119               sum0 =
1120                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1121                                 dst_address);
1122               sum0 =
1123                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1124                                 length);
1125               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1126                 {
1127                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1128                                          s0->ext_host_addr.as_u32,
1129                                          ip4_header_t, dst_address);
1130                   sum0 =
1131                     ip_csum_update (sum0,
1132                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1133                                     s0->ext_host_port, ip4_header_t, length);
1134                   udp0->dst_port = s0->ext_host_port;
1135                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1136                 }
1137               udp0->checksum = ip_csum_fold (sum0);
1138               udp_packets++;
1139             }
1140           else
1141             {
1142               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1143                 {
1144                   new_port0 = udp0->src_port = s0->out2in.port;
1145                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1146                     {
1147                       udp0->dst_port = s0->ext_host_port;
1148                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1149                     }
1150                   udp_packets++;
1151                 }
1152             }
1153
1154           /* Accounting */
1155           nat44_session_update_counters (s0, now,
1156                                          vlib_buffer_length_in_chain
1157                                          (vm, b0), thread_index);
1158           /* Per-user LRU list maintenance */
1159           nat44_session_update_lru (sm, s0, thread_index);
1160
1161         trace0:
1162           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1163                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1164             {
1165               nat_in2out_ed_trace_t *t =
1166                 vlib_add_trace (vm, node, b0, sizeof (*t));
1167               t->sw_if_index = sw_if_index0;
1168               t->next_index = next0;
1169               t->is_slow_path = 0;
1170
1171               if (s0)
1172                 t->session_index = s0 - tsm->sessions;
1173               else
1174                 t->session_index = ~0;
1175             }
1176
1177           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1178           /* verify speculative enqueue, maybe switch current next frame */
1179           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1180                                            to_next, n_left_to_next,
1181                                            bi0, next0);
1182         }
1183
1184       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1185     }
1186
1187   vlib_node_increment_counter (vm, stats_node_index,
1188                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1189                                pkts_processed);
1190   vlib_node_increment_counter (vm, stats_node_index,
1191                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1192   vlib_node_increment_counter (vm, stats_node_index,
1193                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1194   vlib_node_increment_counter (vm, stats_node_index,
1195                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1196                                icmp_packets);
1197   vlib_node_increment_counter (vm, stats_node_index,
1198                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1199                                other_packets);
1200   return frame->n_vectors;
1201 }
1202
1203 static inline uword
1204 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1205                                           vlib_node_runtime_t * node,
1206                                           vlib_frame_t * frame,
1207                                           int is_output_feature)
1208 {
1209   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
1210   nat_next_t next_index;
1211   snat_main_t *sm = &snat_main;
1212   f64 now = vlib_time_now (vm);
1213   u32 thread_index = vm->thread_index;
1214   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1215   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = 0;
1216
1217   stats_node_index = sm->ed_in2out_slowpath_node_index;
1218
1219   from = vlib_frame_vector_args (frame);
1220   n_left_from = frame->n_vectors;
1221   next_index = node->cached_next_index;
1222
1223   while (n_left_from > 0)
1224     {
1225       u32 n_left_to_next;
1226
1227       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1228
1229       while (n_left_from > 0 && n_left_to_next > 0)
1230         {
1231           u32 bi0;
1232           vlib_buffer_t *b0;
1233           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1234             new_addr0, old_addr0;
1235           u16 old_port0, new_port0;
1236           ip4_header_t *ip0;
1237           udp_header_t *udp0;
1238           tcp_header_t *tcp0;
1239           icmp46_header_t *icmp0;
1240           snat_session_t *s0 = 0;
1241           clib_bihash_kv_16_8_t kv0, value0;
1242           ip_csum_t sum0;
1243
1244           /* speculatively enqueue b0 to the current next frame */
1245           bi0 = from[0];
1246           to_next[0] = bi0;
1247           from += 1;
1248           to_next += 1;
1249           n_left_from -= 1;
1250           n_left_to_next -= 1;
1251
1252           b0 = vlib_get_buffer (vm, bi0);
1253
1254           if (is_output_feature)
1255             iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1256
1257           next0 = vnet_buffer2 (b0)->nat.arc_next;
1258
1259           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1260                                   iph_offset0);
1261
1262           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1263           rx_fib_index0 =
1264             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1265                                                  sw_if_index0);
1266
1267           if (PREDICT_FALSE (ip0->ttl == 1))
1268             {
1269               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1270               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1271                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1272                                            0);
1273               next0 = NAT_NEXT_ICMP_ERROR;
1274               goto trace0;
1275             }
1276
1277           udp0 = ip4_next_header (ip0);
1278           tcp0 = (tcp_header_t *) udp0;
1279           icmp0 = (icmp46_header_t *) udp0;
1280           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1281
1282           if (PREDICT_FALSE (proto0 == ~0))
1283             {
1284               s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1285                                                   rx_fib_index0,
1286                                                   thread_index, now,
1287                                                   vm, node);
1288               if (!s0)
1289                 next0 = NAT_NEXT_DROP;
1290
1291               other_packets++;
1292               goto trace0;
1293             }
1294
1295           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1296             {
1297               next0 = icmp_in2out_ed_slow_path
1298                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1299                  node, next0, now, thread_index, &s0);
1300               icmp_packets++;
1301               goto trace0;
1302             }
1303
1304           // move down
1305           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1306                       ip0->protocol, rx_fib_index0,
1307                       vnet_buffer (b0)->ip.reass.l4_src_port,
1308                       vnet_buffer (b0)->ip.reass.l4_dst_port);
1309
1310           if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1311             {
1312               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1313
1314               if (s0->tcp_close_timestamp && now >= s0->tcp_close_timestamp)
1315                 {
1316                   nat_free_session_data (sm, s0, thread_index, 0);
1317                   nat44_ed_delete_session (sm, s0, thread_index, 1);
1318                   s0 = NULL;
1319                 }
1320             }
1321
1322           if (!s0)
1323             {
1324               if (is_output_feature)
1325                 {
1326                   if (PREDICT_FALSE
1327                       (nat44_ed_not_translate_output_feature
1328                        (sm, ip0, ip0->protocol,
1329                         vnet_buffer (b0)->ip.reass.l4_src_port,
1330                         vnet_buffer (b0)->ip.reass.l4_dst_port,
1331                         thread_index, sw_if_index0,
1332                         vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1333                     goto trace0;
1334
1335                   /*
1336                    * Send DHCP packets to the ipv4 stack, or we won't
1337                    * be able to use dhcp client on the outside interface
1338                    */
1339                   if (PREDICT_FALSE
1340                       (proto0 == SNAT_PROTOCOL_UDP
1341                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1342                            clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1343                        && ip0->dst_address.as_u32 == 0xffffffff))
1344                     goto trace0;
1345                 }
1346               else
1347                 {
1348                   if (PREDICT_FALSE
1349                       (nat44_ed_not_translate
1350                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1351                         thread_index)))
1352                     goto trace0;
1353                 }
1354
1355               next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1356                                     next0, thread_index, now);
1357
1358               if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1359                 goto trace0;
1360
1361               if (PREDICT_FALSE (!s0))
1362                 goto trace0;
1363
1364             }
1365
1366           b0->flags |= VNET_BUFFER_F_IS_NATED;
1367
1368           if (!is_output_feature)
1369             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1370
1371           old_addr0 = ip0->src_address.as_u32;
1372           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1373           sum0 = ip0->checksum;
1374           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1375                                  src_address);
1376           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1377             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1378                                    s0->ext_host_addr.as_u32, ip4_header_t,
1379                                    dst_address);
1380           ip0->checksum = ip_csum_fold (sum0);
1381
1382           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1383
1384           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1385             {
1386               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1387                 {
1388                   new_port0 = udp0->src_port = s0->out2in.port;
1389                   sum0 = tcp0->checksum;
1390                   sum0 =
1391                     ip_csum_update (sum0, old_addr0, new_addr0,
1392                                     ip4_header_t, dst_address);
1393                   sum0 =
1394                     ip_csum_update (sum0, old_port0, new_port0,
1395                                     ip4_header_t, length);
1396                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1397                     {
1398                       sum0 =
1399                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1400                                         s0->ext_host_addr.as_u32,
1401                                         ip4_header_t, dst_address);
1402                       sum0 =
1403                         ip_csum_update (sum0,
1404                                         vnet_buffer (b0)->ip.
1405                                         reass.l4_dst_port, s0->ext_host_port,
1406                                         ip4_header_t, length);
1407                       tcp0->dst_port = s0->ext_host_port;
1408                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1409                     }
1410                   mss_clamping (sm, tcp0, &sum0);
1411                   tcp0->checksum = ip_csum_fold (sum0);
1412                 }
1413               tcp_packets++;
1414               if (nat44_set_tcp_session_state_i2o
1415                   (sm, now, s0, b0, thread_index))
1416                 goto trace0;
1417             }
1418           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1419                    && udp0->checksum)
1420             {
1421               new_port0 = udp0->src_port = s0->out2in.port;
1422               sum0 = udp0->checksum;
1423               sum0 =
1424                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1425                                 dst_address);
1426               sum0 =
1427                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1428                                 length);
1429               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1430                 {
1431                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1432                                          s0->ext_host_addr.as_u32,
1433                                          ip4_header_t, dst_address);
1434                   sum0 =
1435                     ip_csum_update (sum0,
1436                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1437                                     s0->ext_host_port, ip4_header_t, length);
1438                   udp0->dst_port = s0->ext_host_port;
1439                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1440                 }
1441               udp0->checksum = ip_csum_fold (sum0);
1442               udp_packets++;
1443             }
1444           else
1445             {
1446               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1447                 {
1448                   new_port0 = udp0->src_port = s0->out2in.port;
1449                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1450                     {
1451                       udp0->dst_port = s0->ext_host_port;
1452                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1453                     }
1454                   udp_packets++;
1455                 }
1456             }
1457
1458           /* Accounting */
1459           nat44_session_update_counters (s0, now,
1460                                          vlib_buffer_length_in_chain
1461                                          (vm, b0), thread_index);
1462           /* Per-user LRU list maintenance */
1463           nat44_session_update_lru (sm, s0, thread_index);
1464
1465         trace0:
1466           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1467                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1468             {
1469               nat_in2out_ed_trace_t *t =
1470                 vlib_add_trace (vm, node, b0, sizeof (*t));
1471               t->sw_if_index = sw_if_index0;
1472               t->next_index = next0;
1473               t->is_slow_path = 1;
1474
1475               if (s0)
1476                 t->session_index = s0 - tsm->sessions;
1477               else
1478                 t->session_index = ~0;
1479             }
1480
1481           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1482
1483           /* verify speculative enqueue, maybe switch current next frame */
1484           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1485                                            to_next, n_left_to_next,
1486                                            bi0, next0);
1487         }
1488
1489       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1490     }
1491
1492   vlib_node_increment_counter (vm, stats_node_index,
1493                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1494                                pkts_processed);
1495   vlib_node_increment_counter (vm, stats_node_index,
1496                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1497   vlib_node_increment_counter (vm, stats_node_index,
1498                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1499   vlib_node_increment_counter (vm, stats_node_index,
1500                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1501                                icmp_packets);
1502   vlib_node_increment_counter (vm, stats_node_index,
1503                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1504                                other_packets);
1505   return frame->n_vectors;
1506 }
1507
1508 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1509                                      vlib_node_runtime_t * node,
1510                                      vlib_frame_t * frame)
1511 {
1512   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1513 }
1514
1515 /* *INDENT-OFF* */
1516 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1517   .name = "nat44-ed-in2out",
1518   .vector_size = sizeof (u32),
1519   .sibling_of = "nat-default",
1520   .format_trace = format_nat_in2out_ed_trace,
1521   .type = VLIB_NODE_TYPE_INTERNAL,
1522   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1523   .error_strings = nat_in2out_ed_error_strings,
1524   .runtime_data_bytes = sizeof (snat_runtime_t),
1525 };
1526 /* *INDENT-ON* */
1527
1528 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1529                                             vlib_node_runtime_t * node,
1530                                             vlib_frame_t * frame)
1531 {
1532   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1533 }
1534
1535 /* *INDENT-OFF* */
1536 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1537   .name = "nat44-ed-in2out-output",
1538   .vector_size = sizeof (u32),
1539   .sibling_of = "nat-default",
1540   .format_trace = format_nat_in2out_ed_trace,
1541   .type = VLIB_NODE_TYPE_INTERNAL,
1542   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1543   .error_strings = nat_in2out_ed_error_strings,
1544   .runtime_data_bytes = sizeof (snat_runtime_t),
1545 };
1546 /* *INDENT-ON* */
1547
1548 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1549                                               vlib_node_runtime_t *
1550                                               node, vlib_frame_t * frame)
1551 {
1552   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1553 }
1554
1555 /* *INDENT-OFF* */
1556 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1557   .name = "nat44-ed-in2out-slowpath",
1558   .vector_size = sizeof (u32),
1559   .sibling_of = "nat-default",
1560   .format_trace = format_nat_in2out_ed_trace,
1561   .type = VLIB_NODE_TYPE_INTERNAL,
1562   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1563   .error_strings = nat_in2out_ed_error_strings,
1564   .runtime_data_bytes = sizeof (snat_runtime_t),
1565 };
1566 /* *INDENT-ON* */
1567
1568 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1569                                                      vlib_node_runtime_t
1570                                                      * node,
1571                                                      vlib_frame_t * frame)
1572 {
1573   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1574 }
1575
1576 /* *INDENT-OFF* */
1577 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1578   .name = "nat44-ed-in2out-output-slowpath",
1579   .vector_size = sizeof (u32),
1580   .sibling_of = "nat-default",
1581   .format_trace = format_nat_in2out_ed_trace,
1582   .type = VLIB_NODE_TYPE_INTERNAL,
1583   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1584   .error_strings = nat_in2out_ed_error_strings,
1585   .runtime_data_bytes = sizeof (snat_runtime_t),
1586 };
1587 /* *INDENT-ON* */
1588
1589 static u8 *
1590 format_nat_pre_trace (u8 * s, va_list * args)
1591 {
1592   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1593   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1594   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1595   return format (s, "in2out next_index %d", t->next_index);
1596 }
1597
1598 VLIB_NODE_FN (nat_pre_in2out_node)
1599   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1600 {
1601   return nat_pre_node_fn_inline (vm, node, frame,
1602                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1603 }
1604
1605 /* *INDENT-OFF* */
1606 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1607   .name = "nat-pre-in2out",
1608   .vector_size = sizeof (u32),
1609   .sibling_of = "nat-default",
1610   .format_trace = format_nat_pre_trace,
1611   .type = VLIB_NODE_TYPE_INTERNAL,
1612   .n_errors = 0,
1613 };
1614 /* *INDENT-ON* */
1615
1616 /*
1617  * fd.io coding-style-patch-verification: ON
1618  *
1619  * Local Variables:
1620  * eval: (c-set-style "gnu")
1621  * End:
1622  */