src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <nat/nat44/ed_inlines.h>
  35 #include <nat/lib/nat_inlines.h>
  36
  37 static char *nat_in2out_ed_error_strings[] = {
  38 #define _(sym,string) string,
  39   foreach_nat_in2out_ed_error
  40 #undef _
  41 };
  42
  43 typedef struct
  44 {
  45   u32 sw_if_index;
  46   u32 next_index;
  47   u32 session_index;
  48   u32 is_slow_path;
  49 } nat_in2out_ed_trace_t;
  50
  51 static u8 *
  52 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  53 {
  54   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  55   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  56   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  57   char *tag;
  58
  59   tag =
  60     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  61     "NAT44_IN2OUT_ED_FAST_PATH";
  62
  63   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  64               t->sw_if_index, t->next_index, t->session_index);
  65
  66   return s;
  67 }
  68
  69 #ifndef CLIB_MARCH_VARIANT
  70 int
  71 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  72 {
  73   snat_main_t *sm = &snat_main;
  74   nat44_is_idle_session_ctx_t *ctx = arg;
  75   snat_session_t *s;
  76   u64 sess_timeout_time;
  77   u8 proto;
  78   u16 r_port, l_port;
  79   ip4_address_t *l_addr, *r_addr;
  80   u32 fib_index;
  81   clib_bihash_kv_16_8_t ed_kv;
  82   int i;
  83   snat_address_t *a;
  84   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  85                                                        ctx->thread_index);
  86
  87   ASSERT (ctx->thread_index == ed_value_get_thread_index (kv));
  88   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (kv));
  89   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  90   if (ctx->now >= sess_timeout_time)
  91     {
  92       if (is_fwd_bypass_session (s))
  93         goto delete;
  94
  95       l_addr = &s->out2in.addr;
  96       r_addr = &s->ext_host_addr;
  97       fib_index = s->out2in.fib_index;
  98       if (snat_is_unk_proto_session (s))
  99         {
 100           proto = s->in2out.port;
 101           r_port = 0;
 102           l_port = 0;
 103         }
 104       else
 105         {
 106           proto = nat_proto_to_ip_proto (s->nat_proto);
 107           l_port = s->out2in.port;
 108           r_port = s->ext_host_port;
 109         }
 110       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 111       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 112         nat_elog_warn ("out2in_ed key del failed");
 113
 114       if (snat_is_unk_proto_session (s))
 115         goto delete;
 116
 117       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 118                                            s->in2out.addr.as_u32,
 119                                            s->out2in.addr.as_u32,
 120                                            s->nat_proto,
 121                                            s->in2out.port,
 122                                            s->out2in.port,
 123                                            s->in2out.fib_index);
 124
 125       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 126                              &s->in2out.addr, s->in2out.port,
 127                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 128                              &s->out2in.addr, s->out2in.port,
 129                              &s->ext_host_addr, s->ext_host_port,
 130                              s->nat_proto, is_twice_nat_session (s));
 131
 132       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 133                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 134                    ctx->thread_index);
 135
 136       if (is_twice_nat_session (s))
 137         {
 138           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 139             {
 140               // TODO FIXME this is obviously broken - which address should be
 141               // freed here?!
 142               a = sm->twice_nat_addresses + i;
 143               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 144                 {
 145                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 146                                                       ctx->thread_index,
 147                                                       &s->ext_host_nat_addr,
 148                                                       s->ext_host_nat_port,
 149                                                       s->nat_proto);
 150                   break;
 151                 }
 152             }
 153         }
 154
 155       if (snat_is_session_static (s))
 156         goto delete;
 157
 158       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 159                                           &s->out2in.addr, s->out2in.port,
 160                                           s->nat_proto);
 161     delete:
 162       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 163       return 1;
 164     }
 165
 166   return 0;
 167 }
 168 #endif
 169
 170 static inline u32
 171 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 172                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 173                           u32 sw_if_index0, u32 rx_fib_index0,
 174                           vlib_node_runtime_t * node, u32 next0, f64 now,
 175                           u32 thread_index, snat_session_t ** p_s0)
 176 {
 177   vlib_main_t *vm = vlib_get_main ();
 178
 179   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 180                        next0, thread_index, p_s0, 0);
 181   snat_session_t *s0 = *p_s0;
 182   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 183     {
 184       /* Accounting */
 185       nat44_session_update_counters (s0, now,
 186                                      vlib_buffer_length_in_chain
 187                                      (vm, b0), thread_index);
 188       /* Per-user LRU list maintenance */
 189       nat44_session_update_lru (sm, s0, thread_index);
 190     }
 191   return next0;
 192 }
 193
 194 static_always_inline u16
 195 snat_random_port (u16 min, u16 max)
 196 {
 197   snat_main_t *sm = &snat_main;
 198   return min + random_u32 (&sm->random_seed) /
 199     (random_u32_max () / (max - min + 1) + 1);
 200 }
 201
 202 static int
 203 nat_ed_alloc_addr_and_port (snat_main_t * sm, u32 rx_fib_index,
 204                             u32 nat_proto, u32 thread_index,
 205                             ip4_address_t r_addr, u16 r_port, u8 proto,
 206                             u16 port_per_thread, u32 snat_thread_index,
 207                             snat_session_t * s,
 208                             ip4_address_t * allocated_addr,
 209                             u16 * allocated_port,
 210                             clib_bihash_kv_16_8_t * out2in_ed_kv)
 211 {
 212   int i;
 213   snat_address_t *a, *ga = 0;
 214   u32 portnum;
 215   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 216
 217   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 218
 219   for (i = 0; i < vec_len (sm->addresses); i++)
 220     {
 221       a = sm->addresses + i;
 222       switch (nat_proto)
 223         {
 224 #define _(N, j, n, unused)                                                   \
 225   case NAT_PROTOCOL_##N:                                                     \
 226     if (a->fib_index == rx_fib_index)                                        \
 227       {                                                                      \
 228         u16 port = snat_random_port (1, port_per_thread);                    \
 229         u16 attempts = port_per_thread;                                      \
 230         while (attempts > 0)                                                 \
 231           {                                                                  \
 232             --attempts;                                                      \
 233             portnum = port_thread_offset + port;                             \
 234             init_ed_kv (out2in_ed_kv, a->addr,                               \
 235                         clib_host_to_net_u16 (portnum), r_addr, r_port,      \
 236                         s->out2in.fib_index, proto, thread_index,            \
 237                         s - tsm->sessions);                                  \
 238             int rv = clib_bihash_add_del_16_8 (&sm->out2in_ed, out2in_ed_kv, \
 239                                                2 /* is_add */);              \
 240             if (0 == rv)                                                     \
 241               {                                                              \
 242                 ++a->busy_##n##_port_refcounts[portnum];                     \
 243                 a->busy_##n##_ports_per_thread[thread_index]++;              \
 244                 a->busy_##n##_ports++;                                       \
 245                 *allocated_addr = a->addr;                                   \
 246                 *allocated_port = clib_host_to_net_u16 (portnum);            \
 247                 return 0;                                                    \
 248               }                                                              \
 249             port = (port + 1) % port_per_thread;                             \
 250           }                                                                  \
 251       }                                                                      \
 252     else if (a->fib_index == ~0)                                             \
 253       {                                                                      \
 254         ga = a;                                                              \
 255       }                                                                      \
 256     break;
 257
 258           foreach_nat_protocol;
 259         default:
 260           nat_elog_info ("unknown protocol");
 261           return 1;
 262         }
 263     }
 264
 265   if (ga)
 266     {
 267       /* fake fib_index to reuse macro */
 268       rx_fib_index = ~0;
 269       a = ga;
 270       switch (nat_proto)
 271         {
 272           foreach_nat_protocol;
 273         default:
 274           nat_elog_info ("unknown protocol");
 275           return 1;
 276         }
 277     }
 278
 279 #undef _
 280
 281   /* Totally out of translations to use... */
 282   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 283   return 1;
 284 }
 285
 286 static_always_inline u32
 287 nat_outside_fib_index_lookup (snat_main_t * sm, ip4_address_t addr)
 288 {
 289   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 290   nat_outside_fib_t *outside_fib;
 291   fib_prefix_t pfx = {
 292     .fp_proto = FIB_PROTOCOL_IP4,
 293     .fp_len = 32,
 294     .fp_addr = {.ip4.as_u32 = addr.as_u32,}
 295     ,
 296   };
 297   // TODO: multiple vrfs none can resolve addr
 298   /* *INDENT-OFF* */
 299   vec_foreach (outside_fib, sm->outside_fibs)
 300     {
 301       fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 302       if (FIB_NODE_INDEX_INVALID != fei)
 303         {
 304           if (fib_entry_get_resolving_interface (fei) != ~0)
 305             {
 306               return outside_fib->fib_index;
 307             }
 308         }
 309     }
 310   /* *INDENT-ON* */
 311   return ~0;
 312 }
 313
 314 static u32
 315 slow_path_ed (snat_main_t * sm,
 316               vlib_buffer_t * b,
 317               ip4_address_t l_addr,
 318               ip4_address_t r_addr,
 319               u16 l_port,
 320               u16 r_port,
 321               u8 proto,
 322               u32 rx_fib_index,
 323               snat_session_t ** sessionp,
 324               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 325 {
 326   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 327   clib_bihash_kv_16_8_t out2in_ed_kv;
 328   nat44_is_idle_session_ctx_t ctx;
 329   ip4_address_t allocated_addr;
 330   u16 allocated_port;
 331   u8 identity_nat;
 332
 333   u32 nat_proto = ip_proto_to_nat_proto (proto);
 334   snat_session_t *s = NULL;
 335   lb_nat_type_t lb = 0;
 336
 337   if (PREDICT_TRUE (nat_proto == NAT_PROTOCOL_TCP))
 338     {
 339       if (PREDICT_FALSE
 340           (!tcp_flags_is_init
 341            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 342         {
 343           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 344           return NAT_NEXT_DROP;
 345         }
 346     }
 347
 348   if (PREDICT_FALSE
 349       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 350     {
 351       if (!nat_lru_free_one (sm, thread_index, now))
 352         {
 353           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 354           nat_ipfix_logging_max_sessions (thread_index,
 355                                           sm->max_translations_per_thread);
 356           nat_elog_notice ("maximum sessions exceeded");
 357           return NAT_NEXT_DROP;
 358         }
 359     }
 360
 361   ip4_address_t sm_addr;
 362   u16 sm_port;
 363   u32 sm_fib_index;
 364   /* First try to match static mapping by local address and port */
 365   if (snat_static_mapping_match
 366       (sm, l_addr, l_port, rx_fib_index, nat_proto, &sm_addr, &sm_port,
 367        &sm_fib_index, 0, 0, 0, &lb, 0, &identity_nat))
 368     {
 369       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 370       if (!s)
 371         {
 372           nat_elog_warn ("create NAT session failed");
 373           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 374           return NAT_NEXT_DROP;
 375         }
 376       s->in2out.addr = l_addr;
 377       s->in2out.port = l_port;
 378       s->nat_proto = nat_proto;
 379       s->in2out.fib_index = rx_fib_index;
 380       s->out2in.fib_index = sm->outside_fib_index;
 381
 382       switch (vec_len (sm->outside_fibs))
 383         {
 384         case 0:
 385           s->out2in.fib_index = sm->outside_fib_index;
 386           break;
 387         case 1:
 388           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 389           break;
 390         default:
 391           s->out2in.fib_index = nat_outside_fib_index_lookup (sm, r_addr);
 392           break;
 393         }
 394
 395       /* Try to create dynamic translation */
 396       if (nat_ed_alloc_addr_and_port (sm, rx_fib_index, nat_proto,
 397                                       thread_index, r_addr, r_port, proto,
 398                                       sm->port_per_thread,
 399                                       tsm->snat_thread_index, s,
 400                                       &allocated_addr,
 401                                       &allocated_port, &out2in_ed_kv))
 402         {
 403           nat_elog_notice ("addresses exhausted");
 404           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 405           nat_ed_session_delete (sm, s, thread_index, 1);
 406           return NAT_NEXT_DROP;
 407         }
 408       s->out2in.addr = allocated_addr;
 409       s->out2in.port = allocated_port;
 410     }
 411   else
 412     {
 413       if (PREDICT_FALSE (identity_nat))
 414         {
 415           *sessionp = NULL;
 416           return next;
 417         }
 418       s = nat_ed_session_alloc (sm, thread_index, now, proto);
 419       if (!s)
 420         {
 421           nat_elog_warn ("create NAT session failed");
 422           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 423           return NAT_NEXT_DROP;
 424         }
 425       s->out2in.addr = sm_addr;
 426       s->out2in.port = sm_port;
 427       s->in2out.addr = l_addr;
 428       s->in2out.port = l_port;
 429       s->nat_proto = nat_proto;
 430       s->in2out.fib_index = rx_fib_index;
 431       s->out2in.fib_index = sm->outside_fib_index;
 432       switch (vec_len (sm->outside_fibs))
 433         {
 434         case 0:
 435           s->out2in.fib_index = sm->outside_fib_index;
 436           break;
 437         case 1:
 438           s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 439           break;
 440         default:
 441           s->out2in.fib_index = nat_outside_fib_index_lookup (sm, r_addr);
 442           break;
 443         }
 444
 445       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 446
 447       init_ed_kv (&out2in_ed_kv, sm_addr, sm_port, r_addr, r_port,
 448                   s->out2in.fib_index, proto, thread_index,
 449                   s - tsm->sessions);
 450       if (clib_bihash_add_or_overwrite_stale_16_8
 451           (&sm->out2in_ed, &out2in_ed_kv, nat44_o2i_ed_is_idle_session_cb,
 452            &ctx))
 453         nat_elog_notice ("out2in-ed key add failed");
 454     }
 455
 456   if (lb)
 457     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 458   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 459   s->ext_host_addr = r_addr;
 460   s->ext_host_port = r_port;
 461
 462   clib_bihash_kv_16_8_t in2out_ed_kv;
 463   init_ed_kv (&in2out_ed_kv, l_addr, l_port, r_addr, r_port, rx_fib_index,
 464               proto, thread_index, s - tsm->sessions);
 465   ctx.now = now;
 466   ctx.thread_index = thread_index;
 467   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &in2out_ed_kv,
 468                                                nat44_i2o_ed_is_idle_session_cb,
 469                                                &ctx))
 470     nat_elog_notice ("in2out-ed key add failed");
 471
 472   *sessionp = s;
 473
 474   /* log NAT event */
 475   snat_ipfix_logging_nat44_ses_create (thread_index,
 476                                        s->in2out.addr.as_u32,
 477                                        s->out2in.addr.as_u32,
 478                                        s->nat_proto,
 479                                        s->in2out.port,
 480                                        s->out2in.port, s->in2out.fib_index);
 481
 482   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 483                          &s->in2out.addr, s->in2out.port,
 484                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 485                          &s->out2in.addr, s->out2in.port,
 486                          &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 487                          0);
 488
 489   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 490                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 491                &s->ext_host_nat_addr, s->ext_host_nat_port,
 492                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 493
 494   return next;
 495 }
 496
 497 static_always_inline int
 498 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 499                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 500                         u32 rx_fib_index, u32 thread_index)
 501 {
 502   udp_header_t *udp = ip4_next_header (ip);
 503   clib_bihash_kv_16_8_t kv, value;
 504
 505   init_ed_k (&kv, ip->dst_address, udp->dst_port, ip->src_address,
 506              udp->src_port, sm->outside_fib_index, ip->protocol);
 507
 508   /* NAT packet aimed at external address if has active sessions */
 509   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 510     {
 511       /* or is static mappings */
 512       ip4_address_t dummy_addr;
 513       u16 dummy_port;
 514       u32 dummy_fib_index;
 515       if (!snat_static_mapping_match
 516           (sm, ip->dst_address, udp->dst_port, sm->outside_fib_index, proto,
 517            &dummy_addr, &dummy_port, &dummy_fib_index, 1, 0, 0, 0, 0, 0))
 518         return 0;
 519     }
 520   else
 521     return 0;
 522
 523   if (sm->forwarding_enabled)
 524     return 1;
 525
 526   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 527                                   rx_fib_index);
 528 }
 529
 530 static_always_inline int
 531 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 532                                       u32 thread_index, f64 now,
 533                                       vlib_main_t * vm, vlib_buffer_t * b)
 534 {
 535   clib_bihash_kv_16_8_t kv, value;
 536   snat_session_t *s = 0;
 537   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 538
 539   if (!sm->forwarding_enabled)
 540     return 0;
 541
 542   if (ip->protocol == IP_PROTOCOL_ICMP)
 543     {
 544       if (get_icmp_i2o_ed_key (b, ip, 0, ~0, ~0, 0, 0, 0, &kv))
 545         return 0;
 546     }
 547   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 548     {
 549       init_ed_k (&kv, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
 550                  ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port, 0,
 551                  ip->protocol);
 552     }
 553   else
 554     {
 555       init_ed_k (&kv, ip->src_address, 0, ip->dst_address, 0, 0,
 556                  ip->protocol);
 557     }
 558
 559   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 560     {
 561       ASSERT (thread_index == ed_value_get_thread_index (&value));
 562       s =
 563         pool_elt_at_index (tsm->sessions,
 564                            ed_value_get_session_index (&value));
 565       if (is_fwd_bypass_session (s))
 566         {
 567           if (ip->protocol == IP_PROTOCOL_TCP)
 568             {
 569               if (nat44_set_tcp_session_state_i2o
 570                   (sm, now, s, b, thread_index))
 571                 return 1;
 572             }
 573           /* Accounting */
 574           nat44_session_update_counters (s, now,
 575                                          vlib_buffer_length_in_chain (vm, b),
 576                                          thread_index);
 577           /* Per-user LRU list maintenance */
 578           nat44_session_update_lru (sm, s, thread_index);
 579           return 1;
 580         }
 581       else
 582         return 0;
 583     }
 584
 585   return 0;
 586 }
 587
 588 static_always_inline int
 589 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 590                                        u16 src_port, u16 dst_port,
 591                                        u32 thread_index, u32 rx_sw_if_index,
 592                                        u32 tx_sw_if_index)
 593 {
 594   clib_bihash_kv_16_8_t kv, value;
 595   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 596   snat_interface_t *i;
 597   snat_session_t *s;
 598   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 599   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 600
 601   /* src NAT check */
 602   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 603              tx_fib_index, ip->protocol);
 604   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 605     {
 606       ASSERT (thread_index == ed_value_get_thread_index (&value));
 607       s =
 608         pool_elt_at_index (tsm->sessions,
 609                            ed_value_get_session_index (&value));
 610       if (nat44_is_ses_closed (s))
 611         {
 612           nat_free_session_data (sm, s, thread_index, 0);
 613           nat_ed_session_delete (sm, s, thread_index, 1);
 614         }
 615       else
 616         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 617       return 1;
 618     }
 619
 620   /* dst NAT check */
 621   init_ed_k (&kv, ip->dst_address, dst_port, ip->src_address, src_port,
 622              rx_fib_index, ip->protocol);
 623   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 624     {
 625       ASSERT (thread_index == ed_value_get_thread_index (&value));
 626       s =
 627         pool_elt_at_index (tsm->sessions,
 628                            ed_value_get_session_index (&value));
 629       if (is_fwd_bypass_session (s))
 630         return 0;
 631
 632       /* hairpinning */
 633       /* *INDENT-OFF* */
 634       pool_foreach (i, sm->output_feature_interfaces,
 635       ({
 636         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 637            return 0;
 638       }));
 639       /* *INDENT-ON* */
 640       return 1;
 641     }
 642
 643   return 0;
 644 }
 645
 646 #ifndef CLIB_MARCH_VARIANT
 647 u32
 648 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 649                       u32 thread_index, vlib_buffer_t * b,
 650                       ip4_header_t * ip, ip4_address_t * addr,
 651                       u16 * port, u32 * fib_index, nat_protocol_t * proto,
 652                       void *d, void *e, u8 * dont_translate)
 653 {
 654   u32 sw_if_index;
 655   u32 rx_fib_index;
 656   clib_bihash_kv_16_8_t kv, value;
 657   u32 next = ~0;
 658   int err;
 659   snat_session_t *s = NULL;
 660   u16 l_port = 0, r_port = 0;   // initialize to workaround gcc warning
 661   vlib_main_t *vm = vlib_get_main ();
 662   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 663   *dont_translate = 0;
 664
 665   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 666   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 667
 668   err =
 669     get_icmp_i2o_ed_key (b, ip, rx_fib_index, ~0, ~0, proto, &l_port,
 670                          &r_port, &kv);
 671   if (err != 0)
 672     {
 673       b->error = node->errors[err];
 674       next = NAT_NEXT_DROP;
 675       goto out;
 676     }
 677
 678   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 679     {
 680       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 681         {
 682           if (PREDICT_FALSE
 683               (nat44_ed_not_translate_output_feature
 684                (sm, ip, l_port, r_port, thread_index,
 685                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX])))
 686             {
 687               *dont_translate = 1;
 688               goto out;
 689             }
 690         }
 691       else
 692         {
 693           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 694                                                      ip, NAT_PROTOCOL_ICMP,
 695                                                      rx_fib_index,
 696                                                      thread_index)))
 697             {
 698               *dont_translate = 1;
 699               goto out;
 700             }
 701         }
 702
 703       if (PREDICT_FALSE
 704           (icmp_type_is_error_message
 705            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 706         {
 707           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 708           next = NAT_NEXT_DROP;
 709           goto out;
 710         }
 711
 712       next =
 713         slow_path_ed (sm, b, ip->src_address, ip->dst_address, l_port, r_port,
 714                       ip->protocol, rx_fib_index, &s, node, next,
 715                       thread_index, vlib_time_now (vm));
 716
 717       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 718         goto out;
 719
 720       if (!s)
 721         {
 722           *dont_translate = 1;
 723           goto out;
 724         }
 725     }
 726   else
 727     {
 728       if (PREDICT_FALSE
 729           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 730            ICMP4_echo_request
 731            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 732            ICMP4_echo_reply
 733            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 734                                            reass.icmp_type_or_tcp_flags)))
 735         {
 736           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 737           next = NAT_NEXT_DROP;
 738           goto out;
 739         }
 740
 741       ASSERT (thread_index == ed_value_get_thread_index (&value));
 742       s =
 743         pool_elt_at_index (tsm->sessions,
 744                            ed_value_get_session_index (&value));
 745     }
 746 out:
 747   if (s)
 748     {
 749       *addr = s->out2in.addr;
 750       *port = s->out2in.port;
 751       *fib_index = s->out2in.fib_index;
 752     }
 753   if (d)
 754     {
 755       *(snat_session_t **) d = s;
 756     }
 757   return next;
 758 }
 759 #endif
 760
 761 static snat_session_t *
 762 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 763                                vlib_buffer_t * b,
 764                                ip4_header_t * ip,
 765                                u32 rx_fib_index,
 766                                u32 thread_index,
 767                                f64 now,
 768                                vlib_main_t * vm, vlib_node_runtime_t * node)
 769 {
 770   clib_bihash_kv_8_8_t kv, value;
 771   clib_bihash_kv_16_8_t s_kv, s_value;
 772   snat_static_mapping_t *m;
 773   u32 old_addr, new_addr = 0;
 774   ip_csum_t sum;
 775   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 776   snat_session_t *s;
 777   u32 outside_fib_index = sm->outside_fib_index;
 778   int i;
 779   u8 is_sm = 0;
 780
 781   switch (vec_len (sm->outside_fibs))
 782     {
 783     case 0:
 784       outside_fib_index = sm->outside_fib_index;
 785       break;
 786     case 1:
 787       outside_fib_index = sm->outside_fibs[0].fib_index;
 788       break;
 789     default:
 790       outside_fib_index = nat_outside_fib_index_lookup (sm, ip->dst_address);
 791       break;
 792     }
 793   old_addr = ip->src_address.as_u32;
 794
 795   init_ed_k (&s_kv, ip->src_address, 0, ip->dst_address, 0, rx_fib_index,
 796              ip->protocol);
 797
 798   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 799     {
 800       ASSERT (thread_index == ed_value_get_thread_index (&s_value));
 801       s =
 802         pool_elt_at_index (tsm->sessions,
 803                            ed_value_get_session_index (&s_value));
 804       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 805     }
 806   else
 807     {
 808       if (PREDICT_FALSE
 809           (nat44_ed_maximum_sessions_exceeded
 810            (sm, rx_fib_index, thread_index)))
 811         {
 812           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 813           nat_ipfix_logging_max_sessions (thread_index,
 814                                           sm->max_translations_per_thread);
 815           nat_elog_notice ("maximum sessions exceeded");
 816           return 0;
 817         }
 818
 819       init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 820
 821       /* Try to find static mapping first */
 822       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 823         {
 824           m = pool_elt_at_index (sm->static_mappings, value.value);
 825           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 826           is_sm = 1;
 827           goto create_ses;
 828         }
 829       else
 830         {
 831           /* *INDENT-OFF* */
 832           pool_foreach (s, tsm->sessions, {
 833             if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 834               {
 835                 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 836
 837                 init_ed_k(&s_kv, s->out2in.addr, 0, ip->dst_address, 0, outside_fib_index, ip->protocol);
 838                 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 839                   goto create_ses;
 840
 841                 break;
 842               }
 843           });
 844           /* *INDENT-ON* */
 845
 846           for (i = 0; i < vec_len (sm->addresses); i++)
 847             {
 848               init_ed_k (&s_kv, sm->addresses[i].addr, 0, ip->dst_address, 0,
 849                          outside_fib_index, ip->protocol);
 850               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 851                 {
 852                   new_addr = ip->src_address.as_u32 =
 853                     sm->addresses[i].addr.as_u32;
 854                   goto create_ses;
 855                 }
 856             }
 857           return 0;
 858         }
 859
 860     create_ses:
 861       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 862       if (!s)
 863         {
 864           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 865           nat_elog_warn ("create NAT session failed");
 866           return 0;
 867         }
 868
 869       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 870       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 871       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 872       s->out2in.addr.as_u32 = new_addr;
 873       s->out2in.fib_index = outside_fib_index;
 874       s->in2out.addr.as_u32 = old_addr;
 875       s->in2out.fib_index = rx_fib_index;
 876       s->in2out.port = s->out2in.port = ip->protocol;
 877       if (is_sm)
 878         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 879
 880       /* Add to lookup tables */
 881       init_ed_kv (&s_kv, s->in2out.addr, 0, ip->dst_address, 0, rx_fib_index,
 882                   ip->protocol, thread_index, s - tsm->sessions);
 883       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 884         nat_elog_notice ("in2out key add failed");
 885
 886       init_ed_kv (&s_kv, s->out2in.addr, 0, ip->dst_address, 0,
 887                   outside_fib_index, ip->protocol, thread_index,
 888                   s - tsm->sessions);
 889       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 890         nat_elog_notice ("out2in key add failed");
 891     }
 892
 893   /* Update IP checksum */
 894   sum = ip->checksum;
 895   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 896   ip->checksum = ip_csum_fold (sum);
 897
 898   /* Accounting */
 899   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 900                                  thread_index);
 901   /* Per-user LRU list maintenance */
 902   nat44_session_update_lru (sm, s, thread_index);
 903
 904   /* Hairpinning */
 905   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 906     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 907
 908   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 909     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 910
 911   return s;
 912 }
 913
 914 static inline uword
 915 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 916                                           vlib_node_runtime_t * node,
 917                                           vlib_frame_t * frame,
 918                                           int is_output_feature)
 919 {
 920   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 921   nat_next_t next_index;
 922   snat_main_t *sm = &snat_main;
 923   f64 now = vlib_time_now (vm);
 924   u32 thread_index = vm->thread_index;
 925   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 926   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 927     0, def_slow;
 928
 929   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 930     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 931
 932   stats_node_index = sm->ed_in2out_node_index;
 933
 934   from = vlib_frame_vector_args (frame);
 935   n_left_from = frame->n_vectors;
 936   next_index = node->cached_next_index;
 937
 938   while (n_left_from > 0)
 939     {
 940       u32 n_left_to_next;
 941
 942       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 943
 944       while (n_left_from > 0 && n_left_to_next > 0)
 945         {
 946           u32 bi0;
 947           vlib_buffer_t *b0;
 948           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 949             new_addr0, old_addr0;
 950           u16 old_port0, new_port0;
 951           ip4_header_t *ip0;
 952           udp_header_t *udp0;
 953           tcp_header_t *tcp0;
 954           snat_session_t *s0 = 0;
 955           clib_bihash_kv_16_8_t kv0, value0;
 956           ip_csum_t sum0;
 957
 958           /* speculatively enqueue b0 to the current next frame */
 959           bi0 = from[0];
 960           to_next[0] = bi0;
 961           from += 1;
 962           to_next += 1;
 963           n_left_from -= 1;
 964           n_left_to_next -= 1;
 965
 966           b0 = vlib_get_buffer (vm, bi0);
 967
 968           if (is_output_feature)
 969             {
 970               vnet_feature_next (&vnet_buffer2 (b0)->nat.arc_next, b0);
 971               iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 972             }
 973
 974           next0 = vnet_buffer2 (b0)->nat.arc_next;
 975
 976           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 977                                   iph_offset0);
 978
 979           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 980           rx_fib_index0 =
 981             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 982                                                  sw_if_index0);
 983
 984           if (PREDICT_FALSE (ip0->ttl == 1))
 985             {
 986               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 987               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 988                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 989                                            0);
 990               next0 = NAT_NEXT_ICMP_ERROR;
 991               goto trace0;
 992             }
 993
 994           udp0 = ip4_next_header (ip0);
 995           tcp0 = (tcp_header_t *) udp0;
 996           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 997
 998           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 999             {
1000               next0 = def_slow;
1001               goto trace0;
1002             }
1003
1004           if (is_output_feature)
1005             {
1006               if (PREDICT_FALSE (nat_not_translate_output_feature_fwd
1007                                  (sm, ip0, thread_index, now, vm, b0)))
1008                 goto trace0;
1009             }
1010
1011           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1012             {
1013               next0 = def_slow;
1014               goto trace0;
1015             }
1016
1017           init_ed_k (&kv0, ip0->src_address,
1018                      vnet_buffer (b0)->ip.reass.l4_src_port, ip0->dst_address,
1019                      vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1020                      ip0->protocol);
1021
1022           // lookup for session
1023           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1024             {
1025               // session does not exist go slow path
1026               next0 = def_slow;
1027               goto trace0;
1028             }
1029           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1030           s0 =
1031             pool_elt_at_index (tsm->sessions,
1032                                ed_value_get_session_index (&value0));
1033
1034           if (s0->tcp_closed_timestamp)
1035             {
1036               if (now >= s0->tcp_closed_timestamp)
1037                 {
1038                   // session is closed, go slow path
1039                   next0 = def_slow;
1040                 }
1041               else
1042                 {
1043                   // session in transitory timeout, drop
1044                   b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1045                   next0 = NAT_NEXT_DROP;
1046                 }
1047               goto trace0;
1048             }
1049
1050           // drop if session expired
1051           u64 sess_timeout_time;
1052           sess_timeout_time = s0->last_heard +
1053             (f64) nat44_session_get_timeout (sm, s0);
1054           if (now >= sess_timeout_time)
1055             {
1056               nat_free_session_data (sm, s0, thread_index, 0);
1057               nat_ed_session_delete (sm, s0, thread_index, 1);
1058               // session is closed, go slow path
1059               next0 = def_slow;
1060               goto trace0;
1061             }
1062
1063           b0->flags |= VNET_BUFFER_F_IS_NATED;
1064
1065           if (!is_output_feature)
1066             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1067
1068           old_addr0 = ip0->src_address.as_u32;
1069           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1070           sum0 = ip0->checksum;
1071           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1072                                  src_address);
1073           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1074             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1075                                    s0->ext_host_addr.as_u32, ip4_header_t,
1076                                    dst_address);
1077           ip0->checksum = ip_csum_fold (sum0);
1078
1079           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1080
1081           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1082             {
1083               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1084                 {
1085                   new_port0 = udp0->src_port = s0->out2in.port;
1086                   sum0 = tcp0->checksum;
1087                   sum0 =
1088                     ip_csum_update (sum0, old_addr0, new_addr0,
1089                                     ip4_header_t, dst_address);
1090                   sum0 =
1091                     ip_csum_update (sum0, old_port0, new_port0,
1092                                     ip4_header_t, length);
1093                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1094                     {
1095                       sum0 =
1096                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1097                                         s0->ext_host_addr.as_u32,
1098                                         ip4_header_t, dst_address);
1099                       sum0 =
1100                         ip_csum_update (sum0,
1101                                         vnet_buffer (b0)->ip.
1102                                         reass.l4_dst_port, s0->ext_host_port,
1103                                         ip4_header_t, length);
1104                       tcp0->dst_port = s0->ext_host_port;
1105                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1106                     }
1107                   mss_clamping (sm->mss_clamping, tcp0, &sum0);
1108                   tcp0->checksum = ip_csum_fold (sum0);
1109                 }
1110               tcp_packets++;
1111               if (nat44_set_tcp_session_state_i2o
1112                   (sm, now, s0, b0, thread_index))
1113                 goto trace0;
1114             }
1115           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1116                    && udp0->checksum)
1117             {
1118               new_port0 = udp0->src_port = s0->out2in.port;
1119               sum0 = udp0->checksum;
1120               sum0 =
1121                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1122                                 dst_address);
1123               sum0 =
1124                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1125                                 length);
1126               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1127                 {
1128                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1129                                          s0->ext_host_addr.as_u32,
1130                                          ip4_header_t, dst_address);
1131                   sum0 =
1132                     ip_csum_update (sum0,
1133                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1134                                     s0->ext_host_port, ip4_header_t, length);
1135                   udp0->dst_port = s0->ext_host_port;
1136                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1137                 }
1138               udp0->checksum = ip_csum_fold (sum0);
1139               udp_packets++;
1140             }
1141           else
1142             {
1143               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1144                 {
1145                   new_port0 = udp0->src_port = s0->out2in.port;
1146                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1147                     {
1148                       udp0->dst_port = s0->ext_host_port;
1149                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1150                     }
1151                   udp_packets++;
1152                 }
1153             }
1154
1155           /* Accounting */
1156           nat44_session_update_counters (s0, now,
1157                                          vlib_buffer_length_in_chain
1158                                          (vm, b0), thread_index);
1159           /* Per-user LRU list maintenance */
1160           nat44_session_update_lru (sm, s0, thread_index);
1161
1162         trace0:
1163           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1164                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1165             {
1166               nat_in2out_ed_trace_t *t =
1167                 vlib_add_trace (vm, node, b0, sizeof (*t));
1168               t->sw_if_index = sw_if_index0;
1169               t->next_index = next0;
1170               t->is_slow_path = 0;
1171
1172               if (s0)
1173                 t->session_index = s0 - tsm->sessions;
1174               else
1175                 t->session_index = ~0;
1176             }
1177
1178           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1179           /* verify speculative enqueue, maybe switch current next frame */
1180           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1181                                            to_next, n_left_to_next,
1182                                            bi0, next0);
1183         }
1184
1185       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1186     }
1187
1188   vlib_node_increment_counter (vm, stats_node_index,
1189                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1190                                pkts_processed);
1191   vlib_node_increment_counter (vm, stats_node_index,
1192                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1193   vlib_node_increment_counter (vm, stats_node_index,
1194                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1195   vlib_node_increment_counter (vm, stats_node_index,
1196                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1197                                icmp_packets);
1198   vlib_node_increment_counter (vm, stats_node_index,
1199                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1200                                other_packets);
1201   return frame->n_vectors;
1202 }
1203
1204 static inline uword
1205 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1206                                           vlib_node_runtime_t * node,
1207                                           vlib_frame_t * frame,
1208                                           int is_output_feature)
1209 {
1210   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
1211   nat_next_t next_index;
1212   snat_main_t *sm = &snat_main;
1213   f64 now = vlib_time_now (vm);
1214   u32 thread_index = vm->thread_index;
1215   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1216   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = 0;
1217
1218   stats_node_index = sm->ed_in2out_slowpath_node_index;
1219
1220   from = vlib_frame_vector_args (frame);
1221   n_left_from = frame->n_vectors;
1222   next_index = node->cached_next_index;
1223
1224   while (n_left_from > 0)
1225     {
1226       u32 n_left_to_next;
1227
1228       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1229
1230       while (n_left_from > 0 && n_left_to_next > 0)
1231         {
1232           u32 bi0;
1233           vlib_buffer_t *b0;
1234           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1235             new_addr0, old_addr0;
1236           u16 old_port0, new_port0;
1237           ip4_header_t *ip0;
1238           udp_header_t *udp0;
1239           tcp_header_t *tcp0;
1240           icmp46_header_t *icmp0;
1241           snat_session_t *s0 = 0;
1242           clib_bihash_kv_16_8_t kv0, value0;
1243           ip_csum_t sum0;
1244
1245           /* speculatively enqueue b0 to the current next frame */
1246           bi0 = from[0];
1247           to_next[0] = bi0;
1248           from += 1;
1249           to_next += 1;
1250           n_left_from -= 1;
1251           n_left_to_next -= 1;
1252
1253           b0 = vlib_get_buffer (vm, bi0);
1254
1255           if (is_output_feature)
1256             iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1257
1258           next0 = vnet_buffer2 (b0)->nat.arc_next;
1259
1260           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1261                                   iph_offset0);
1262
1263           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1264           rx_fib_index0 =
1265             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1266                                                  sw_if_index0);
1267
1268           if (PREDICT_FALSE (ip0->ttl == 1))
1269             {
1270               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1271               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1272                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1273                                            0);
1274               next0 = NAT_NEXT_ICMP_ERROR;
1275               goto trace0;
1276             }
1277
1278           udp0 = ip4_next_header (ip0);
1279           tcp0 = (tcp_header_t *) udp0;
1280           icmp0 = (icmp46_header_t *) udp0;
1281           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1282
1283           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1284             {
1285               s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1286                                                   rx_fib_index0,
1287                                                   thread_index, now,
1288                                                   vm, node);
1289               if (!s0)
1290                 next0 = NAT_NEXT_DROP;
1291
1292               other_packets++;
1293               goto trace0;
1294             }
1295
1296           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1297             {
1298               next0 = icmp_in2out_ed_slow_path
1299                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1300                  node, next0, now, thread_index, &s0);
1301               icmp_packets++;
1302               goto trace0;
1303             }
1304
1305           init_ed_k (&kv0, ip0->src_address,
1306                      vnet_buffer (b0)->ip.reass.l4_src_port, ip0->dst_address,
1307                      vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1308                      ip0->protocol);
1309           if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1310             {
1311               ASSERT (thread_index == ed_value_get_thread_index (&value0));
1312               s0 =
1313                 pool_elt_at_index (tsm->sessions,
1314                                    ed_value_get_session_index (&value0));
1315
1316               if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1317                 {
1318                   nat_free_session_data (sm, s0, thread_index, 0);
1319                   nat_ed_session_delete (sm, s0, thread_index, 1);
1320                   s0 = NULL;
1321                 }
1322             }
1323
1324           if (!s0)
1325             {
1326               if (is_output_feature)
1327                 {
1328                   if (PREDICT_FALSE
1329                       (nat44_ed_not_translate_output_feature
1330                        (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1331                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1332                         sw_if_index0,
1333                         vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1334                     goto trace0;
1335
1336                   /*
1337                    * Send DHCP packets to the ipv4 stack, or we won't
1338                    * be able to use dhcp client on the outside interface
1339                    */
1340                   if (PREDICT_FALSE
1341                       (proto0 == NAT_PROTOCOL_UDP
1342                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1343                            clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1344                        && ip0->dst_address.as_u32 == 0xffffffff))
1345                     goto trace0;
1346                 }
1347               else
1348                 {
1349                   if (PREDICT_FALSE
1350                       (nat44_ed_not_translate
1351                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1352                         thread_index)))
1353                     goto trace0;
1354                 }
1355
1356               next0 =
1357                 slow_path_ed (sm, b0, ip0->src_address, ip0->dst_address,
1358                               vnet_buffer (b0)->ip.reass.l4_src_port,
1359                               vnet_buffer (b0)->ip.reass.l4_dst_port,
1360                               ip0->protocol, rx_fib_index0, &s0, node, next0,
1361                               thread_index, now);
1362
1363               if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1364                 goto trace0;
1365
1366               if (PREDICT_FALSE (!s0))
1367                 goto trace0;
1368
1369             }
1370
1371           b0->flags |= VNET_BUFFER_F_IS_NATED;
1372
1373           if (!is_output_feature)
1374             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1375
1376           old_addr0 = ip0->src_address.as_u32;
1377           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1378           sum0 = ip0->checksum;
1379           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1380                                  src_address);
1381           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1382             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1383                                    s0->ext_host_addr.as_u32, ip4_header_t,
1384                                    dst_address);
1385           ip0->checksum = ip_csum_fold (sum0);
1386
1387           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1388
1389           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1390             {
1391               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1392                 {
1393                   new_port0 = udp0->src_port = s0->out2in.port;
1394                   sum0 = tcp0->checksum;
1395                   sum0 =
1396                     ip_csum_update (sum0, old_addr0, new_addr0,
1397                                     ip4_header_t, dst_address);
1398                   sum0 =
1399                     ip_csum_update (sum0, old_port0, new_port0,
1400                                     ip4_header_t, length);
1401                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1402                     {
1403                       sum0 =
1404                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1405                                         s0->ext_host_addr.as_u32,
1406                                         ip4_header_t, dst_address);
1407                       sum0 =
1408                         ip_csum_update (sum0,
1409                                         vnet_buffer (b0)->ip.
1410                                         reass.l4_dst_port, s0->ext_host_port,
1411                                         ip4_header_t, length);
1412                       tcp0->dst_port = s0->ext_host_port;
1413                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1414                     }
1415                   mss_clamping (sm->mss_clamping, tcp0, &sum0);
1416                   tcp0->checksum = ip_csum_fold (sum0);
1417                 }
1418               tcp_packets++;
1419               if (nat44_set_tcp_session_state_i2o
1420                   (sm, now, s0, b0, thread_index))
1421                 goto trace0;
1422             }
1423           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1424                    && udp0->checksum)
1425             {
1426               new_port0 = udp0->src_port = s0->out2in.port;
1427               sum0 = udp0->checksum;
1428               sum0 =
1429                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1430                                 dst_address);
1431               sum0 =
1432                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1433                                 length);
1434               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1435                 {
1436                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1437                                          s0->ext_host_addr.as_u32,
1438                                          ip4_header_t, dst_address);
1439                   sum0 =
1440                     ip_csum_update (sum0,
1441                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1442                                     s0->ext_host_port, ip4_header_t, length);
1443                   udp0->dst_port = s0->ext_host_port;
1444                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1445                 }
1446               udp0->checksum = ip_csum_fold (sum0);
1447               udp_packets++;
1448             }
1449           else
1450             {
1451               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1452                 {
1453                   new_port0 = udp0->src_port = s0->out2in.port;
1454                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1455                     {
1456                       udp0->dst_port = s0->ext_host_port;
1457                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1458                     }
1459                   udp_packets++;
1460                 }
1461             }
1462
1463           /* Accounting */
1464           nat44_session_update_counters (s0, now,
1465                                          vlib_buffer_length_in_chain
1466                                          (vm, b0), thread_index);
1467           /* Per-user LRU list maintenance */
1468           nat44_session_update_lru (sm, s0, thread_index);
1469
1470         trace0:
1471           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1472                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1473             {
1474               nat_in2out_ed_trace_t *t =
1475                 vlib_add_trace (vm, node, b0, sizeof (*t));
1476               t->sw_if_index = sw_if_index0;
1477               t->next_index = next0;
1478               t->is_slow_path = 1;
1479
1480               if (s0)
1481                 t->session_index = s0 - tsm->sessions;
1482               else
1483                 t->session_index = ~0;
1484             }
1485
1486           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1487
1488           /* verify speculative enqueue, maybe switch current next frame */
1489           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1490                                            to_next, n_left_to_next,
1491                                            bi0, next0);
1492         }
1493
1494       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1495     }
1496
1497   vlib_node_increment_counter (vm, stats_node_index,
1498                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1499                                pkts_processed);
1500   vlib_node_increment_counter (vm, stats_node_index,
1501                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1502   vlib_node_increment_counter (vm, stats_node_index,
1503                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1504   vlib_node_increment_counter (vm, stats_node_index,
1505                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1506                                icmp_packets);
1507   vlib_node_increment_counter (vm, stats_node_index,
1508                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1509                                other_packets);
1510   return frame->n_vectors;
1511 }
1512
1513 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1514                                      vlib_node_runtime_t * node,
1515                                      vlib_frame_t * frame)
1516 {
1517   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1518 }
1519
1520 /* *INDENT-OFF* */
1521 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1522   .name = "nat44-ed-in2out",
1523   .vector_size = sizeof (u32),
1524   .sibling_of = "nat-default",
1525   .format_trace = format_nat_in2out_ed_trace,
1526   .type = VLIB_NODE_TYPE_INTERNAL,
1527   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1528   .error_strings = nat_in2out_ed_error_strings,
1529   .runtime_data_bytes = sizeof (snat_runtime_t),
1530 };
1531 /* *INDENT-ON* */
1532
1533 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1534                                             vlib_node_runtime_t * node,
1535                                             vlib_frame_t * frame)
1536 {
1537   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1538 }
1539
1540 /* *INDENT-OFF* */
1541 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1542   .name = "nat44-ed-in2out-output",
1543   .vector_size = sizeof (u32),
1544   .sibling_of = "nat-default",
1545   .format_trace = format_nat_in2out_ed_trace,
1546   .type = VLIB_NODE_TYPE_INTERNAL,
1547   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1548   .error_strings = nat_in2out_ed_error_strings,
1549   .runtime_data_bytes = sizeof (snat_runtime_t),
1550 };
1551 /* *INDENT-ON* */
1552
1553 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1554                                               vlib_node_runtime_t *
1555                                               node, vlib_frame_t * frame)
1556 {
1557   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1558 }
1559
1560 /* *INDENT-OFF* */
1561 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1562   .name = "nat44-ed-in2out-slowpath",
1563   .vector_size = sizeof (u32),
1564   .sibling_of = "nat-default",
1565   .format_trace = format_nat_in2out_ed_trace,
1566   .type = VLIB_NODE_TYPE_INTERNAL,
1567   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1568   .error_strings = nat_in2out_ed_error_strings,
1569   .runtime_data_bytes = sizeof (snat_runtime_t),
1570 };
1571 /* *INDENT-ON* */
1572
1573 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1574                                                      vlib_node_runtime_t
1575                                                      * node,
1576                                                      vlib_frame_t * frame)
1577 {
1578   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1579 }
1580
1581 /* *INDENT-OFF* */
1582 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1583   .name = "nat44-ed-in2out-output-slowpath",
1584   .vector_size = sizeof (u32),
1585   .sibling_of = "nat-default",
1586   .format_trace = format_nat_in2out_ed_trace,
1587   .type = VLIB_NODE_TYPE_INTERNAL,
1588   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1589   .error_strings = nat_in2out_ed_error_strings,
1590   .runtime_data_bytes = sizeof (snat_runtime_t),
1591 };
1592 /* *INDENT-ON* */
1593
1594 static u8 *
1595 format_nat_pre_trace (u8 * s, va_list * args)
1596 {
1597   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1598   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1599   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1600   return format (s, "in2out next_index %d arc_next_index %d", t->next_index,
1601                  t->arc_next_index);
1602 }
1603
1604 VLIB_NODE_FN (nat_pre_in2out_node)
1605   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1606 {
1607   return nat_pre_node_fn_inline (vm, node, frame,
1608                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1609 }
1610
1611 /* *INDENT-OFF* */
1612 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1613   .name = "nat-pre-in2out",
1614   .vector_size = sizeof (u32),
1615   .sibling_of = "nat-default",
1616   .format_trace = format_nat_pre_trace,
1617   .type = VLIB_NODE_TYPE_INTERNAL,
1618   .n_errors = 0,
1619 };
1620 /* *INDENT-ON* */
1621
1622 /*
1623  * fd.io coding-style-patch-verification: ON
1624  *
1625  * Local Variables:
1626  * eval: (c-set-style "gnu")
1627  * End:
1628  */