src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/lib/ipfix_logging.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat44/inlines.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <nat/nat44/ed_inlines.h>
  34
  35 static char *nat_out2in_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_out2in_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat44_ed_out2in_trace_t;
  48
  49 static u8 *
  50 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  59     "NAT44_OUT2IN_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 static inline u32
  68 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  69                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  70                           u32 sw_if_index0, u32 rx_fib_index0,
  71                           vlib_node_runtime_t * node, u32 next0, f64 now,
  72                           u32 thread_index, snat_session_t ** p_s0)
  73 {
  74   vlib_main_t *vm = vlib_get_main ();
  75
  76   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  77                        next0, thread_index, p_s0, 0);
  78   snat_session_t *s0 = *p_s0;
  79   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  80     {
  81       /* Accounting */
  82       nat44_session_update_counters (s0, now,
  83                                      vlib_buffer_length_in_chain
  84                                      (vm, b0), thread_index);
  85       /* Per-user LRU list maintenance */
  86       nat44_session_update_lru (sm, s0, thread_index);
  87     }
  88   return next0;
  89 }
  90
  91 #ifndef CLIB_MARCH_VARIANT
  92 int
  93 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  94 {
  95   snat_main_t *sm = &snat_main;
  96   nat44_is_idle_session_ctx_t *ctx = arg;
  97   snat_session_t *s;
  98   u64 sess_timeout_time;
  99   u8 proto;
 100   u16 r_port, l_port;
 101   ip4_address_t *l_addr, *r_addr;
 102   u32 fib_index;
 103   clib_bihash_kv_16_8_t ed_kv;
 104   int i;
 105   //snat_address_t *a;
 106   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 107                                                        ctx->thread_index);
 108
 109   s = pool_elt_at_index (tsm->sessions, kv->value);
 110   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 111   if (ctx->now >= sess_timeout_time)
 112     {
 113       l_addr = &s->in2out.addr;
 114       r_addr = &s->ext_host_addr;
 115       fib_index = s->in2out.fib_index;
 116       if (snat_is_unk_proto_session (s))
 117         {
 118           proto = s->in2out.port;
 119           r_port = 0;
 120           l_port = 0;
 121         }
 122       else
 123         {
 124           proto = nat_proto_to_ip_proto (s->nat_proto);
 125           l_port = s->in2out.port;
 126           r_port = s->ext_host_port;
 127         }
 128       if (is_twice_nat_session (s))
 129         {
 130           r_addr = &s->ext_host_nat_addr;
 131           r_port = s->ext_host_nat_port;
 132         }
 133       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 134       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 135         nat_elog_warn ("in2out_ed key del failed");
 136
 137       if (snat_is_unk_proto_session (s))
 138         goto delete;
 139
 140       nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 141                                           s->in2out.addr.as_u32,
 142                                           s->out2in.addr.as_u32,
 143                                           s->nat_proto,
 144                                           s->in2out.port,
 145                                           s->out2in.port,
 146                                           s->in2out.fib_index);
 147
 148       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 149                              &s->in2out.addr, s->in2out.port,
 150                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 151                              &s->out2in.addr, s->out2in.port,
 152                              &s->ext_host_addr, s->ext_host_port,
 153                              s->nat_proto, is_twice_nat_session (s));
 154
 155       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 156                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 157                    ctx->thread_index);
 158
 159       if (is_twice_nat_session (s))
 160         {
 161           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 162             {
 163               // FIXME TODO this is obviously wrong code ... needs fix!
 164               //       key.protocol = s->nat_proto;
 165               //       key.port = s->ext_host_nat_port;
 166               //       a = sm->twice_nat_addresses + i;
 167               //       if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 168               //      {
 169               //        snat_free_outside_address_and_port (sm->twice_nat_addresses,
 170               //                                            ctx->thread_index,
 171               //                                            &key);
 172               //        break;
 173               //      }
 174             }
 175         }
 176
 177       if (snat_is_session_static (s))
 178         goto delete;
 179
 180       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 181                                           &s->out2in.addr, s->out2in.port,
 182                                           s->nat_proto);
 183     delete:
 184       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 185       return 1;
 186     }
 187
 188   return 0;
 189 }
 190 #endif
 191
 192 // allocate exact address based on preference
 193 static_always_inline int
 194 nat_alloc_addr_and_port_exact (snat_address_t * a,
 195                                u32 thread_index,
 196                                nat_protocol_t proto,
 197                                ip4_address_t * addr,
 198                                u16 * port,
 199                                u16 port_per_thread, u32 snat_thread_index)
 200 {
 201   u32 portnum;
 202
 203   switch (proto)
 204     {
 205 #define _(N, j, n, s) \
 206     case NAT_PROTOCOL_##N: \
 207       if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
 208         { \
 209           while (1) \
 210             { \
 211               portnum = (port_per_thread * \
 212                 snat_thread_index) + \
 213                 snat_random_port(0, port_per_thread - 1) + 1024; \
 214               if (a->busy_##n##_port_refcounts[portnum]) \
 215                 continue; \
 216               --a->busy_##n##_port_refcounts[portnum]; \
 217               a->busy_##n##_ports_per_thread[thread_index]++; \
 218               a->busy_##n##_ports++; \
 219               *addr = a->addr; \
 220               *port = clib_host_to_net_u16(portnum); \
 221               return 0; \
 222             } \
 223         } \
 224       break;
 225       foreach_nat_protocol
 226 #undef _
 227     default:
 228       nat_elog_info ("unknown protocol");
 229       return 1;
 230     }
 231
 232   /* Totally out of translations to use... */
 233   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 234   return 1;
 235 }
 236
 237
 238 static snat_session_t *
 239 create_session_for_static_mapping_ed (snat_main_t * sm,
 240                                       vlib_buffer_t * b,
 241                                       ip4_address_t i2o_addr,
 242                                       u16 i2o_port,
 243                                       u32 i2o_fib_index,
 244                                       ip4_address_t o2i_addr,
 245                                       u16 o2i_port,
 246                                       u32 o2i_fib_index,
 247                                       nat_protocol_t nat_proto,
 248                                       vlib_node_runtime_t * node,
 249                                       u32 rx_fib_index,
 250                                       u32 thread_index,
 251                                       twice_nat_type_t twice_nat,
 252                                       lb_nat_type_t lb_nat, f64 now,
 253                                       snat_static_mapping_t * mapping)
 254 {
 255   snat_session_t *s;
 256   ip4_header_t *ip;
 257   udp_header_t *udp;
 258   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 259   clib_bihash_kv_16_8_t kv;
 260   nat44_is_idle_session_ctx_t ctx;
 261
 262   if (PREDICT_FALSE
 263       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 264     {
 265       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 266       nat_elog_notice ("maximum sessions exceeded");
 267       return 0;
 268     }
 269
 270   s = nat_ed_session_alloc (sm, thread_index, now, nat_proto);
 271   if (!s)
 272     {
 273       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 274       nat_elog_warn ("create NAT session failed");
 275       return 0;
 276     }
 277
 278   ip = vlib_buffer_get_current (b);
 279   udp = ip4_next_header (ip);
 280
 281   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 282   s->ext_host_port = nat_proto == NAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 283   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 284   if (lb_nat)
 285     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 286   if (lb_nat == AFFINITY_LB_NAT)
 287     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 288   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 289   s->out2in.addr = o2i_addr;
 290   s->out2in.port = o2i_port;
 291   s->out2in.fib_index = o2i_fib_index;
 292   s->in2out.addr = i2o_addr;
 293   s->in2out.port = i2o_port;
 294   s->in2out.fib_index = i2o_fib_index;
 295   s->nat_proto = nat_proto;
 296
 297   /* Add to lookup tables */
 298   init_ed_kv (&kv, o2i_addr, o2i_port, s->ext_host_addr, s->ext_host_port,
 299               o2i_fib_index, ip->protocol, thread_index, s - tsm->sessions);
 300   ctx.now = now;
 301   ctx.thread_index = thread_index;
 302   if (clib_bihash_add_or_overwrite_stale_16_8 (&sm->out2in_ed, &kv,
 303                                                nat44_o2i_ed_is_idle_session_cb,
 304                                                &ctx))
 305     nat_elog_notice ("out2in-ed key add failed");
 306
 307   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 308                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 309     {
 310       int rc = 0;
 311       snat_address_t *filter = 0;
 312
 313       // if exact address is specified use this address
 314       if (is_exact_address (mapping))
 315         {
 316           snat_address_t *ap;
 317           vec_foreach (ap, sm->twice_nat_addresses)
 318           {
 319             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 320               {
 321                 filter = ap;
 322                 break;
 323               }
 324           }
 325         }
 326
 327       if (filter)
 328         {
 329           rc = nat_alloc_addr_and_port_exact (filter,
 330                                               thread_index,
 331                                               nat_proto,
 332                                               &s->ext_host_nat_addr,
 333                                               &s->ext_host_nat_port,
 334                                               sm->port_per_thread,
 335                                               tsm->snat_thread_index);
 336           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 337         }
 338       else
 339         {
 340           rc =
 341             snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 342                                                  thread_index, nat_proto,
 343                                                  &s->ext_host_nat_addr,
 344                                                  &s->ext_host_nat_port,
 345                                                  sm->port_per_thread,
 346                                                  tsm->snat_thread_index);
 347         }
 348
 349       if (rc)
 350         {
 351           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 352           nat_ed_session_delete (sm, s, thread_index, 1);
 353           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 0))
 354             nat_elog_notice ("out2in-ed key del failed");
 355           return 0;
 356         }
 357
 358       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 359       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_nat_addr,
 360                   s->ext_host_nat_port, i2o_fib_index, ip->protocol,
 361                   thread_index, s - tsm->sessions);
 362     }
 363   else
 364     {
 365       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_addr,
 366                   s->ext_host_port, i2o_fib_index, ip->protocol,
 367                   thread_index, s - tsm->sessions);
 368     }
 369   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 370                                                nat44_i2o_ed_is_idle_session_cb,
 371                                                &ctx))
 372     nat_elog_notice ("in2out-ed key add failed");
 373
 374   nat_ipfix_logging_nat44_ses_create (thread_index,
 375                                       s->in2out.addr.as_u32,
 376                                       s->out2in.addr.as_u32,
 377                                       s->nat_proto,
 378                                       s->in2out.port,
 379                                       s->out2in.port, s->in2out.fib_index);
 380
 381   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 382                          &s->in2out.addr, s->in2out.port,
 383                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 384                          &s->out2in.addr, s->out2in.port,
 385                          &s->ext_host_addr, s->ext_host_port,
 386                          s->nat_proto, is_twice_nat_session (s));
 387
 388   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 389                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 390                &s->ext_host_nat_addr, s->ext_host_nat_port,
 391                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 392
 393   per_vrf_sessions_register_session (s, thread_index);
 394
 395   return s;
 396 }
 397
 398 static int
 399 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
 400               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 401 {
 402   clib_bihash_kv_16_8_t kv, value;
 403   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 404
 405   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 406              rx_fib_index, ip->protocol);
 407   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 408     return 1;
 409
 410   return 0;
 411 }
 412
 413 static void
 414 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
 415                        u32 rx_fib_index, u32 thread_index)
 416 {
 417   clib_bihash_kv_16_8_t kv, value;
 418   udp_header_t *udp;
 419   snat_session_t *s = 0;
 420   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 421   vlib_main_t *vm = vlib_get_main ();
 422   f64 now = vlib_time_now (vm);
 423   u16 l_port, r_port;
 424
 425   if (ip->protocol == IP_PROTOCOL_ICMP)
 426     {
 427       if (get_icmp_o2i_ed_key
 428           (b, ip, rx_fib_index, ~0, ~0, 0, &l_port, &r_port, &kv))
 429         return;
 430     }
 431   else
 432     {
 433       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 434         {
 435           udp = ip4_next_header (ip);
 436           l_port = udp->dst_port;
 437           r_port = udp->src_port;
 438         }
 439       else
 440         {
 441           l_port = 0;
 442           r_port = 0;
 443         }
 444       init_ed_k (&kv, ip->dst_address, l_port, ip->src_address, r_port,
 445                  rx_fib_index, ip->protocol);
 446     }
 447
 448   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 449     {
 450       ASSERT (thread_index == ed_value_get_thread_index (&value));
 451       s =
 452         pool_elt_at_index (tsm->sessions,
 453                            ed_value_get_session_index (&value));
 454     }
 455   else if (ip->protocol == IP_PROTOCOL_ICMP &&
 456            icmp_type_is_error_message
 457            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 458     {
 459       return;
 460     }
 461   else
 462     {
 463       u32 proto;
 464
 465       if (PREDICT_FALSE
 466           (nat44_ed_maximum_sessions_exceeded
 467            (sm, rx_fib_index, thread_index)))
 468         return;
 469
 470       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 471       if (!s)
 472         {
 473           nat_elog_warn ("create NAT session failed");
 474           return;
 475         }
 476
 477       proto = ip_proto_to_nat_proto (ip->protocol);
 478
 479       s->ext_host_addr = ip->src_address;
 480       s->ext_host_port = r_port;
 481       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 482       s->out2in.addr = ip->dst_address;
 483       s->out2in.port = l_port;
 484       s->nat_proto = proto;
 485       if (proto == NAT_PROTOCOL_OTHER)
 486         {
 487           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 488           s->out2in.port = ip->protocol;
 489         }
 490       s->out2in.fib_index = rx_fib_index;
 491       s->in2out.addr = s->out2in.addr;
 492       s->in2out.port = s->out2in.port;
 493       s->in2out.fib_index = s->out2in.fib_index;
 494
 495       kv.value = s - tsm->sessions;
 496       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 497         nat_elog_notice ("in2out_ed key add failed");
 498
 499       per_vrf_sessions_register_session (s, thread_index);
 500     }
 501
 502   if (ip->protocol == IP_PROTOCOL_TCP)
 503     {
 504       tcp_header_t *tcp = ip4_next_header (ip);
 505       if (nat44_set_tcp_session_state_o2i
 506           (sm, now, s, tcp->flags, tcp->ack_number, tcp->seq_number,
 507            thread_index))
 508         return;
 509     }
 510
 511   /* Accounting */
 512   nat44_session_update_counters (s, now, 0, thread_index);
 513   /* Per-user LRU list maintenance */
 514   nat44_session_update_lru (sm, s, thread_index);
 515 }
 516
 517 static inline void
 518 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
 519                               ip4_header_t * ip, u32 rx_fib_index)
 520 {
 521   ip4_header_t ip_wkr = {
 522     .src_address = ip->dst_address,
 523   };
 524   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 525
 526   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 527 }
 528
 529 #ifndef CLIB_MARCH_VARIANT
 530 u32
 531 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 532                       u32 thread_index, vlib_buffer_t * b,
 533                       ip4_header_t * ip, ip4_address_t * addr,
 534                       u16 * port, u32 * fib_index, nat_protocol_t * proto,
 535                       void *d, void *e, u8 * dont_translate)
 536 {
 537   u32 next = ~0, sw_if_index, rx_fib_index;
 538   clib_bihash_kv_16_8_t kv, value;
 539   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 540   snat_session_t *s = 0;
 541   u8 is_addr_only, identity_nat;
 542   u16 l_port, r_port;
 543   vlib_main_t *vm = vlib_get_main ();
 544   ip4_address_t sm_addr;
 545   u16 sm_port;
 546   u32 sm_fib_index;
 547   *dont_translate = 0;
 548   snat_static_mapping_t *m;
 549
 550   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 551   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 552
 553   if (get_icmp_o2i_ed_key
 554       (b, ip, rx_fib_index, ~0, ~0, proto, &l_port, &r_port, &kv))
 555     {
 556       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 557       next = NAT_NEXT_DROP;
 558       goto out;
 559     }
 560
 561   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 562     {
 563       if (snat_static_mapping_match
 564           (sm, ip->dst_address, l_port, rx_fib_index,
 565            ip_proto_to_nat_proto (ip->protocol), &sm_addr, &sm_port,
 566            &sm_fib_index, 1, &is_addr_only, 0, 0, 0, &identity_nat, &m))
 567         {
 568           // static mapping not matched
 569           if (!sm->forwarding_enabled)
 570             {
 571               /* Don't NAT packet aimed at the intfc address */
 572               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 573                                                     ip->dst_address.as_u32)))
 574                 {
 575                   *dont_translate = 1;
 576                 }
 577               else
 578                 {
 579                   b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 580                   next = NAT_NEXT_DROP;
 581                 }
 582             }
 583           else
 584             {
 585               *dont_translate = 1;
 586               if (next_src_nat (sm, ip, l_port, r_port,
 587                                 thread_index, rx_fib_index))
 588                 {
 589                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 590                 }
 591               else
 592                 {
 593                   if (sm->num_workers > 1)
 594                     create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
 595                   else
 596                     create_bypass_for_fwd (sm, b, ip, rx_fib_index,
 597                                            thread_index);
 598                 }
 599             }
 600           goto out;
 601         }
 602
 603       if (PREDICT_FALSE
 604           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 605            ICMP4_echo_reply
 606            && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 607                ICMP4_echo_request || !is_addr_only)))
 608         {
 609           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 610           next = NAT_NEXT_DROP;
 611           goto out;
 612         }
 613
 614       if (PREDICT_FALSE (identity_nat))
 615         {
 616           *dont_translate = 1;
 617           goto out;
 618         }
 619
 620       /* Create session initiated by host from external network */
 621       s =
 622         create_session_for_static_mapping_ed (sm, b, sm_addr, sm_port,
 623                                               sm_fib_index, ip->dst_address,
 624                                               l_port, rx_fib_index, *proto,
 625                                               node, rx_fib_index,
 626                                               thread_index, 0, 0,
 627                                               vlib_time_now (vm), m);
 628       if (!s)
 629         next = NAT_NEXT_DROP;
 630     }
 631   else
 632     {
 633       if (PREDICT_FALSE
 634           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 635            ICMP4_echo_reply
 636            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 637            ICMP4_echo_request
 638            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 639                                            reass.icmp_type_or_tcp_flags)))
 640         {
 641           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 642           next = NAT_NEXT_DROP;
 643           goto out;
 644         }
 645
 646       ASSERT (thread_index == ed_value_get_thread_index (&value));
 647       s =
 648         pool_elt_at_index (tsm->sessions,
 649                            ed_value_get_session_index (&value));
 650     }
 651 out:
 652   if (s)
 653     {
 654       *addr = s->in2out.addr;
 655       *port = s->in2out.port;
 656       *fib_index = s->in2out.fib_index;
 657     }
 658   if (d)
 659     *(snat_session_t **) d = s;
 660   return next;
 661 }
 662 #endif
 663
 664 static snat_session_t *
 665 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 666                                vlib_buffer_t * b,
 667                                ip4_header_t * ip,
 668                                u32 rx_fib_index,
 669                                u32 thread_index,
 670                                f64 now,
 671                                vlib_main_t * vm, vlib_node_runtime_t * node)
 672 {
 673   clib_bihash_kv_8_8_t kv, value;
 674   clib_bihash_kv_16_8_t s_kv, s_value;
 675   snat_static_mapping_t *m;
 676   u32 old_addr, new_addr;
 677   ip_csum_t sum;
 678   snat_session_t *s;
 679   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 680
 681   old_addr = ip->dst_address.as_u32;
 682
 683   init_ed_k (&s_kv, ip->dst_address, 0, ip->src_address, 0, rx_fib_index,
 684              ip->protocol);
 685
 686   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 687     {
 688       ASSERT (thread_index == ed_value_get_thread_index (&s_value));
 689       s =
 690         pool_elt_at_index (tsm->sessions,
 691                            ed_value_get_session_index (&s_value));
 692       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 693     }
 694   else
 695     {
 696       if (PREDICT_FALSE
 697           (nat44_ed_maximum_sessions_exceeded
 698            (sm, rx_fib_index, thread_index)))
 699         {
 700           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 701           nat_elog_notice ("maximum sessions exceeded");
 702           return 0;
 703         }
 704
 705       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 706       if (clib_bihash_search_8_8
 707           (&sm->static_mapping_by_external, &kv, &value))
 708         {
 709           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 710           return 0;
 711         }
 712
 713       m = pool_elt_at_index (sm->static_mappings, value.value);
 714
 715       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 716
 717       /* Create a new session */
 718       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 719       if (!s)
 720         {
 721           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 722           nat_elog_warn ("create NAT session failed");
 723           return 0;
 724         }
 725
 726       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 727       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 728       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 729       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 730       s->out2in.addr.as_u32 = old_addr;
 731       s->out2in.fib_index = rx_fib_index;
 732       s->in2out.addr.as_u32 = new_addr;
 733       s->in2out.fib_index = m->fib_index;
 734       s->in2out.port = s->out2in.port = ip->protocol;
 735
 736       /* Add to lookup tables */
 737       s_kv.value = s - tsm->sessions;
 738       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 739         nat_elog_notice ("out2in key add failed");
 740
 741       init_ed_kv (&s_kv, ip->dst_address, 0, ip->src_address, 0, m->fib_index,
 742                   ip->protocol, thread_index, s - tsm->sessions);
 743       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 744         nat_elog_notice ("in2out key add failed");
 745
 746       per_vrf_sessions_register_session (s, thread_index);
 747     }
 748
 749   /* Update IP checksum */
 750   sum = ip->checksum;
 751   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 752   ip->checksum = ip_csum_fold (sum);
 753
 754   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 755
 756   /* Accounting */
 757   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 758                                  thread_index);
 759   /* Per-user LRU list maintenance */
 760   nat44_session_update_lru (sm, s, thread_index);
 761
 762   return s;
 763 }
 764
 765 static inline uword
 766 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 767                                           vlib_node_runtime_t * node,
 768                                           vlib_frame_t * frame,
 769                                           int is_multi_worker)
 770 {
 771   u32 n_left_from, *from;
 772   snat_main_t *sm = &snat_main;
 773   f64 now = vlib_time_now (vm);
 774   u32 thread_index = vm->thread_index;
 775   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 776
 777   from = vlib_frame_vector_args (frame);
 778   n_left_from = frame->n_vectors;
 779
 780   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 781   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 782   vlib_get_buffers (vm, from, b, n_left_from);
 783
 784   while (n_left_from > 0)
 785     {
 786       vlib_buffer_t *b0;
 787       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
 788       u16 old_port0, new_port0;
 789       ip4_header_t *ip0;
 790       udp_header_t *udp0;
 791       tcp_header_t *tcp0;
 792       snat_session_t *s0 = 0;
 793       clib_bihash_kv_16_8_t kv0, value0;
 794       ip_csum_t sum0;
 795
 796       b0 = *b;
 797       b++;
 798
 799       /* Prefetch next iteration. */
 800       if (PREDICT_TRUE (n_left_from >= 2))
 801         {
 802           vlib_buffer_t *p2;
 803
 804           p2 = *b;
 805
 806           vlib_prefetch_buffer_header (p2, LOAD);
 807
 808           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 809         }
 810
 811       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 812
 813       vnet_buffer (b0)->snat.flags = 0;
 814       ip0 = vlib_buffer_get_current (b0);
 815
 816       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 817       rx_fib_index0 =
 818         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 819
 820       if (PREDICT_FALSE (ip0->ttl == 1))
 821         {
 822           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 823           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 824                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 825                                        0);
 826           next[0] = NAT_NEXT_ICMP_ERROR;
 827           goto trace0;
 828         }
 829
 830       udp0 = ip4_next_header (ip0);
 831       tcp0 = (tcp_header_t *) udp0;
 832       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 833
 834       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 835         {
 836           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 837           goto trace0;
 838         }
 839
 840       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 841         {
 842           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 843           goto trace0;
 844         }
 845
 846       init_ed_k (&kv0, ip0->dst_address,
 847                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
 848                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
 849                  ip0->protocol);
 850
 851       /* there is a stashed index in vnet_buffer2 from handoff node,
 852        * see if we can use it */
 853       if (is_multi_worker
 854           &&
 855           PREDICT_TRUE (!pool_is_free_index
 856                         (tsm->sessions,
 857                          vnet_buffer2 (b0)->nat.ed_out2in_nat_session_index)))
 858         {
 859           s0 = pool_elt_at_index (tsm->sessions,
 860                                   vnet_buffer2 (b0)->
 861                                   nat.ed_out2in_nat_session_index);
 862           if (PREDICT_TRUE
 863               (s0->out2in.addr.as_u32 == ip0->dst_address.as_u32
 864                && s0->out2in.port == vnet_buffer (b0)->ip.reass.l4_dst_port
 865                && s0->nat_proto == ip_proto_to_nat_proto (ip0->protocol)
 866                && s0->out2in.fib_index == rx_fib_index0
 867                && s0->ext_host_addr.as_u32 == ip0->src_address.as_u32
 868                && s0->ext_host_port ==
 869                vnet_buffer (b0)->ip.reass.l4_src_port))
 870             {
 871               /* yes, this is the droid we're looking for */
 872               goto skip_lookup;
 873             }
 874         }
 875
 876       // lookup for session
 877       if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
 878         {
 879           // session does not exist go slow path
 880           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 881           goto trace0;
 882         }
 883       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 884       s0 =
 885         pool_elt_at_index (tsm->sessions,
 886                            ed_value_get_session_index (&value0));
 887
 888     skip_lookup:
 889
 890       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 891         {
 892           // session is closed, go slow path
 893           nat_free_session_data (sm, s0, thread_index, 0);
 894           nat_ed_session_delete (sm, s0, thread_index, 1);
 895           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 896           goto trace0;
 897         }
 898
 899       if (s0->tcp_closed_timestamp)
 900         {
 901           if (now >= s0->tcp_closed_timestamp)
 902             {
 903               // session is closed, go slow path, freed in slow path
 904               next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 905             }
 906           else
 907             {
 908               // session in transitory timeout, drop
 909               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 910               next[0] = NAT_NEXT_DROP;
 911             }
 912           goto trace0;
 913         }
 914
 915       // drop if session expired
 916       u64 sess_timeout_time;
 917       sess_timeout_time =
 918         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 919       if (now >= sess_timeout_time)
 920         {
 921           // session is closed, go slow path
 922           nat_free_session_data (sm, s0, thread_index, 0);
 923           nat_ed_session_delete (sm, s0, thread_index, 1);
 924           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 925           goto trace0;
 926         }
 927
 928       old_addr0 = ip0->dst_address.as_u32;
 929       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 930       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 931
 932       sum0 = ip0->checksum;
 933       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 934                              dst_address);
 935       if (PREDICT_FALSE (is_twice_nat_session (s0)))
 936         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 937                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
 938                                src_address);
 939       ip0->checksum = ip_csum_fold (sum0);
 940
 941       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 942
 943       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 944         {
 945           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 946             {
 947               new_port0 = udp0->dst_port = s0->in2out.port;
 948               sum0 = tcp0->checksum;
 949               sum0 =
 950                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 951                                 dst_address);
 952               sum0 =
 953                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 954                                 length);
 955               if (is_twice_nat_session (s0))
 956                 {
 957                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 958                                          s0->ext_host_nat_addr.as_u32,
 959                                          ip4_header_t, dst_address);
 960                   sum0 =
 961                     ip_csum_update (sum0,
 962                                     vnet_buffer (b0)->ip.reass.l4_src_port,
 963                                     s0->ext_host_nat_port, ip4_header_t,
 964                                     length);
 965                   tcp0->src_port = s0->ext_host_nat_port;
 966                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 967                 }
 968               tcp0->checksum = ip_csum_fold (sum0);
 969             }
 970           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.tcp,
 971                                          thread_index, sw_if_index0, 1);
 972           if (nat44_set_tcp_session_state_o2i
 973               (sm, now, s0, vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
 974                vnet_buffer (b0)->ip.reass.tcp_ack_number,
 975                vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
 976             goto trace0;
 977         }
 978       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
 979                && udp0->checksum)
 980         {
 981           new_port0 = udp0->dst_port = s0->in2out.port;
 982           sum0 = udp0->checksum;
 983           sum0 =
 984             ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 985                             dst_address);
 986           sum0 =
 987             ip_csum_update (sum0, old_port0, new_port0, ip4_header_t, length);
 988           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 989             {
 990               sum0 =
 991                 ip_csum_update (sum0, ip0->src_address.as_u32,
 992                                 s0->ext_host_nat_addr.as_u32, ip4_header_t,
 993                                 dst_address);
 994               sum0 =
 995                 ip_csum_update (sum0, vnet_buffer (b0)->ip.reass.l4_src_port,
 996                                 s0->ext_host_nat_port, ip4_header_t, length);
 997               udp0->src_port = s0->ext_host_nat_port;
 998               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 999             }
1000           udp0->checksum = ip_csum_fold (sum0);
1001           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
1002                                          thread_index, sw_if_index0, 1);
1003         }
1004       else
1005         {
1006           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1007             {
1008               new_port0 = udp0->dst_port = s0->in2out.port;
1009               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1010                 {
1011                   udp0->src_port = s0->ext_host_nat_port;
1012                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1013                 }
1014             }
1015           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
1016                                          thread_index, sw_if_index0, 1);
1017         }
1018
1019       /* Accounting */
1020       nat44_session_update_counters (s0, now,
1021                                      vlib_buffer_length_in_chain (vm, b0),
1022                                      thread_index);
1023       /* Per-user LRU list maintenance */
1024       nat44_session_update_lru (sm, s0, thread_index);
1025
1026     trace0:
1027       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1028                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1029         {
1030           nat44_ed_out2in_trace_t *t =
1031             vlib_add_trace (vm, node, b0, sizeof (*t));
1032           t->sw_if_index = sw_if_index0;
1033           t->next_index = next[0];
1034           t->is_slow_path = 0;
1035
1036           if (s0)
1037             t->session_index = s0 - tsm->sessions;
1038           else
1039             t->session_index = ~0;
1040         }
1041
1042       if (next[0] == NAT_NEXT_DROP)
1043         {
1044           vlib_increment_simple_counter (&sm->counters.fastpath.
1045                                          out2in_ed.drops, thread_index,
1046                                          sw_if_index0, 1);
1047         }
1048
1049       n_left_from--;
1050       next++;
1051     }
1052
1053   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1054                                frame->n_vectors);
1055   return frame->n_vectors;
1056 }
1057
1058 static inline uword
1059 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1060                                           vlib_node_runtime_t * node,
1061                                           vlib_frame_t * frame)
1062 {
1063   u32 n_left_from, *from;
1064   snat_main_t *sm = &snat_main;
1065   f64 now = vlib_time_now (vm);
1066   u32 thread_index = vm->thread_index;
1067   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1068   snat_static_mapping_t *m;
1069
1070   from = vlib_frame_vector_args (frame);
1071   n_left_from = frame->n_vectors;
1072
1073   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1074   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1075   vlib_get_buffers (vm, from, b, n_left_from);
1076
1077   while (n_left_from > 0)
1078     {
1079       vlib_buffer_t *b0;
1080       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
1081       u16 old_port0, new_port0;
1082       ip4_header_t *ip0;
1083       udp_header_t *udp0;
1084       tcp_header_t *tcp0;
1085       icmp46_header_t *icmp0;
1086       snat_session_t *s0 = 0;
1087       clib_bihash_kv_16_8_t kv0, value0;
1088       ip_csum_t sum0;
1089       lb_nat_type_t lb_nat0;
1090       twice_nat_type_t twice_nat0;
1091       u8 identity_nat0;
1092       ip4_address_t sm_addr;
1093       u16 sm_port;
1094       u32 sm_fib_index;
1095
1096       b0 = *b;
1097       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1098
1099       vnet_buffer (b0)->snat.flags = 0;
1100       ip0 = vlib_buffer_get_current (b0);
1101
1102       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1103       rx_fib_index0 =
1104         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1105
1106       if (PREDICT_FALSE (ip0->ttl == 1))
1107         {
1108           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1109           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1110                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1111                                        0);
1112           next[0] = NAT_NEXT_ICMP_ERROR;
1113           goto trace0;
1114         }
1115
1116       udp0 = ip4_next_header (ip0);
1117       tcp0 = (tcp_header_t *) udp0;
1118       icmp0 = (icmp46_header_t *) udp0;
1119       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1120
1121       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1122         {
1123           s0 =
1124             nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1125                                            thread_index, now, vm, node);
1126           if (!sm->forwarding_enabled)
1127             {
1128               if (!s0)
1129                 next[0] = NAT_NEXT_DROP;
1130             }
1131           vlib_increment_simple_counter (&sm->counters.slowpath.
1132                                          out2in_ed.other, thread_index,
1133                                          sw_if_index0, 1);
1134           goto trace0;
1135         }
1136
1137       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1138         {
1139           next[0] = icmp_out2in_ed_slow_path
1140             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1141              next[0], now, thread_index, &s0);
1142           vlib_increment_simple_counter (&sm->counters.slowpath.
1143                                          out2in_ed.icmp, thread_index,
1144                                          sw_if_index0, 1);
1145           goto trace0;
1146         }
1147
1148       init_ed_k (&kv0, ip0->dst_address,
1149                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
1150                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1151                  ip0->protocol);
1152
1153       s0 = NULL;
1154       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
1155         {
1156           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1157           s0 =
1158             pool_elt_at_index (tsm->sessions,
1159                                ed_value_get_session_index (&value0));
1160
1161           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1162             {
1163               nat_free_session_data (sm, s0, thread_index, 0);
1164               nat_ed_session_delete (sm, s0, thread_index, 1);
1165               s0 = NULL;
1166             }
1167         }
1168
1169       if (!s0)
1170         {
1171           /* Try to match static mapping by external address and port,
1172              destination address and port in packet */
1173
1174           if (snat_static_mapping_match
1175               (sm, ip0->dst_address,
1176                vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1177                proto0, &sm_addr, &sm_port, &sm_fib_index, 1, 0,
1178                &twice_nat0, &lb_nat0, &ip0->src_address, &identity_nat0, &m))
1179             {
1180               /*
1181                * Send DHCP packets to the ipv4 stack, or we won't
1182                * be able to use dhcp client on the outside interface
1183                */
1184               if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_UDP
1185                                  && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1186                                      clib_host_to_net_u16
1187                                      (UDP_DST_PORT_dhcp_to_client))))
1188                 {
1189                   goto trace0;
1190                 }
1191
1192               if (!sm->forwarding_enabled)
1193                 {
1194                   b0->error =
1195                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1196                   next[0] = NAT_NEXT_DROP;
1197                 }
1198               else
1199                 {
1200                   if (next_src_nat
1201                       (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1202                        vnet_buffer (b0)->ip.reass.l4_dst_port,
1203                        thread_index, rx_fib_index0))
1204                     {
1205                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1206                     }
1207                   else
1208                     {
1209                       if (sm->num_workers > 1)
1210                         create_bypass_for_fwd_worker (sm, b0, ip0,
1211                                                       rx_fib_index0);
1212                       else
1213                         create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1214                                                thread_index);
1215                     }
1216                 }
1217               goto trace0;
1218             }
1219
1220           if (PREDICT_FALSE (identity_nat0))
1221             goto trace0;
1222
1223           if ((proto0 == NAT_PROTOCOL_TCP)
1224               && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1225                                      reass.icmp_type_or_tcp_flags))
1226             {
1227               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1228               next[0] = NAT_NEXT_DROP;
1229               goto trace0;
1230             }
1231
1232           /* Create session initiated by host from external network */
1233           s0 = create_session_for_static_mapping_ed (sm, b0,
1234                                                      sm_addr, sm_port,
1235                                                      sm_fib_index,
1236                                                      ip0->dst_address,
1237                                                      vnet_buffer (b0)->
1238                                                      ip.reass.l4_dst_port,
1239                                                      rx_fib_index0, proto0,
1240                                                      node, rx_fib_index0,
1241                                                      thread_index, twice_nat0,
1242                                                      lb_nat0, now, m);
1243           if (!s0)
1244             {
1245               next[0] = NAT_NEXT_DROP;
1246               goto trace0;
1247             }
1248         }
1249
1250       old_addr0 = ip0->dst_address.as_u32;
1251       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1252       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1253
1254       sum0 = ip0->checksum;
1255       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1256                              dst_address);
1257       if (PREDICT_FALSE (is_twice_nat_session (s0)))
1258         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1259                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
1260                                src_address);
1261       ip0->checksum = ip_csum_fold (sum0);
1262
1263       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1264
1265       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1266         {
1267           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1268             {
1269               new_port0 = udp0->dst_port = s0->in2out.port;
1270               sum0 = tcp0->checksum;
1271               sum0 =
1272                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1273                                 dst_address);
1274               sum0 =
1275                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1276                                 length);
1277               if (is_twice_nat_session (s0))
1278                 {
1279                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1280                                          s0->ext_host_nat_addr.as_u32,
1281                                          ip4_header_t, dst_address);
1282                   sum0 =
1283                     ip_csum_update (sum0,
1284                                     vnet_buffer (b0)->ip.reass.l4_src_port,
1285                                     s0->ext_host_nat_port, ip4_header_t,
1286                                     length);
1287                   tcp0->src_port = s0->ext_host_nat_port;
1288                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1289                 }
1290               tcp0->checksum = ip_csum_fold (sum0);
1291             }
1292           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.tcp,
1293                                          thread_index, sw_if_index0, 1);
1294           if (nat44_set_tcp_session_state_o2i
1295               (sm, now, s0, vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
1296                vnet_buffer (b0)->ip.reass.tcp_ack_number,
1297                vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
1298             goto trace0;
1299         }
1300       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1301                && udp0->checksum)
1302         {
1303           new_port0 = udp0->dst_port = s0->in2out.port;
1304           sum0 = udp0->checksum;
1305           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1306                                  dst_address);
1307           sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1308                                  length);
1309           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1310             {
1311               sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1312                                      s0->ext_host_nat_addr.as_u32,
1313                                      ip4_header_t, dst_address);
1314               sum0 =
1315                 ip_csum_update (sum0,
1316                                 vnet_buffer (b0)->ip.reass.l4_src_port,
1317                                 s0->ext_host_nat_port, ip4_header_t, length);
1318               udp0->src_port = s0->ext_host_nat_port;
1319               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1320             }
1321           udp0->checksum = ip_csum_fold (sum0);
1322           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1323                                          thread_index, sw_if_index0, 1);
1324         }
1325       else
1326         {
1327           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1328             {
1329               new_port0 = udp0->dst_port = s0->in2out.port;
1330               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1331                 {
1332                   udp0->src_port = s0->ext_host_nat_port;
1333                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1334                 }
1335             }
1336           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1337                                          thread_index, sw_if_index0, 1);
1338         }
1339
1340       /* Accounting */
1341       nat44_session_update_counters (s0, now,
1342                                      vlib_buffer_length_in_chain (vm, b0),
1343                                      thread_index);
1344       /* Per-user LRU list maintenance */
1345       nat44_session_update_lru (sm, s0, thread_index);
1346
1347     trace0:
1348       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1349                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1350         {
1351           nat44_ed_out2in_trace_t *t =
1352             vlib_add_trace (vm, node, b0, sizeof (*t));
1353           t->sw_if_index = sw_if_index0;
1354           t->next_index = next[0];
1355           t->is_slow_path = 1;
1356
1357           if (s0)
1358             t->session_index = s0 - tsm->sessions;
1359           else
1360             t->session_index = ~0;
1361         }
1362
1363       if (next[0] == NAT_NEXT_DROP)
1364         {
1365           vlib_increment_simple_counter (&sm->counters.slowpath.
1366                                          out2in_ed.drops, thread_index,
1367                                          sw_if_index0, 1);
1368         }
1369
1370       n_left_from--;
1371       next++;
1372       b++;
1373     }
1374
1375   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1376                                frame->n_vectors);
1377
1378   return frame->n_vectors;
1379 }
1380
1381 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1382                                      vlib_node_runtime_t * node,
1383                                      vlib_frame_t * frame)
1384 {
1385   if (snat_main.num_workers > 1)
1386     {
1387       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1388     }
1389   else
1390     {
1391       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1392     }
1393 }
1394
1395 /* *INDENT-OFF* */
1396 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1397   .name = "nat44-ed-out2in",
1398   .vector_size = sizeof (u32),
1399   .sibling_of = "nat-default",
1400   .format_trace = format_nat44_ed_out2in_trace,
1401   .type = VLIB_NODE_TYPE_INTERNAL,
1402   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1403   .error_strings = nat_out2in_ed_error_strings,
1404   .runtime_data_bytes = sizeof (snat_runtime_t),
1405 };
1406 /* *INDENT-ON* */
1407
1408 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1409                                               vlib_node_runtime_t * node,
1410                                               vlib_frame_t * frame)
1411 {
1412   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1413 }
1414
1415 /* *INDENT-OFF* */
1416 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1417   .name = "nat44-ed-out2in-slowpath",
1418   .vector_size = sizeof (u32),
1419   .sibling_of = "nat-default",
1420   .format_trace = format_nat44_ed_out2in_trace,
1421   .type = VLIB_NODE_TYPE_INTERNAL,
1422   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1423   .error_strings = nat_out2in_ed_error_strings,
1424   .runtime_data_bytes = sizeof (snat_runtime_t),
1425 };
1426 /* *INDENT-ON* */
1427
1428 static u8 *
1429 format_nat_pre_trace (u8 * s, va_list * args)
1430 {
1431   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1432   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1433   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1434   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1435                  t->arc_next_index);
1436 }
1437
1438 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1439                                     vlib_node_runtime_t * node,
1440                                     vlib_frame_t * frame)
1441 {
1442   return nat_pre_node_fn_inline (vm, node, frame,
1443                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1444 }
1445
1446 /* *INDENT-OFF* */
1447 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1448   .name = "nat-pre-out2in",
1449   .vector_size = sizeof (u32),
1450   .sibling_of = "nat-default",
1451   .format_trace = format_nat_pre_trace,
1452   .type = VLIB_NODE_TYPE_INTERNAL,
1453   .n_errors = 0,
1454  };
1455 /* *INDENT-ON* */
1456
1457 /*
1458  * fd.io coding-style-patch-verification: ON
1459  *
1460  * Local Variables:
1461  * eval: (c-set-style "gnu")
1462  * End:
1463  */