src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34 #include <nat/nat44/ed_inlines.h>
  35
  36 static char *nat_out2in_ed_error_strings[] = {
  37 #define _(sym,string) string,
  38   foreach_nat_out2in_ed_error
  39 #undef _
  40 };
  41
  42 typedef struct
  43 {
  44   u32 sw_if_index;
  45   u32 next_index;
  46   u32 session_index;
  47   u32 is_slow_path;
  48 } nat44_ed_out2in_trace_t;
  49
  50 static u8 *
  51 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  56   char *tag;
  57
  58   tag =
  59     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  60     "NAT44_OUT2IN_ED_FAST_PATH";
  61
  62   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  63               t->sw_if_index, t->next_index, t->session_index);
  64
  65   return s;
  66 }
  67
  68 static inline u32
  69 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  70                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  71                           u32 sw_if_index0, u32 rx_fib_index0,
  72                           vlib_node_runtime_t * node, u32 next0, f64 now,
  73                           u32 thread_index, snat_session_t ** p_s0)
  74 {
  75   vlib_main_t *vm = vlib_get_main ();
  76
  77   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  78                        next0, thread_index, p_s0, 0);
  79   snat_session_t *s0 = *p_s0;
  80   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  81     {
  82       /* Accounting */
  83       nat44_session_update_counters (s0, now,
  84                                      vlib_buffer_length_in_chain
  85                                      (vm, b0), thread_index);
  86       /* Per-user LRU list maintenance */
  87       nat44_session_update_lru (sm, s0, thread_index);
  88     }
  89   return next0;
  90 }
  91
  92 #ifndef CLIB_MARCH_VARIANT
  93 int
  94 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  95 {
  96   snat_main_t *sm = &snat_main;
  97   nat44_is_idle_session_ctx_t *ctx = arg;
  98   snat_session_t *s;
  99   u64 sess_timeout_time;
 100   u8 proto;
 101   u16 r_port, l_port;
 102   ip4_address_t *l_addr, *r_addr;
 103   u32 fib_index;
 104   clib_bihash_kv_16_8_t ed_kv;
 105   int i;
 106   //snat_address_t *a;
 107   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 108                                                        ctx->thread_index);
 109
 110   s = pool_elt_at_index (tsm->sessions, kv->value);
 111   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 112   if (ctx->now >= sess_timeout_time)
 113     {
 114       l_addr = &s->in2out.addr;
 115       r_addr = &s->ext_host_addr;
 116       fib_index = s->in2out.fib_index;
 117       if (snat_is_unk_proto_session (s))
 118         {
 119           proto = s->in2out.port;
 120           r_port = 0;
 121           l_port = 0;
 122         }
 123       else
 124         {
 125           proto = nat_proto_to_ip_proto (s->nat_proto);
 126           l_port = s->in2out.port;
 127           r_port = s->ext_host_port;
 128         }
 129       if (is_twice_nat_session (s))
 130         {
 131           r_addr = &s->ext_host_nat_addr;
 132           r_port = s->ext_host_nat_port;
 133         }
 134       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 135       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 136         nat_elog_warn ("in2out_ed key del failed");
 137
 138       if (snat_is_unk_proto_session (s))
 139         goto delete;
 140
 141       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 142                                            s->in2out.addr.as_u32,
 143                                            s->out2in.addr.as_u32,
 144                                            s->nat_proto,
 145                                            s->in2out.port,
 146                                            s->out2in.port,
 147                                            s->in2out.fib_index);
 148
 149       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 150                              &s->in2out.addr, s->in2out.port,
 151                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 152                              &s->out2in.addr, s->out2in.port,
 153                              &s->ext_host_addr, s->ext_host_port,
 154                              s->nat_proto, is_twice_nat_session (s));
 155
 156       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 157                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 158                    ctx->thread_index);
 159
 160       if (is_twice_nat_session (s))
 161         {
 162           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 163             {
 164               // FIXME TODO this is obviously wrong code ... needs fix!
 165               //       key.protocol = s->nat_proto;
 166               //       key.port = s->ext_host_nat_port;
 167               //       a = sm->twice_nat_addresses + i;
 168               //       if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 169               //      {
 170               //        snat_free_outside_address_and_port (sm->twice_nat_addresses,
 171               //                                            ctx->thread_index,
 172               //                                            &key);
 173               //        break;
 174               //      }
 175             }
 176         }
 177
 178       if (snat_is_session_static (s))
 179         goto delete;
 180
 181       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 182                                           &s->out2in.addr, s->out2in.port,
 183                                           s->nat_proto);
 184     delete:
 185       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 186       return 1;
 187     }
 188
 189   return 0;
 190 }
 191 #endif
 192
 193 // allocate exact address based on preference
 194 static_always_inline int
 195 nat_alloc_addr_and_port_exact (snat_address_t * a,
 196                                u32 thread_index,
 197                                nat_protocol_t proto,
 198                                ip4_address_t * addr,
 199                                u16 * port,
 200                                u16 port_per_thread, u32 snat_thread_index)
 201 {
 202   u32 portnum;
 203
 204   switch (proto)
 205     {
 206 #define _(N, j, n, s) \
 207     case NAT_PROTOCOL_##N: \
 208       if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
 209         { \
 210           while (1) \
 211             { \
 212               portnum = (port_per_thread * \
 213                 snat_thread_index) + \
 214                 snat_random_port(0, port_per_thread - 1) + 1024; \
 215               if (a->busy_##n##_port_refcounts[portnum]) \
 216                 continue; \
 217               --a->busy_##n##_port_refcounts[portnum]; \
 218               a->busy_##n##_ports_per_thread[thread_index]++; \
 219               a->busy_##n##_ports++; \
 220               *addr = a->addr; \
 221               *port = clib_host_to_net_u16(portnum); \
 222               return 0; \
 223             } \
 224         } \
 225       break;
 226       foreach_nat_protocol
 227 #undef _
 228     default:
 229       nat_elog_info ("unknown protocol");
 230       return 1;
 231     }
 232
 233   /* Totally out of translations to use... */
 234   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 235   return 1;
 236 }
 237
 238
 239 static snat_session_t *
 240 create_session_for_static_mapping_ed (snat_main_t * sm,
 241                                       vlib_buffer_t * b,
 242                                       ip4_address_t i2o_addr,
 243                                       u16 i2o_port,
 244                                       u32 i2o_fib_index,
 245                                       ip4_address_t o2i_addr,
 246                                       u16 o2i_port,
 247                                       u32 o2i_fib_index,
 248                                       nat_protocol_t nat_proto,
 249                                       vlib_node_runtime_t * node,
 250                                       u32 rx_fib_index,
 251                                       u32 thread_index,
 252                                       twice_nat_type_t twice_nat,
 253                                       lb_nat_type_t lb_nat, f64 now,
 254                                       snat_static_mapping_t * mapping)
 255 {
 256   snat_session_t *s;
 257   ip4_header_t *ip;
 258   udp_header_t *udp;
 259   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 260   clib_bihash_kv_16_8_t kv;
 261   nat44_is_idle_session_ctx_t ctx;
 262
 263   if (PREDICT_FALSE
 264       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 265     {
 266       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 267       nat_elog_notice ("maximum sessions exceeded");
 268       return 0;
 269     }
 270
 271   s = nat_ed_session_alloc (sm, thread_index, now, nat_proto);
 272   if (!s)
 273     {
 274       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 275       nat_elog_warn ("create NAT session failed");
 276       return 0;
 277     }
 278
 279   ip = vlib_buffer_get_current (b);
 280   udp = ip4_next_header (ip);
 281
 282   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 283   s->ext_host_port = nat_proto == NAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 284   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 285   if (lb_nat)
 286     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 287   if (lb_nat == AFFINITY_LB_NAT)
 288     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 289   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 290   s->out2in.addr = o2i_addr;
 291   s->out2in.port = o2i_port;
 292   s->out2in.fib_index = o2i_fib_index;
 293   s->in2out.addr = i2o_addr;
 294   s->in2out.port = i2o_port;
 295   s->in2out.fib_index = i2o_fib_index;
 296   s->nat_proto = nat_proto;
 297
 298   /* Add to lookup tables */
 299   init_ed_kv (&kv, o2i_addr, o2i_port, s->ext_host_addr, s->ext_host_port,
 300               o2i_fib_index, ip->protocol, thread_index, s - tsm->sessions);
 301   ctx.now = now;
 302   ctx.thread_index = thread_index;
 303   if (clib_bihash_add_or_overwrite_stale_16_8 (&sm->out2in_ed, &kv,
 304                                                nat44_o2i_ed_is_idle_session_cb,
 305                                                &ctx))
 306     nat_elog_notice ("out2in-ed key add failed");
 307
 308   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 309                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 310     {
 311       int rc = 0;
 312       snat_address_t *filter = 0;
 313
 314       // if exact address is specified use this address
 315       if (is_exact_address (mapping))
 316         {
 317           snat_address_t *ap;
 318           vec_foreach (ap, sm->twice_nat_addresses)
 319           {
 320             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 321               {
 322                 filter = ap;
 323                 break;
 324               }
 325           }
 326         }
 327
 328       if (filter)
 329         {
 330           rc = nat_alloc_addr_and_port_exact (filter,
 331                                               thread_index,
 332                                               nat_proto,
 333                                               &s->ext_host_nat_addr,
 334                                               &s->ext_host_nat_port,
 335                                               sm->port_per_thread,
 336                                               tsm->snat_thread_index);
 337           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 338         }
 339       else
 340         {
 341           rc =
 342             snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 343                                                  thread_index, nat_proto,
 344                                                  &s->ext_host_nat_addr,
 345                                                  &s->ext_host_nat_port,
 346                                                  sm->port_per_thread,
 347                                                  tsm->snat_thread_index);
 348         }
 349
 350       if (rc)
 351         {
 352           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 353           nat_ed_session_delete (sm, s, thread_index, 1);
 354           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 0))
 355             nat_elog_notice ("out2in-ed key del failed");
 356           return 0;
 357         }
 358
 359       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 360       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_nat_addr,
 361                   s->ext_host_nat_port, i2o_fib_index, ip->protocol,
 362                   thread_index, s - tsm->sessions);
 363     }
 364   else
 365     {
 366       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_addr,
 367                   s->ext_host_port, i2o_fib_index, ip->protocol,
 368                   thread_index, s - tsm->sessions);
 369     }
 370   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 371                                                nat44_i2o_ed_is_idle_session_cb,
 372                                                &ctx))
 373     nat_elog_notice ("in2out-ed key add failed");
 374
 375   snat_ipfix_logging_nat44_ses_create (thread_index,
 376                                        s->in2out.addr.as_u32,
 377                                        s->out2in.addr.as_u32,
 378                                        s->nat_proto,
 379                                        s->in2out.port,
 380                                        s->out2in.port, s->in2out.fib_index);
 381
 382   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 383                          &s->in2out.addr, s->in2out.port,
 384                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 385                          &s->out2in.addr, s->out2in.port,
 386                          &s->ext_host_addr, s->ext_host_port,
 387                          s->nat_proto, is_twice_nat_session (s));
 388
 389   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 390                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 391                &s->ext_host_nat_addr, s->ext_host_nat_port,
 392                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 393
 394   per_vrf_sessions_register_session (s, thread_index);
 395
 396   return s;
 397 }
 398
 399 static int
 400 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
 401               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 402 {
 403   clib_bihash_kv_16_8_t kv, value;
 404   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 405
 406   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 407              rx_fib_index, ip->protocol);
 408   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 409     return 1;
 410
 411   return 0;
 412 }
 413
 414 static void
 415 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
 416                        u32 rx_fib_index, u32 thread_index)
 417 {
 418   clib_bihash_kv_16_8_t kv, value;
 419   udp_header_t *udp;
 420   snat_session_t *s = 0;
 421   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 422   vlib_main_t *vm = vlib_get_main ();
 423   f64 now = vlib_time_now (vm);
 424   u16 l_port, r_port;
 425
 426   if (ip->protocol == IP_PROTOCOL_ICMP)
 427     {
 428       if (get_icmp_o2i_ed_key
 429           (b, ip, rx_fib_index, ~0, ~0, 0, &l_port, &r_port, &kv))
 430         return;
 431     }
 432   else
 433     {
 434       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 435         {
 436           udp = ip4_next_header (ip);
 437           l_port = udp->dst_port;
 438           r_port = udp->src_port;
 439         }
 440       else
 441         {
 442           l_port = 0;
 443           r_port = 0;
 444         }
 445       init_ed_k (&kv, ip->dst_address, l_port, ip->src_address, r_port,
 446                  rx_fib_index, ip->protocol);
 447     }
 448
 449   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 450     {
 451       ASSERT (thread_index == ed_value_get_thread_index (&value));
 452       s =
 453         pool_elt_at_index (tsm->sessions,
 454                            ed_value_get_session_index (&value));
 455     }
 456   else
 457     {
 458       u32 proto;
 459
 460       if (PREDICT_FALSE
 461           (nat44_ed_maximum_sessions_exceeded
 462            (sm, rx_fib_index, thread_index)))
 463         return;
 464
 465       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 466       if (!s)
 467         {
 468           nat_elog_warn ("create NAT session failed");
 469           return;
 470         }
 471
 472       proto = ip_proto_to_nat_proto (ip->protocol);
 473
 474       s->ext_host_addr = ip->src_address;
 475       s->ext_host_port = r_port;
 476       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 477       s->out2in.addr = ip->dst_address;
 478       s->out2in.port = l_port;
 479       s->nat_proto = proto;
 480       if (proto == NAT_PROTOCOL_OTHER)
 481         {
 482           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 483           s->out2in.port = ip->protocol;
 484         }
 485       s->out2in.fib_index = 0;
 486       s->in2out.addr = s->out2in.addr;
 487       s->in2out.port = s->out2in.port;
 488       s->in2out.fib_index = s->out2in.fib_index;
 489
 490       kv.value = s - tsm->sessions;
 491       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 492         nat_elog_notice ("in2out_ed key add failed");
 493
 494       per_vrf_sessions_register_session (s, thread_index);
 495     }
 496
 497   if (ip->protocol == IP_PROTOCOL_TCP)
 498     {
 499       tcp_header_t *tcp = ip4_next_header (ip);
 500       if (nat44_set_tcp_session_state_o2i
 501           (sm, now, s, tcp->flags, tcp->ack_number, tcp->seq_number,
 502            thread_index))
 503         return;
 504     }
 505
 506   /* Accounting */
 507   nat44_session_update_counters (s, now, 0, thread_index);
 508   /* Per-user LRU list maintenance */
 509   nat44_session_update_lru (sm, s, thread_index);
 510 }
 511
 512 static inline void
 513 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
 514                               ip4_header_t * ip, u32 rx_fib_index)
 515 {
 516   ip4_header_t ip_wkr = {
 517     .src_address = ip->dst_address,
 518   };
 519   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 520
 521   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 522 }
 523
 524 #ifndef CLIB_MARCH_VARIANT
 525 u32
 526 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 527                       u32 thread_index, vlib_buffer_t * b,
 528                       ip4_header_t * ip, ip4_address_t * addr,
 529                       u16 * port, u32 * fib_index, nat_protocol_t * proto,
 530                       void *d, void *e, u8 * dont_translate)
 531 {
 532   u32 next = ~0, sw_if_index, rx_fib_index;
 533   clib_bihash_kv_16_8_t kv, value;
 534   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 535   snat_session_t *s = 0;
 536   u8 is_addr_only, identity_nat;
 537   u16 l_port, r_port;
 538   vlib_main_t *vm = vlib_get_main ();
 539   ip4_address_t sm_addr;
 540   u16 sm_port;
 541   u32 sm_fib_index;
 542   *dont_translate = 0;
 543   snat_static_mapping_t *m;
 544
 545   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 546   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 547
 548   if (get_icmp_o2i_ed_key
 549       (b, ip, rx_fib_index, ~0, ~0, proto, &l_port, &r_port, &kv))
 550     {
 551       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 552       next = NAT_NEXT_DROP;
 553       goto out;
 554     }
 555
 556   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 557     {
 558       if (snat_static_mapping_match
 559           (sm, ip->dst_address, l_port, rx_fib_index,
 560            ip_proto_to_nat_proto (ip->protocol), &sm_addr, &sm_port,
 561            &sm_fib_index, 1, &is_addr_only, 0, 0, 0, &identity_nat, &m))
 562         {
 563           // static mapping not matched
 564           if (!sm->forwarding_enabled)
 565             {
 566               /* Don't NAT packet aimed at the intfc address */
 567               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 568                                                     ip->dst_address.as_u32)))
 569                 {
 570                   *dont_translate = 1;
 571                 }
 572               else
 573                 {
 574                   b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 575                   next = NAT_NEXT_DROP;
 576                 }
 577             }
 578           else
 579             {
 580               *dont_translate = 1;
 581               if (next_src_nat (sm, ip, l_port, r_port,
 582                                 thread_index, rx_fib_index))
 583                 {
 584                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 585                 }
 586               else
 587                 {
 588                   if (sm->num_workers > 1)
 589                     create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
 590                   else
 591                     create_bypass_for_fwd (sm, b, ip, rx_fib_index,
 592                                            thread_index);
 593                 }
 594             }
 595           goto out;
 596         }
 597
 598       if (PREDICT_FALSE
 599           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 600            ICMP4_echo_reply
 601            && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 602                ICMP4_echo_request || !is_addr_only)))
 603         {
 604           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 605           next = NAT_NEXT_DROP;
 606           goto out;
 607         }
 608
 609       if (PREDICT_FALSE (identity_nat))
 610         {
 611           *dont_translate = 1;
 612           goto out;
 613         }
 614
 615       /* Create session initiated by host from external network */
 616       s =
 617         create_session_for_static_mapping_ed (sm, b, sm_addr, sm_port,
 618                                               sm_fib_index, ip->dst_address,
 619                                               l_port, rx_fib_index, *proto,
 620                                               node, rx_fib_index,
 621                                               thread_index, 0, 0,
 622                                               vlib_time_now (vm), m);
 623       if (!s)
 624         next = NAT_NEXT_DROP;
 625     }
 626   else
 627     {
 628       if (PREDICT_FALSE
 629           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 630            ICMP4_echo_reply
 631            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 632            ICMP4_echo_request
 633            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 634                                            reass.icmp_type_or_tcp_flags)))
 635         {
 636           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 637           next = NAT_NEXT_DROP;
 638           goto out;
 639         }
 640
 641       ASSERT (thread_index == ed_value_get_thread_index (&value));
 642       s =
 643         pool_elt_at_index (tsm->sessions,
 644                            ed_value_get_session_index (&value));
 645     }
 646 out:
 647   if (s)
 648     {
 649       *addr = s->in2out.addr;
 650       *port = s->in2out.port;
 651       *fib_index = s->in2out.fib_index;
 652     }
 653   if (d)
 654     *(snat_session_t **) d = s;
 655   return next;
 656 }
 657 #endif
 658
 659 static snat_session_t *
 660 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 661                                vlib_buffer_t * b,
 662                                ip4_header_t * ip,
 663                                u32 rx_fib_index,
 664                                u32 thread_index,
 665                                f64 now,
 666                                vlib_main_t * vm, vlib_node_runtime_t * node)
 667 {
 668   clib_bihash_kv_8_8_t kv, value;
 669   clib_bihash_kv_16_8_t s_kv, s_value;
 670   snat_static_mapping_t *m;
 671   u32 old_addr, new_addr;
 672   ip_csum_t sum;
 673   snat_session_t *s;
 674   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 675
 676   old_addr = ip->dst_address.as_u32;
 677
 678   init_ed_k (&s_kv, ip->dst_address, 0, ip->src_address, 0, rx_fib_index,
 679              ip->protocol);
 680
 681   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 682     {
 683       ASSERT (thread_index == ed_value_get_thread_index (&s_value));
 684       s =
 685         pool_elt_at_index (tsm->sessions,
 686                            ed_value_get_session_index (&s_value));
 687       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 688     }
 689   else
 690     {
 691       if (PREDICT_FALSE
 692           (nat44_ed_maximum_sessions_exceeded
 693            (sm, rx_fib_index, thread_index)))
 694         {
 695           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 696           nat_elog_notice ("maximum sessions exceeded");
 697           return 0;
 698         }
 699
 700       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 701       if (clib_bihash_search_8_8
 702           (&sm->static_mapping_by_external, &kv, &value))
 703         {
 704           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 705           return 0;
 706         }
 707
 708       m = pool_elt_at_index (sm->static_mappings, value.value);
 709
 710       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 711
 712       /* Create a new session */
 713       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 714       if (!s)
 715         {
 716           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 717           nat_elog_warn ("create NAT session failed");
 718           return 0;
 719         }
 720
 721       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 722       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 723       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 724       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 725       s->out2in.addr.as_u32 = old_addr;
 726       s->out2in.fib_index = rx_fib_index;
 727       s->in2out.addr.as_u32 = new_addr;
 728       s->in2out.fib_index = m->fib_index;
 729       s->in2out.port = s->out2in.port = ip->protocol;
 730
 731       /* Add to lookup tables */
 732       s_kv.value = s - tsm->sessions;
 733       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 734         nat_elog_notice ("out2in key add failed");
 735
 736       init_ed_kv (&s_kv, ip->dst_address, 0, ip->src_address, 0, m->fib_index,
 737                   ip->protocol, thread_index, s - tsm->sessions);
 738       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 739         nat_elog_notice ("in2out key add failed");
 740
 741       per_vrf_sessions_register_session (s, thread_index);
 742     }
 743
 744   /* Update IP checksum */
 745   sum = ip->checksum;
 746   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 747   ip->checksum = ip_csum_fold (sum);
 748
 749   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 750
 751   /* Accounting */
 752   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 753                                  thread_index);
 754   /* Per-user LRU list maintenance */
 755   nat44_session_update_lru (sm, s, thread_index);
 756
 757   return s;
 758 }
 759
 760 static inline uword
 761 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 762                                           vlib_node_runtime_t * node,
 763                                           vlib_frame_t * frame,
 764                                           int is_multi_worker)
 765 {
 766   u32 n_left_from, *from;
 767   snat_main_t *sm = &snat_main;
 768   f64 now = vlib_time_now (vm);
 769   u32 thread_index = vm->thread_index;
 770   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 771
 772   from = vlib_frame_vector_args (frame);
 773   n_left_from = frame->n_vectors;
 774
 775   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 776   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 777   vlib_get_buffers (vm, from, b, n_left_from);
 778
 779   while (n_left_from > 0)
 780     {
 781       vlib_buffer_t *b0;
 782       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
 783       u16 old_port0, new_port0;
 784       ip4_header_t *ip0;
 785       udp_header_t *udp0;
 786       tcp_header_t *tcp0;
 787       snat_session_t *s0 = 0;
 788       clib_bihash_kv_16_8_t kv0, value0;
 789       ip_csum_t sum0;
 790
 791       b0 = *b;
 792       b++;
 793
 794       /* Prefetch next iteration. */
 795       if (PREDICT_TRUE (n_left_from >= 2))
 796         {
 797           vlib_buffer_t *p2;
 798
 799           p2 = *b;
 800
 801           vlib_prefetch_buffer_header (p2, LOAD);
 802
 803           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 804         }
 805
 806       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 807
 808       vnet_buffer (b0)->snat.flags = 0;
 809       ip0 = vlib_buffer_get_current (b0);
 810
 811       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 812       rx_fib_index0 =
 813         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 814
 815       if (PREDICT_FALSE (ip0->ttl == 1))
 816         {
 817           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 818           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 819                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 820                                        0);
 821           next[0] = NAT_NEXT_ICMP_ERROR;
 822           goto trace0;
 823         }
 824
 825       udp0 = ip4_next_header (ip0);
 826       tcp0 = (tcp_header_t *) udp0;
 827       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 828
 829       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 830         {
 831           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 832           goto trace0;
 833         }
 834
 835       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 836         {
 837           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 838           goto trace0;
 839         }
 840
 841       init_ed_k (&kv0, ip0->dst_address,
 842                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
 843                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
 844                  ip0->protocol);
 845
 846       /* there is a stashed index in vnet_buffer2 from handoff node,
 847        * see if we can use it */
 848       if (is_multi_worker
 849           &&
 850           PREDICT_TRUE (!pool_is_free_index
 851                         (tsm->sessions,
 852                          vnet_buffer2 (b0)->nat.ed_out2in_nat_session_index)))
 853         {
 854           s0 = pool_elt_at_index (tsm->sessions,
 855                                   vnet_buffer2 (b0)->
 856                                   nat.ed_out2in_nat_session_index);
 857           if (PREDICT_TRUE
 858               (s0->out2in.addr.as_u32 == ip0->dst_address.as_u32
 859                && s0->out2in.port == vnet_buffer (b0)->ip.reass.l4_dst_port
 860                && s0->nat_proto == ip_proto_to_nat_proto (ip0->protocol)
 861                && s0->out2in.fib_index == rx_fib_index0
 862                && s0->ext_host_addr.as_u32 == ip0->src_address.as_u32
 863                && s0->ext_host_port ==
 864                vnet_buffer (b0)->ip.reass.l4_src_port))
 865             {
 866               /* yes, this is the droid we're looking for */
 867               goto skip_lookup;
 868             }
 869         }
 870
 871       // lookup for session
 872       if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
 873         {
 874           // session does not exist go slow path
 875           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 876           goto trace0;
 877         }
 878       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 879       s0 =
 880         pool_elt_at_index (tsm->sessions,
 881                            ed_value_get_session_index (&value0));
 882
 883     skip_lookup:
 884
 885       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 886         {
 887           // session is closed, go slow path
 888           nat_free_session_data (sm, s0, thread_index, 0);
 889           nat_ed_session_delete (sm, s0, thread_index, 1);
 890           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 891           goto trace0;
 892         }
 893
 894       if (s0->tcp_closed_timestamp)
 895         {
 896           if (now >= s0->tcp_closed_timestamp)
 897             {
 898               // session is closed, go slow path, freed in slow path
 899               next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 900             }
 901           else
 902             {
 903               // session in transitory timeout, drop
 904               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 905               next[0] = NAT_NEXT_DROP;
 906             }
 907           goto trace0;
 908         }
 909
 910       // drop if session expired
 911       u64 sess_timeout_time;
 912       sess_timeout_time =
 913         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 914       if (now >= sess_timeout_time)
 915         {
 916           // session is closed, go slow path
 917           nat_free_session_data (sm, s0, thread_index, 0);
 918           nat_ed_session_delete (sm, s0, thread_index, 1);
 919           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 920           goto trace0;
 921         }
 922
 923       old_addr0 = ip0->dst_address.as_u32;
 924       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 925       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 926
 927       sum0 = ip0->checksum;
 928       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 929                              dst_address);
 930       if (PREDICT_FALSE (is_twice_nat_session (s0)))
 931         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 932                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
 933                                src_address);
 934       ip0->checksum = ip_csum_fold (sum0);
 935
 936       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 937
 938       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 939         {
 940           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 941             {
 942               new_port0 = udp0->dst_port = s0->in2out.port;
 943               sum0 = tcp0->checksum;
 944               sum0 =
 945                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 946                                 dst_address);
 947               sum0 =
 948                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 949                                 length);
 950               if (is_twice_nat_session (s0))
 951                 {
 952                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 953                                          s0->ext_host_nat_addr.as_u32,
 954                                          ip4_header_t, dst_address);
 955                   sum0 =
 956                     ip_csum_update (sum0,
 957                                     vnet_buffer (b0)->ip.reass.l4_src_port,
 958                                     s0->ext_host_nat_port, ip4_header_t,
 959                                     length);
 960                   tcp0->src_port = s0->ext_host_nat_port;
 961                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 962                 }
 963               tcp0->checksum = ip_csum_fold (sum0);
 964             }
 965           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.tcp,
 966                                          thread_index, sw_if_index0, 1);
 967           if (nat44_set_tcp_session_state_o2i
 968               (sm, now, s0, vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
 969                vnet_buffer (b0)->ip.reass.tcp_ack_number,
 970                vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
 971             goto trace0;
 972         }
 973       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
 974                && udp0->checksum)
 975         {
 976           new_port0 = udp0->dst_port = s0->in2out.port;
 977           sum0 = udp0->checksum;
 978           sum0 =
 979             ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 980                             dst_address);
 981           sum0 =
 982             ip_csum_update (sum0, old_port0, new_port0, ip4_header_t, length);
 983           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 984             {
 985               sum0 =
 986                 ip_csum_update (sum0, ip0->src_address.as_u32,
 987                                 s0->ext_host_nat_addr.as_u32, ip4_header_t,
 988                                 dst_address);
 989               sum0 =
 990                 ip_csum_update (sum0, vnet_buffer (b0)->ip.reass.l4_src_port,
 991                                 s0->ext_host_nat_port, ip4_header_t, length);
 992               udp0->src_port = s0->ext_host_nat_port;
 993               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 994             }
 995           udp0->checksum = ip_csum_fold (sum0);
 996           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
 997                                          thread_index, sw_if_index0, 1);
 998         }
 999       else
1000         {
1001           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1002             {
1003               new_port0 = udp0->dst_port = s0->in2out.port;
1004               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1005                 {
1006                   udp0->src_port = s0->ext_host_nat_port;
1007                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1008                 }
1009             }
1010           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
1011                                          thread_index, sw_if_index0, 1);
1012         }
1013
1014       /* Accounting */
1015       nat44_session_update_counters (s0, now,
1016                                      vlib_buffer_length_in_chain (vm, b0),
1017                                      thread_index);
1018       /* Per-user LRU list maintenance */
1019       nat44_session_update_lru (sm, s0, thread_index);
1020
1021     trace0:
1022       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1023                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1024         {
1025           nat44_ed_out2in_trace_t *t =
1026             vlib_add_trace (vm, node, b0, sizeof (*t));
1027           t->sw_if_index = sw_if_index0;
1028           t->next_index = next[0];
1029           t->is_slow_path = 0;
1030
1031           if (s0)
1032             t->session_index = s0 - tsm->sessions;
1033           else
1034             t->session_index = ~0;
1035         }
1036
1037       if (next[0] == NAT_NEXT_DROP)
1038         {
1039           vlib_increment_simple_counter (&sm->counters.fastpath.
1040                                          out2in_ed.drops, thread_index,
1041                                          sw_if_index0, 1);
1042         }
1043
1044       n_left_from--;
1045       next++;
1046     }
1047
1048   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1049                                frame->n_vectors);
1050   return frame->n_vectors;
1051 }
1052
1053 static inline uword
1054 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1055                                           vlib_node_runtime_t * node,
1056                                           vlib_frame_t * frame)
1057 {
1058   u32 n_left_from, *from;
1059   snat_main_t *sm = &snat_main;
1060   f64 now = vlib_time_now (vm);
1061   u32 thread_index = vm->thread_index;
1062   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1063   snat_static_mapping_t *m;
1064
1065   from = vlib_frame_vector_args (frame);
1066   n_left_from = frame->n_vectors;
1067
1068   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1069   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1070   vlib_get_buffers (vm, from, b, n_left_from);
1071
1072   while (n_left_from > 0)
1073     {
1074       vlib_buffer_t *b0;
1075       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
1076       u16 old_port0, new_port0;
1077       ip4_header_t *ip0;
1078       udp_header_t *udp0;
1079       tcp_header_t *tcp0;
1080       icmp46_header_t *icmp0;
1081       snat_session_t *s0 = 0;
1082       clib_bihash_kv_16_8_t kv0, value0;
1083       ip_csum_t sum0;
1084       lb_nat_type_t lb_nat0;
1085       twice_nat_type_t twice_nat0;
1086       u8 identity_nat0;
1087       ip4_address_t sm_addr;
1088       u16 sm_port;
1089       u32 sm_fib_index;
1090
1091       b0 = *b;
1092       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1093
1094       vnet_buffer (b0)->snat.flags = 0;
1095       ip0 = vlib_buffer_get_current (b0);
1096
1097       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1098       rx_fib_index0 =
1099         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1100
1101       if (PREDICT_FALSE (ip0->ttl == 1))
1102         {
1103           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1104           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1105                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1106                                        0);
1107           next[0] = NAT_NEXT_ICMP_ERROR;
1108           goto trace0;
1109         }
1110
1111       udp0 = ip4_next_header (ip0);
1112       tcp0 = (tcp_header_t *) udp0;
1113       icmp0 = (icmp46_header_t *) udp0;
1114       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1115
1116       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1117         {
1118           s0 =
1119             nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1120                                            thread_index, now, vm, node);
1121           if (!sm->forwarding_enabled)
1122             {
1123               if (!s0)
1124                 next[0] = NAT_NEXT_DROP;
1125             }
1126           vlib_increment_simple_counter (&sm->counters.slowpath.
1127                                          out2in_ed.other, thread_index,
1128                                          sw_if_index0, 1);
1129           goto trace0;
1130         }
1131
1132       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1133         {
1134           next[0] = icmp_out2in_ed_slow_path
1135             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1136              next[0], now, thread_index, &s0);
1137           vlib_increment_simple_counter (&sm->counters.slowpath.
1138                                          out2in_ed.icmp, thread_index,
1139                                          sw_if_index0, 1);
1140           goto trace0;
1141         }
1142
1143       init_ed_k (&kv0, ip0->dst_address,
1144                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
1145                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1146                  ip0->protocol);
1147
1148       s0 = NULL;
1149       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
1150         {
1151           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1152           s0 =
1153             pool_elt_at_index (tsm->sessions,
1154                                ed_value_get_session_index (&value0));
1155
1156           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1157             {
1158               nat_free_session_data (sm, s0, thread_index, 0);
1159               nat_ed_session_delete (sm, s0, thread_index, 1);
1160               s0 = NULL;
1161             }
1162         }
1163
1164       if (!s0)
1165         {
1166           /* Try to match static mapping by external address and port,
1167              destination address and port in packet */
1168
1169           if (snat_static_mapping_match
1170               (sm, ip0->dst_address,
1171                vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1172                proto0, &sm_addr, &sm_port, &sm_fib_index, 1, 0,
1173                &twice_nat0, &lb_nat0, &ip0->src_address, &identity_nat0, &m))
1174             {
1175               /*
1176                * Send DHCP packets to the ipv4 stack, or we won't
1177                * be able to use dhcp client on the outside interface
1178                */
1179               if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_UDP
1180                                  && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1181                                      clib_host_to_net_u16
1182                                      (UDP_DST_PORT_dhcp_to_client))))
1183                 {
1184                   goto trace0;
1185                 }
1186
1187               if (!sm->forwarding_enabled)
1188                 {
1189                   b0->error =
1190                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1191                   next[0] = NAT_NEXT_DROP;
1192                 }
1193               else
1194                 {
1195                   if (next_src_nat
1196                       (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1197                        vnet_buffer (b0)->ip.reass.l4_dst_port,
1198                        thread_index, rx_fib_index0))
1199                     {
1200                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1201                     }
1202                   else
1203                     {
1204                       if (sm->num_workers > 1)
1205                         create_bypass_for_fwd_worker (sm, b0, ip0,
1206                                                       rx_fib_index0);
1207                       else
1208                         create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1209                                                thread_index);
1210                     }
1211                 }
1212               goto trace0;
1213             }
1214
1215           if (PREDICT_FALSE (identity_nat0))
1216             goto trace0;
1217
1218           if ((proto0 == NAT_PROTOCOL_TCP)
1219               && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1220                                      reass.icmp_type_or_tcp_flags))
1221             {
1222               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1223               next[0] = NAT_NEXT_DROP;
1224               goto trace0;
1225             }
1226
1227           /* Create session initiated by host from external network */
1228           s0 = create_session_for_static_mapping_ed (sm, b0,
1229                                                      sm_addr, sm_port,
1230                                                      sm_fib_index,
1231                                                      ip0->dst_address,
1232                                                      vnet_buffer (b0)->
1233                                                      ip.reass.l4_dst_port,
1234                                                      rx_fib_index0, proto0,
1235                                                      node, rx_fib_index0,
1236                                                      thread_index, twice_nat0,
1237                                                      lb_nat0, now, m);
1238           if (!s0)
1239             {
1240               next[0] = NAT_NEXT_DROP;
1241               goto trace0;
1242             }
1243         }
1244
1245       old_addr0 = ip0->dst_address.as_u32;
1246       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1247       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1248
1249       sum0 = ip0->checksum;
1250       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1251                              dst_address);
1252       if (PREDICT_FALSE (is_twice_nat_session (s0)))
1253         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1254                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
1255                                src_address);
1256       ip0->checksum = ip_csum_fold (sum0);
1257
1258       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1259
1260       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1261         {
1262           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1263             {
1264               new_port0 = udp0->dst_port = s0->in2out.port;
1265               sum0 = tcp0->checksum;
1266               sum0 =
1267                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1268                                 dst_address);
1269               sum0 =
1270                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1271                                 length);
1272               if (is_twice_nat_session (s0))
1273                 {
1274                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1275                                          s0->ext_host_nat_addr.as_u32,
1276                                          ip4_header_t, dst_address);
1277                   sum0 =
1278                     ip_csum_update (sum0,
1279                                     vnet_buffer (b0)->ip.reass.l4_src_port,
1280                                     s0->ext_host_nat_port, ip4_header_t,
1281                                     length);
1282                   tcp0->src_port = s0->ext_host_nat_port;
1283                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1284                 }
1285               tcp0->checksum = ip_csum_fold (sum0);
1286             }
1287           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.tcp,
1288                                          thread_index, sw_if_index0, 1);
1289           if (nat44_set_tcp_session_state_o2i
1290               (sm, now, s0, vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
1291                vnet_buffer (b0)->ip.reass.tcp_ack_number,
1292                vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
1293             goto trace0;
1294         }
1295       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1296                && udp0->checksum)
1297         {
1298           new_port0 = udp0->dst_port = s0->in2out.port;
1299           sum0 = udp0->checksum;
1300           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1301                                  dst_address);
1302           sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1303                                  length);
1304           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1305             {
1306               sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1307                                      s0->ext_host_nat_addr.as_u32,
1308                                      ip4_header_t, dst_address);
1309               sum0 =
1310                 ip_csum_update (sum0,
1311                                 vnet_buffer (b0)->ip.reass.l4_src_port,
1312                                 s0->ext_host_nat_port, ip4_header_t, length);
1313               udp0->src_port = s0->ext_host_nat_port;
1314               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1315             }
1316           udp0->checksum = ip_csum_fold (sum0);
1317           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1318                                          thread_index, sw_if_index0, 1);
1319         }
1320       else
1321         {
1322           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1323             {
1324               new_port0 = udp0->dst_port = s0->in2out.port;
1325               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1326                 {
1327                   udp0->src_port = s0->ext_host_nat_port;
1328                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1329                 }
1330             }
1331           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1332                                          thread_index, sw_if_index0, 1);
1333         }
1334
1335       /* Accounting */
1336       nat44_session_update_counters (s0, now,
1337                                      vlib_buffer_length_in_chain (vm, b0),
1338                                      thread_index);
1339       /* Per-user LRU list maintenance */
1340       nat44_session_update_lru (sm, s0, thread_index);
1341
1342     trace0:
1343       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1344                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1345         {
1346           nat44_ed_out2in_trace_t *t =
1347             vlib_add_trace (vm, node, b0, sizeof (*t));
1348           t->sw_if_index = sw_if_index0;
1349           t->next_index = next[0];
1350           t->is_slow_path = 1;
1351
1352           if (s0)
1353             t->session_index = s0 - tsm->sessions;
1354           else
1355             t->session_index = ~0;
1356         }
1357
1358       if (next[0] == NAT_NEXT_DROP)
1359         {
1360           vlib_increment_simple_counter (&sm->counters.slowpath.
1361                                          out2in_ed.drops, thread_index,
1362                                          sw_if_index0, 1);
1363         }
1364
1365       n_left_from--;
1366       next++;
1367       b++;
1368     }
1369
1370   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1371                                frame->n_vectors);
1372
1373   return frame->n_vectors;
1374 }
1375
1376 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1377                                      vlib_node_runtime_t * node,
1378                                      vlib_frame_t * frame)
1379 {
1380   if (snat_main.num_workers > 1)
1381     {
1382       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1383     }
1384   else
1385     {
1386       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1387     }
1388 }
1389
1390 /* *INDENT-OFF* */
1391 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1392   .name = "nat44-ed-out2in",
1393   .vector_size = sizeof (u32),
1394   .sibling_of = "nat-default",
1395   .format_trace = format_nat44_ed_out2in_trace,
1396   .type = VLIB_NODE_TYPE_INTERNAL,
1397   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1398   .error_strings = nat_out2in_ed_error_strings,
1399   .runtime_data_bytes = sizeof (snat_runtime_t),
1400 };
1401 /* *INDENT-ON* */
1402
1403 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1404                                               vlib_node_runtime_t * node,
1405                                               vlib_frame_t * frame)
1406 {
1407   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1408 }
1409
1410 /* *INDENT-OFF* */
1411 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1412   .name = "nat44-ed-out2in-slowpath",
1413   .vector_size = sizeof (u32),
1414   .sibling_of = "nat-default",
1415   .format_trace = format_nat44_ed_out2in_trace,
1416   .type = VLIB_NODE_TYPE_INTERNAL,
1417   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1418   .error_strings = nat_out2in_ed_error_strings,
1419   .runtime_data_bytes = sizeof (snat_runtime_t),
1420 };
1421 /* *INDENT-ON* */
1422
1423 static u8 *
1424 format_nat_pre_trace (u8 * s, va_list * args)
1425 {
1426   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1427   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1428   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1429   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1430                  t->arc_next_index);
1431 }
1432
1433 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1434                                     vlib_node_runtime_t * node,
1435                                     vlib_frame_t * frame)
1436 {
1437   return nat_pre_node_fn_inline (vm, node, frame,
1438                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1439 }
1440
1441 /* *INDENT-OFF* */
1442 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1443   .name = "nat-pre-out2in",
1444   .vector_size = sizeof (u32),
1445   .sibling_of = "nat-default",
1446   .format_trace = format_nat_pre_trace,
1447   .type = VLIB_NODE_TYPE_INTERNAL,
1448   .n_errors = 0,
1449  };
1450 /* *INDENT-ON* */
1451
1452 /*
1453  * fd.io coding-style-patch-verification: ON
1454  *
1455  * Local Variables:
1456  * eval: (c-set-style "gnu")
1457  * End:
1458  */