src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat44/inlines.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <nat/nat44/ed_inlines.h>
  34
  35 static char *nat_out2in_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_out2in_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat44_ed_out2in_trace_t;
  48
  49 static u8 *
  50 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  59     "NAT44_OUT2IN_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 static inline u32
  68 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  69                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  70                           u32 sw_if_index0, u32 rx_fib_index0,
  71                           vlib_node_runtime_t * node, u32 next0, f64 now,
  72                           u32 thread_index, snat_session_t ** p_s0)
  73 {
  74   vlib_main_t *vm = vlib_get_main ();
  75
  76   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  77                        next0, thread_index, p_s0, 0);
  78   snat_session_t *s0 = *p_s0;
  79   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  80     {
  81       /* Accounting */
  82       nat44_session_update_counters (s0, now,
  83                                      vlib_buffer_length_in_chain
  84                                      (vm, b0), thread_index);
  85       /* Per-user LRU list maintenance */
  86       nat44_session_update_lru (sm, s0, thread_index);
  87     }
  88   return next0;
  89 }
  90
  91 #ifndef CLIB_MARCH_VARIANT
  92 int
  93 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  94 {
  95   snat_main_t *sm = &snat_main;
  96   nat44_is_idle_session_ctx_t *ctx = arg;
  97   snat_session_t *s;
  98   u64 sess_timeout_time;
  99   u8 proto;
 100   u16 r_port, l_port;
 101   ip4_address_t *l_addr, *r_addr;
 102   u32 fib_index;
 103   clib_bihash_kv_16_8_t ed_kv;
 104   int i;
 105   //snat_address_t *a;
 106   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 107                                                        ctx->thread_index);
 108
 109   s = pool_elt_at_index (tsm->sessions, kv->value);
 110   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 111   if (ctx->now >= sess_timeout_time)
 112     {
 113       l_addr = &s->in2out.addr;
 114       r_addr = &s->ext_host_addr;
 115       fib_index = s->in2out.fib_index;
 116       if (snat_is_unk_proto_session (s))
 117         {
 118           proto = s->in2out.port;
 119           r_port = 0;
 120           l_port = 0;
 121         }
 122       else
 123         {
 124           proto = nat_proto_to_ip_proto (s->nat_proto);
 125           l_port = s->in2out.port;
 126           r_port = s->ext_host_port;
 127         }
 128       if (is_twice_nat_session (s))
 129         {
 130           r_addr = &s->ext_host_nat_addr;
 131           r_port = s->ext_host_nat_port;
 132         }
 133       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 134       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 135         nat_elog_warn ("in2out_ed key del failed");
 136
 137       if (snat_is_unk_proto_session (s))
 138         goto delete;
 139
 140       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 141                                            s->in2out.addr.as_u32,
 142                                            s->out2in.addr.as_u32,
 143                                            s->nat_proto,
 144                                            s->in2out.port,
 145                                            s->out2in.port,
 146                                            s->in2out.fib_index);
 147
 148       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 149                              &s->in2out.addr, s->in2out.port,
 150                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 151                              &s->out2in.addr, s->out2in.port,
 152                              &s->ext_host_addr, s->ext_host_port,
 153                              s->nat_proto, is_twice_nat_session (s));
 154
 155       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 156                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 157                    ctx->thread_index);
 158
 159       if (is_twice_nat_session (s))
 160         {
 161           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 162             {
 163               // FIXME TODO this is obviously wrong code ... needs fix!
 164               //       key.protocol = s->nat_proto;
 165               //       key.port = s->ext_host_nat_port;
 166               //       a = sm->twice_nat_addresses + i;
 167               //       if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 168               //      {
 169               //        snat_free_outside_address_and_port (sm->twice_nat_addresses,
 170               //                                            ctx->thread_index,
 171               //                                            &key);
 172               //        break;
 173               //      }
 174             }
 175         }
 176
 177       if (snat_is_session_static (s))
 178         goto delete;
 179
 180       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 181                                           &s->out2in.addr, s->out2in.port,
 182                                           s->nat_proto);
 183     delete:
 184       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 185       return 1;
 186     }
 187
 188   return 0;
 189 }
 190 #endif
 191
 192 // allocate exact address based on preference
 193 static_always_inline int
 194 nat_alloc_addr_and_port_exact (snat_address_t * a,
 195                                u32 thread_index,
 196                                nat_protocol_t proto,
 197                                ip4_address_t * addr,
 198                                u16 * port,
 199                                u16 port_per_thread, u32 snat_thread_index)
 200 {
 201   u32 portnum;
 202
 203   switch (proto)
 204     {
 205 #define _(N, j, n, s) \
 206     case NAT_PROTOCOL_##N: \
 207       if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
 208         { \
 209           while (1) \
 210             { \
 211               portnum = (port_per_thread * \
 212                 snat_thread_index) + \
 213                 snat_random_port(0, port_per_thread - 1) + 1024; \
 214               if (a->busy_##n##_port_refcounts[portnum]) \
 215                 continue; \
 216               --a->busy_##n##_port_refcounts[portnum]; \
 217               a->busy_##n##_ports_per_thread[thread_index]++; \
 218               a->busy_##n##_ports++; \
 219               *addr = a->addr; \
 220               *port = clib_host_to_net_u16(portnum); \
 221               return 0; \
 222             } \
 223         } \
 224       break;
 225       foreach_nat_protocol
 226 #undef _
 227     default:
 228       nat_elog_info ("unknown protocol");
 229       return 1;
 230     }
 231
 232   /* Totally out of translations to use... */
 233   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
 234   return 1;
 235 }
 236
 237
 238 static snat_session_t *
 239 create_session_for_static_mapping_ed (snat_main_t * sm,
 240                                       vlib_buffer_t * b,
 241                                       ip4_address_t i2o_addr,
 242                                       u16 i2o_port,
 243                                       u32 i2o_fib_index,
 244                                       ip4_address_t o2i_addr,
 245                                       u16 o2i_port,
 246                                       u32 o2i_fib_index,
 247                                       nat_protocol_t nat_proto,
 248                                       vlib_node_runtime_t * node,
 249                                       u32 rx_fib_index,
 250                                       u32 thread_index,
 251                                       twice_nat_type_t twice_nat,
 252                                       lb_nat_type_t lb_nat, f64 now,
 253                                       snat_static_mapping_t * mapping)
 254 {
 255   snat_session_t *s;
 256   ip4_header_t *ip;
 257   udp_header_t *udp;
 258   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 259   clib_bihash_kv_16_8_t kv;
 260   nat44_is_idle_session_ctx_t ctx;
 261
 262   if (PREDICT_FALSE
 263       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 264     {
 265       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 266       nat_elog_notice ("maximum sessions exceeded");
 267       return 0;
 268     }
 269
 270   s = nat_ed_session_alloc (sm, thread_index, now, nat_proto);
 271   if (!s)
 272     {
 273       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 274       nat_elog_warn ("create NAT session failed");
 275       return 0;
 276     }
 277
 278   ip = vlib_buffer_get_current (b);
 279   udp = ip4_next_header (ip);
 280
 281   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 282   s->ext_host_port = nat_proto == NAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 283   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 284   if (lb_nat)
 285     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 286   if (lb_nat == AFFINITY_LB_NAT)
 287     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 288   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 289   s->out2in.addr = o2i_addr;
 290   s->out2in.port = o2i_port;
 291   s->out2in.fib_index = o2i_fib_index;
 292   s->in2out.addr = i2o_addr;
 293   s->in2out.port = i2o_port;
 294   s->in2out.fib_index = i2o_fib_index;
 295   s->nat_proto = nat_proto;
 296
 297   /* Add to lookup tables */
 298   init_ed_kv (&kv, o2i_addr, o2i_port, s->ext_host_addr, s->ext_host_port,
 299               o2i_fib_index, ip->protocol, thread_index, s - tsm->sessions);
 300   ctx.now = now;
 301   ctx.thread_index = thread_index;
 302   if (clib_bihash_add_or_overwrite_stale_16_8 (&sm->out2in_ed, &kv,
 303                                                nat44_o2i_ed_is_idle_session_cb,
 304                                                &ctx))
 305     nat_elog_notice ("out2in-ed key add failed");
 306
 307   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 308                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 309     {
 310       int rc = 0;
 311       snat_address_t *filter = 0;
 312
 313       // if exact address is specified use this address
 314       if (is_exact_address (mapping))
 315         {
 316           snat_address_t *ap;
 317           vec_foreach (ap, sm->twice_nat_addresses)
 318           {
 319             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 320               {
 321                 filter = ap;
 322                 break;
 323               }
 324           }
 325         }
 326
 327       if (filter)
 328         {
 329           rc = nat_alloc_addr_and_port_exact (filter,
 330                                               thread_index,
 331                                               nat_proto,
 332                                               &s->ext_host_nat_addr,
 333                                               &s->ext_host_nat_port,
 334                                               sm->port_per_thread,
 335                                               tsm->snat_thread_index);
 336           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 337         }
 338       else
 339         {
 340           rc =
 341             snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 342                                                  thread_index, nat_proto,
 343                                                  &s->ext_host_nat_addr,
 344                                                  &s->ext_host_nat_port,
 345                                                  sm->port_per_thread,
 346                                                  tsm->snat_thread_index);
 347         }
 348
 349       if (rc)
 350         {
 351           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 352           nat_ed_session_delete (sm, s, thread_index, 1);
 353           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 0))
 354             nat_elog_notice ("out2in-ed key del failed");
 355           return 0;
 356         }
 357
 358       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 359       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_nat_addr,
 360                   s->ext_host_nat_port, i2o_fib_index, ip->protocol,
 361                   thread_index, s - tsm->sessions);
 362     }
 363   else
 364     {
 365       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_addr,
 366                   s->ext_host_port, i2o_fib_index, ip->protocol,
 367                   thread_index, s - tsm->sessions);
 368     }
 369   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 370                                                nat44_i2o_ed_is_idle_session_cb,
 371                                                &ctx))
 372     nat_elog_notice ("in2out-ed key add failed");
 373
 374   snat_ipfix_logging_nat44_ses_create (thread_index,
 375                                        s->in2out.addr.as_u32,
 376                                        s->out2in.addr.as_u32,
 377                                        s->nat_proto,
 378                                        s->in2out.port,
 379                                        s->out2in.port, s->in2out.fib_index);
 380
 381   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 382                          &s->in2out.addr, s->in2out.port,
 383                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 384                          &s->out2in.addr, s->out2in.port,
 385                          &s->ext_host_addr, s->ext_host_port,
 386                          s->nat_proto, is_twice_nat_session (s));
 387
 388   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 389                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 390                &s->ext_host_nat_addr, s->ext_host_nat_port,
 391                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 392
 393   per_vrf_sessions_register_session (s, thread_index);
 394
 395   return s;
 396 }
 397
 398 static int
 399 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
 400               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 401 {
 402   clib_bihash_kv_16_8_t kv, value;
 403   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 404
 405   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 406              rx_fib_index, ip->protocol);
 407   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 408     return 1;
 409
 410   return 0;
 411 }
 412
 413 static void
 414 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
 415                        u32 rx_fib_index, u32 thread_index)
 416 {
 417   clib_bihash_kv_16_8_t kv, value;
 418   udp_header_t *udp;
 419   snat_session_t *s = 0;
 420   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 421   vlib_main_t *vm = vlib_get_main ();
 422   f64 now = vlib_time_now (vm);
 423   u16 l_port, r_port;
 424
 425   if (ip->protocol == IP_PROTOCOL_ICMP)
 426     {
 427       if (get_icmp_o2i_ed_key
 428           (b, ip, rx_fib_index, ~0, ~0, 0, &l_port, &r_port, &kv))
 429         return;
 430     }
 431   else
 432     {
 433       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 434         {
 435           udp = ip4_next_header (ip);
 436           l_port = udp->dst_port;
 437           r_port = udp->src_port;
 438         }
 439       else
 440         {
 441           l_port = 0;
 442           r_port = 0;
 443         }
 444       init_ed_k (&kv, ip->dst_address, l_port, ip->src_address, r_port,
 445                  rx_fib_index, ip->protocol);
 446     }
 447
 448   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 449     {
 450       ASSERT (thread_index == ed_value_get_thread_index (&value));
 451       s =
 452         pool_elt_at_index (tsm->sessions,
 453                            ed_value_get_session_index (&value));
 454     }
 455   else
 456     {
 457       u32 proto;
 458
 459       if (PREDICT_FALSE
 460           (nat44_ed_maximum_sessions_exceeded
 461            (sm, rx_fib_index, thread_index)))
 462         return;
 463
 464       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 465       if (!s)
 466         {
 467           nat_elog_warn ("create NAT session failed");
 468           return;
 469         }
 470
 471       proto = ip_proto_to_nat_proto (ip->protocol);
 472
 473       s->ext_host_addr = ip->src_address;
 474       s->ext_host_port = r_port;
 475       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 476       s->out2in.addr = ip->dst_address;
 477       s->out2in.port = l_port;
 478       s->nat_proto = proto;
 479       if (proto == NAT_PROTOCOL_OTHER)
 480         {
 481           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 482           s->out2in.port = ip->protocol;
 483         }
 484       s->out2in.fib_index = 0;
 485       s->in2out.addr = s->out2in.addr;
 486       s->in2out.port = s->out2in.port;
 487       s->in2out.fib_index = s->out2in.fib_index;
 488
 489       kv.value = s - tsm->sessions;
 490       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 491         nat_elog_notice ("in2out_ed key add failed");
 492
 493       per_vrf_sessions_register_session (s, thread_index);
 494     }
 495
 496   if (ip->protocol == IP_PROTOCOL_TCP)
 497     {
 498       tcp_header_t *tcp = ip4_next_header (ip);
 499       if (nat44_set_tcp_session_state_o2i
 500           (sm, now, s, tcp->flags, tcp->ack_number, tcp->seq_number,
 501            thread_index))
 502         return;
 503     }
 504
 505   /* Accounting */
 506   nat44_session_update_counters (s, now, 0, thread_index);
 507   /* Per-user LRU list maintenance */
 508   nat44_session_update_lru (sm, s, thread_index);
 509 }
 510
 511 static inline void
 512 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
 513                               ip4_header_t * ip, u32 rx_fib_index)
 514 {
 515   ip4_header_t ip_wkr = {
 516     .src_address = ip->dst_address,
 517   };
 518   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 519
 520   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 521 }
 522
 523 #ifndef CLIB_MARCH_VARIANT
 524 u32
 525 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 526                       u32 thread_index, vlib_buffer_t * b,
 527                       ip4_header_t * ip, ip4_address_t * addr,
 528                       u16 * port, u32 * fib_index, nat_protocol_t * proto,
 529                       void *d, void *e, u8 * dont_translate)
 530 {
 531   u32 next = ~0, sw_if_index, rx_fib_index;
 532   clib_bihash_kv_16_8_t kv, value;
 533   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 534   snat_session_t *s = 0;
 535   u8 is_addr_only, identity_nat;
 536   u16 l_port, r_port;
 537   vlib_main_t *vm = vlib_get_main ();
 538   ip4_address_t sm_addr;
 539   u16 sm_port;
 540   u32 sm_fib_index;
 541   *dont_translate = 0;
 542   snat_static_mapping_t *m;
 543
 544   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 545   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 546
 547   if (get_icmp_o2i_ed_key
 548       (b, ip, rx_fib_index, ~0, ~0, proto, &l_port, &r_port, &kv))
 549     {
 550       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 551       next = NAT_NEXT_DROP;
 552       goto out;
 553     }
 554
 555   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 556     {
 557       if (snat_static_mapping_match
 558           (sm, ip->dst_address, l_port, rx_fib_index,
 559            ip_proto_to_nat_proto (ip->protocol), &sm_addr, &sm_port,
 560            &sm_fib_index, 1, &is_addr_only, 0, 0, 0, &identity_nat, &m))
 561         {
 562           // static mapping not matched
 563           if (!sm->forwarding_enabled)
 564             {
 565               /* Don't NAT packet aimed at the intfc address */
 566               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 567                                                     ip->dst_address.as_u32)))
 568                 {
 569                   *dont_translate = 1;
 570                 }
 571               else
 572                 {
 573                   b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 574                   next = NAT_NEXT_DROP;
 575                 }
 576             }
 577           else
 578             {
 579               *dont_translate = 1;
 580               if (next_src_nat (sm, ip, l_port, r_port,
 581                                 thread_index, rx_fib_index))
 582                 {
 583                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 584                 }
 585               else
 586                 {
 587                   if (sm->num_workers > 1)
 588                     create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
 589                   else
 590                     create_bypass_for_fwd (sm, b, ip, rx_fib_index,
 591                                            thread_index);
 592                 }
 593             }
 594           goto out;
 595         }
 596
 597       if (PREDICT_FALSE
 598           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 599            ICMP4_echo_reply
 600            && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 601                ICMP4_echo_request || !is_addr_only)))
 602         {
 603           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 604           next = NAT_NEXT_DROP;
 605           goto out;
 606         }
 607
 608       if (PREDICT_FALSE (identity_nat))
 609         {
 610           *dont_translate = 1;
 611           goto out;
 612         }
 613
 614       /* Create session initiated by host from external network */
 615       s =
 616         create_session_for_static_mapping_ed (sm, b, sm_addr, sm_port,
 617                                               sm_fib_index, ip->dst_address,
 618                                               l_port, rx_fib_index, *proto,
 619                                               node, rx_fib_index,
 620                                               thread_index, 0, 0,
 621                                               vlib_time_now (vm), m);
 622       if (!s)
 623         next = NAT_NEXT_DROP;
 624     }
 625   else
 626     {
 627       if (PREDICT_FALSE
 628           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 629            ICMP4_echo_reply
 630            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 631            ICMP4_echo_request
 632            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 633                                            reass.icmp_type_or_tcp_flags)))
 634         {
 635           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 636           next = NAT_NEXT_DROP;
 637           goto out;
 638         }
 639
 640       ASSERT (thread_index == ed_value_get_thread_index (&value));
 641       s =
 642         pool_elt_at_index (tsm->sessions,
 643                            ed_value_get_session_index (&value));
 644     }
 645 out:
 646   if (s)
 647     {
 648       *addr = s->in2out.addr;
 649       *port = s->in2out.port;
 650       *fib_index = s->in2out.fib_index;
 651     }
 652   if (d)
 653     *(snat_session_t **) d = s;
 654   return next;
 655 }
 656 #endif
 657
 658 static snat_session_t *
 659 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 660                                vlib_buffer_t * b,
 661                                ip4_header_t * ip,
 662                                u32 rx_fib_index,
 663                                u32 thread_index,
 664                                f64 now,
 665                                vlib_main_t * vm, vlib_node_runtime_t * node)
 666 {
 667   clib_bihash_kv_8_8_t kv, value;
 668   clib_bihash_kv_16_8_t s_kv, s_value;
 669   snat_static_mapping_t *m;
 670   u32 old_addr, new_addr;
 671   ip_csum_t sum;
 672   snat_session_t *s;
 673   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 674
 675   old_addr = ip->dst_address.as_u32;
 676
 677   init_ed_k (&s_kv, ip->dst_address, 0, ip->src_address, 0, rx_fib_index,
 678              ip->protocol);
 679
 680   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 681     {
 682       ASSERT (thread_index == ed_value_get_thread_index (&s_value));
 683       s =
 684         pool_elt_at_index (tsm->sessions,
 685                            ed_value_get_session_index (&s_value));
 686       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 687     }
 688   else
 689     {
 690       if (PREDICT_FALSE
 691           (nat44_ed_maximum_sessions_exceeded
 692            (sm, rx_fib_index, thread_index)))
 693         {
 694           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 695           nat_elog_notice ("maximum sessions exceeded");
 696           return 0;
 697         }
 698
 699       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 700       if (clib_bihash_search_8_8
 701           (&sm->static_mapping_by_external, &kv, &value))
 702         {
 703           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 704           return 0;
 705         }
 706
 707       m = pool_elt_at_index (sm->static_mappings, value.value);
 708
 709       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 710
 711       /* Create a new session */
 712       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 713       if (!s)
 714         {
 715           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 716           nat_elog_warn ("create NAT session failed");
 717           return 0;
 718         }
 719
 720       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 721       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 722       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 723       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 724       s->out2in.addr.as_u32 = old_addr;
 725       s->out2in.fib_index = rx_fib_index;
 726       s->in2out.addr.as_u32 = new_addr;
 727       s->in2out.fib_index = m->fib_index;
 728       s->in2out.port = s->out2in.port = ip->protocol;
 729
 730       /* Add to lookup tables */
 731       s_kv.value = s - tsm->sessions;
 732       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 733         nat_elog_notice ("out2in key add failed");
 734
 735       init_ed_kv (&s_kv, ip->dst_address, 0, ip->src_address, 0, m->fib_index,
 736                   ip->protocol, thread_index, s - tsm->sessions);
 737       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 738         nat_elog_notice ("in2out key add failed");
 739
 740       per_vrf_sessions_register_session (s, thread_index);
 741     }
 742
 743   /* Update IP checksum */
 744   sum = ip->checksum;
 745   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 746   ip->checksum = ip_csum_fold (sum);
 747
 748   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 749
 750   /* Accounting */
 751   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 752                                  thread_index);
 753   /* Per-user LRU list maintenance */
 754   nat44_session_update_lru (sm, s, thread_index);
 755
 756   return s;
 757 }
 758
 759 static inline uword
 760 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 761                                           vlib_node_runtime_t * node,
 762                                           vlib_frame_t * frame,
 763                                           int is_multi_worker)
 764 {
 765   u32 n_left_from, *from;
 766   snat_main_t *sm = &snat_main;
 767   f64 now = vlib_time_now (vm);
 768   u32 thread_index = vm->thread_index;
 769   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 770
 771   from = vlib_frame_vector_args (frame);
 772   n_left_from = frame->n_vectors;
 773
 774   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 775   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 776   vlib_get_buffers (vm, from, b, n_left_from);
 777
 778   while (n_left_from > 0)
 779     {
 780       vlib_buffer_t *b0;
 781       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
 782       u16 old_port0, new_port0;
 783       ip4_header_t *ip0;
 784       udp_header_t *udp0;
 785       tcp_header_t *tcp0;
 786       snat_session_t *s0 = 0;
 787       clib_bihash_kv_16_8_t kv0, value0;
 788       ip_csum_t sum0;
 789
 790       b0 = *b;
 791       b++;
 792
 793       /* Prefetch next iteration. */
 794       if (PREDICT_TRUE (n_left_from >= 2))
 795         {
 796           vlib_buffer_t *p2;
 797
 798           p2 = *b;
 799
 800           vlib_prefetch_buffer_header (p2, LOAD);
 801
 802           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 803         }
 804
 805       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 806
 807       vnet_buffer (b0)->snat.flags = 0;
 808       ip0 = vlib_buffer_get_current (b0);
 809
 810       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 811       rx_fib_index0 =
 812         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 813
 814       if (PREDICT_FALSE (ip0->ttl == 1))
 815         {
 816           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 817           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 818                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 819                                        0);
 820           next[0] = NAT_NEXT_ICMP_ERROR;
 821           goto trace0;
 822         }
 823
 824       udp0 = ip4_next_header (ip0);
 825       tcp0 = (tcp_header_t *) udp0;
 826       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 827
 828       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 829         {
 830           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 831           goto trace0;
 832         }
 833
 834       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 835         {
 836           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 837           goto trace0;
 838         }
 839
 840       init_ed_k (&kv0, ip0->dst_address,
 841                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
 842                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
 843                  ip0->protocol);
 844
 845       /* there is a stashed index in vnet_buffer2 from handoff node,
 846        * see if we can use it */
 847       if (is_multi_worker
 848           &&
 849           PREDICT_TRUE (!pool_is_free_index
 850                         (tsm->sessions,
 851                          vnet_buffer2 (b0)->nat.ed_out2in_nat_session_index)))
 852         {
 853           s0 = pool_elt_at_index (tsm->sessions,
 854                                   vnet_buffer2 (b0)->
 855                                   nat.ed_out2in_nat_session_index);
 856           if (PREDICT_TRUE
 857               (s0->out2in.addr.as_u32 == ip0->dst_address.as_u32
 858                && s0->out2in.port == vnet_buffer (b0)->ip.reass.l4_dst_port
 859                && s0->nat_proto == ip_proto_to_nat_proto (ip0->protocol)
 860                && s0->out2in.fib_index == rx_fib_index0
 861                && s0->ext_host_addr.as_u32 == ip0->src_address.as_u32
 862                && s0->ext_host_port ==
 863                vnet_buffer (b0)->ip.reass.l4_src_port))
 864             {
 865               /* yes, this is the droid we're looking for */
 866               goto skip_lookup;
 867             }
 868         }
 869
 870       // lookup for session
 871       if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
 872         {
 873           // session does not exist go slow path
 874           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 875           goto trace0;
 876         }
 877       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 878       s0 =
 879         pool_elt_at_index (tsm->sessions,
 880                            ed_value_get_session_index (&value0));
 881
 882     skip_lookup:
 883
 884       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 885         {
 886           // session is closed, go slow path
 887           nat_free_session_data (sm, s0, thread_index, 0);
 888           nat_ed_session_delete (sm, s0, thread_index, 1);
 889           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 890           goto trace0;
 891         }
 892
 893       if (s0->tcp_closed_timestamp)
 894         {
 895           if (now >= s0->tcp_closed_timestamp)
 896             {
 897               // session is closed, go slow path, freed in slow path
 898               next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 899             }
 900           else
 901             {
 902               // session in transitory timeout, drop
 903               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 904               next[0] = NAT_NEXT_DROP;
 905             }
 906           goto trace0;
 907         }
 908
 909       // drop if session expired
 910       u64 sess_timeout_time;
 911       sess_timeout_time =
 912         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 913       if (now >= sess_timeout_time)
 914         {
 915           // session is closed, go slow path
 916           nat_free_session_data (sm, s0, thread_index, 0);
 917           nat_ed_session_delete (sm, s0, thread_index, 1);
 918           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 919           goto trace0;
 920         }
 921
 922       old_addr0 = ip0->dst_address.as_u32;
 923       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 924       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 925
 926       sum0 = ip0->checksum;
 927       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 928                              dst_address);
 929       if (PREDICT_FALSE (is_twice_nat_session (s0)))
 930         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 931                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
 932                                src_address);
 933       ip0->checksum = ip_csum_fold (sum0);
 934
 935       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 936
 937       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 938         {
 939           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 940             {
 941               new_port0 = udp0->dst_port = s0->in2out.port;
 942               sum0 = tcp0->checksum;
 943               sum0 =
 944                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 945                                 dst_address);
 946               sum0 =
 947                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 948                                 length);
 949               if (is_twice_nat_session (s0))
 950                 {
 951                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 952                                          s0->ext_host_nat_addr.as_u32,
 953                                          ip4_header_t, dst_address);
 954                   sum0 =
 955                     ip_csum_update (sum0,
 956                                     vnet_buffer (b0)->ip.reass.l4_src_port,
 957                                     s0->ext_host_nat_port, ip4_header_t,
 958                                     length);
 959                   tcp0->src_port = s0->ext_host_nat_port;
 960                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 961                 }
 962               tcp0->checksum = ip_csum_fold (sum0);
 963             }
 964           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.tcp,
 965                                          thread_index, sw_if_index0, 1);
 966           if (nat44_set_tcp_session_state_o2i
 967               (sm, now, s0, vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
 968                vnet_buffer (b0)->ip.reass.tcp_ack_number,
 969                vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
 970             goto trace0;
 971         }
 972       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
 973                && udp0->checksum)
 974         {
 975           new_port0 = udp0->dst_port = s0->in2out.port;
 976           sum0 = udp0->checksum;
 977           sum0 =
 978             ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 979                             dst_address);
 980           sum0 =
 981             ip_csum_update (sum0, old_port0, new_port0, ip4_header_t, length);
 982           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 983             {
 984               sum0 =
 985                 ip_csum_update (sum0, ip0->src_address.as_u32,
 986                                 s0->ext_host_nat_addr.as_u32, ip4_header_t,
 987                                 dst_address);
 988               sum0 =
 989                 ip_csum_update (sum0, vnet_buffer (b0)->ip.reass.l4_src_port,
 990                                 s0->ext_host_nat_port, ip4_header_t, length);
 991               udp0->src_port = s0->ext_host_nat_port;
 992               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 993             }
 994           udp0->checksum = ip_csum_fold (sum0);
 995           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
 996                                          thread_index, sw_if_index0, 1);
 997         }
 998       else
 999         {
1000           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1001             {
1002               new_port0 = udp0->dst_port = s0->in2out.port;
1003               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1004                 {
1005                   udp0->src_port = s0->ext_host_nat_port;
1006                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1007                 }
1008             }
1009           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
1010                                          thread_index, sw_if_index0, 1);
1011         }
1012
1013       /* Accounting */
1014       nat44_session_update_counters (s0, now,
1015                                      vlib_buffer_length_in_chain (vm, b0),
1016                                      thread_index);
1017       /* Per-user LRU list maintenance */
1018       nat44_session_update_lru (sm, s0, thread_index);
1019
1020     trace0:
1021       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1022                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1023         {
1024           nat44_ed_out2in_trace_t *t =
1025             vlib_add_trace (vm, node, b0, sizeof (*t));
1026           t->sw_if_index = sw_if_index0;
1027           t->next_index = next[0];
1028           t->is_slow_path = 0;
1029
1030           if (s0)
1031             t->session_index = s0 - tsm->sessions;
1032           else
1033             t->session_index = ~0;
1034         }
1035
1036       if (next[0] == NAT_NEXT_DROP)
1037         {
1038           vlib_increment_simple_counter (&sm->counters.fastpath.
1039                                          out2in_ed.drops, thread_index,
1040                                          sw_if_index0, 1);
1041         }
1042
1043       n_left_from--;
1044       next++;
1045     }
1046
1047   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1048                                frame->n_vectors);
1049   return frame->n_vectors;
1050 }
1051
1052 static inline uword
1053 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1054                                           vlib_node_runtime_t * node,
1055                                           vlib_frame_t * frame)
1056 {
1057   u32 n_left_from, *from;
1058   snat_main_t *sm = &snat_main;
1059   f64 now = vlib_time_now (vm);
1060   u32 thread_index = vm->thread_index;
1061   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1062   snat_static_mapping_t *m;
1063
1064   from = vlib_frame_vector_args (frame);
1065   n_left_from = frame->n_vectors;
1066
1067   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1068   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1069   vlib_get_buffers (vm, from, b, n_left_from);
1070
1071   while (n_left_from > 0)
1072     {
1073       vlib_buffer_t *b0;
1074       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
1075       u16 old_port0, new_port0;
1076       ip4_header_t *ip0;
1077       udp_header_t *udp0;
1078       tcp_header_t *tcp0;
1079       icmp46_header_t *icmp0;
1080       snat_session_t *s0 = 0;
1081       clib_bihash_kv_16_8_t kv0, value0;
1082       ip_csum_t sum0;
1083       lb_nat_type_t lb_nat0;
1084       twice_nat_type_t twice_nat0;
1085       u8 identity_nat0;
1086       ip4_address_t sm_addr;
1087       u16 sm_port;
1088       u32 sm_fib_index;
1089
1090       b0 = *b;
1091       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1092
1093       vnet_buffer (b0)->snat.flags = 0;
1094       ip0 = vlib_buffer_get_current (b0);
1095
1096       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1097       rx_fib_index0 =
1098         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1099
1100       if (PREDICT_FALSE (ip0->ttl == 1))
1101         {
1102           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1103           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1104                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1105                                        0);
1106           next[0] = NAT_NEXT_ICMP_ERROR;
1107           goto trace0;
1108         }
1109
1110       udp0 = ip4_next_header (ip0);
1111       tcp0 = (tcp_header_t *) udp0;
1112       icmp0 = (icmp46_header_t *) udp0;
1113       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1114
1115       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1116         {
1117           s0 =
1118             nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1119                                            thread_index, now, vm, node);
1120           if (!sm->forwarding_enabled)
1121             {
1122               if (!s0)
1123                 next[0] = NAT_NEXT_DROP;
1124             }
1125           vlib_increment_simple_counter (&sm->counters.slowpath.
1126                                          out2in_ed.other, thread_index,
1127                                          sw_if_index0, 1);
1128           goto trace0;
1129         }
1130
1131       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1132         {
1133           next[0] = icmp_out2in_ed_slow_path
1134             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1135              next[0], now, thread_index, &s0);
1136           vlib_increment_simple_counter (&sm->counters.slowpath.
1137                                          out2in_ed.icmp, thread_index,
1138                                          sw_if_index0, 1);
1139           goto trace0;
1140         }
1141
1142       init_ed_k (&kv0, ip0->dst_address,
1143                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
1144                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1145                  ip0->protocol);
1146
1147       s0 = NULL;
1148       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
1149         {
1150           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1151           s0 =
1152             pool_elt_at_index (tsm->sessions,
1153                                ed_value_get_session_index (&value0));
1154
1155           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1156             {
1157               nat_free_session_data (sm, s0, thread_index, 0);
1158               nat_ed_session_delete (sm, s0, thread_index, 1);
1159               s0 = NULL;
1160             }
1161         }
1162
1163       if (!s0)
1164         {
1165           /* Try to match static mapping by external address and port,
1166              destination address and port in packet */
1167
1168           if (snat_static_mapping_match
1169               (sm, ip0->dst_address,
1170                vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1171                proto0, &sm_addr, &sm_port, &sm_fib_index, 1, 0,
1172                &twice_nat0, &lb_nat0, &ip0->src_address, &identity_nat0, &m))
1173             {
1174               /*
1175                * Send DHCP packets to the ipv4 stack, or we won't
1176                * be able to use dhcp client on the outside interface
1177                */
1178               if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_UDP
1179                                  && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1180                                      clib_host_to_net_u16
1181                                      (UDP_DST_PORT_dhcp_to_client))))
1182                 {
1183                   goto trace0;
1184                 }
1185
1186               if (!sm->forwarding_enabled)
1187                 {
1188                   b0->error =
1189                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1190                   next[0] = NAT_NEXT_DROP;
1191                 }
1192               else
1193                 {
1194                   if (next_src_nat
1195                       (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1196                        vnet_buffer (b0)->ip.reass.l4_dst_port,
1197                        thread_index, rx_fib_index0))
1198                     {
1199                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1200                     }
1201                   else
1202                     {
1203                       if (sm->num_workers > 1)
1204                         create_bypass_for_fwd_worker (sm, b0, ip0,
1205                                                       rx_fib_index0);
1206                       else
1207                         create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1208                                                thread_index);
1209                     }
1210                 }
1211               goto trace0;
1212             }
1213
1214           if (PREDICT_FALSE (identity_nat0))
1215             goto trace0;
1216
1217           if ((proto0 == NAT_PROTOCOL_TCP)
1218               && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1219                                      reass.icmp_type_or_tcp_flags))
1220             {
1221               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1222               next[0] = NAT_NEXT_DROP;
1223               goto trace0;
1224             }
1225
1226           /* Create session initiated by host from external network */
1227           s0 = create_session_for_static_mapping_ed (sm, b0,
1228                                                      sm_addr, sm_port,
1229                                                      sm_fib_index,
1230                                                      ip0->dst_address,
1231                                                      vnet_buffer (b0)->
1232                                                      ip.reass.l4_dst_port,
1233                                                      rx_fib_index0, proto0,
1234                                                      node, rx_fib_index0,
1235                                                      thread_index, twice_nat0,
1236                                                      lb_nat0, now, m);
1237           if (!s0)
1238             {
1239               next[0] = NAT_NEXT_DROP;
1240               goto trace0;
1241             }
1242         }
1243
1244       old_addr0 = ip0->dst_address.as_u32;
1245       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1246       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1247
1248       sum0 = ip0->checksum;
1249       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1250                              dst_address);
1251       if (PREDICT_FALSE (is_twice_nat_session (s0)))
1252         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1253                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
1254                                src_address);
1255       ip0->checksum = ip_csum_fold (sum0);
1256
1257       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1258
1259       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1260         {
1261           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1262             {
1263               new_port0 = udp0->dst_port = s0->in2out.port;
1264               sum0 = tcp0->checksum;
1265               sum0 =
1266                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1267                                 dst_address);
1268               sum0 =
1269                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1270                                 length);
1271               if (is_twice_nat_session (s0))
1272                 {
1273                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1274                                          s0->ext_host_nat_addr.as_u32,
1275                                          ip4_header_t, dst_address);
1276                   sum0 =
1277                     ip_csum_update (sum0,
1278                                     vnet_buffer (b0)->ip.reass.l4_src_port,
1279                                     s0->ext_host_nat_port, ip4_header_t,
1280                                     length);
1281                   tcp0->src_port = s0->ext_host_nat_port;
1282                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1283                 }
1284               tcp0->checksum = ip_csum_fold (sum0);
1285             }
1286           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.tcp,
1287                                          thread_index, sw_if_index0, 1);
1288           if (nat44_set_tcp_session_state_o2i
1289               (sm, now, s0, vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
1290                vnet_buffer (b0)->ip.reass.tcp_ack_number,
1291                vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
1292             goto trace0;
1293         }
1294       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1295                && udp0->checksum)
1296         {
1297           new_port0 = udp0->dst_port = s0->in2out.port;
1298           sum0 = udp0->checksum;
1299           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1300                                  dst_address);
1301           sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1302                                  length);
1303           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1304             {
1305               sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1306                                      s0->ext_host_nat_addr.as_u32,
1307                                      ip4_header_t, dst_address);
1308               sum0 =
1309                 ip_csum_update (sum0,
1310                                 vnet_buffer (b0)->ip.reass.l4_src_port,
1311                                 s0->ext_host_nat_port, ip4_header_t, length);
1312               udp0->src_port = s0->ext_host_nat_port;
1313               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1314             }
1315           udp0->checksum = ip_csum_fold (sum0);
1316           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1317                                          thread_index, sw_if_index0, 1);
1318         }
1319       else
1320         {
1321           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1322             {
1323               new_port0 = udp0->dst_port = s0->in2out.port;
1324               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1325                 {
1326                   udp0->src_port = s0->ext_host_nat_port;
1327                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1328                 }
1329             }
1330           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1331                                          thread_index, sw_if_index0, 1);
1332         }
1333
1334       /* Accounting */
1335       nat44_session_update_counters (s0, now,
1336                                      vlib_buffer_length_in_chain (vm, b0),
1337                                      thread_index);
1338       /* Per-user LRU list maintenance */
1339       nat44_session_update_lru (sm, s0, thread_index);
1340
1341     trace0:
1342       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1343                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1344         {
1345           nat44_ed_out2in_trace_t *t =
1346             vlib_add_trace (vm, node, b0, sizeof (*t));
1347           t->sw_if_index = sw_if_index0;
1348           t->next_index = next[0];
1349           t->is_slow_path = 1;
1350
1351           if (s0)
1352             t->session_index = s0 - tsm->sessions;
1353           else
1354             t->session_index = ~0;
1355         }
1356
1357       if (next[0] == NAT_NEXT_DROP)
1358         {
1359           vlib_increment_simple_counter (&sm->counters.slowpath.
1360                                          out2in_ed.drops, thread_index,
1361                                          sw_if_index0, 1);
1362         }
1363
1364       n_left_from--;
1365       next++;
1366       b++;
1367     }
1368
1369   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1370                                frame->n_vectors);
1371
1372   return frame->n_vectors;
1373 }
1374
1375 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1376                                      vlib_node_runtime_t * node,
1377                                      vlib_frame_t * frame)
1378 {
1379   if (snat_main.num_workers > 1)
1380     {
1381       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1382     }
1383   else
1384     {
1385       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1386     }
1387 }
1388
1389 /* *INDENT-OFF* */
1390 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1391   .name = "nat44-ed-out2in",
1392   .vector_size = sizeof (u32),
1393   .sibling_of = "nat-default",
1394   .format_trace = format_nat44_ed_out2in_trace,
1395   .type = VLIB_NODE_TYPE_INTERNAL,
1396   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1397   .error_strings = nat_out2in_ed_error_strings,
1398   .runtime_data_bytes = sizeof (snat_runtime_t),
1399 };
1400 /* *INDENT-ON* */
1401
1402 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1403                                               vlib_node_runtime_t * node,
1404                                               vlib_frame_t * frame)
1405 {
1406   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1407 }
1408
1409 /* *INDENT-OFF* */
1410 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1411   .name = "nat44-ed-out2in-slowpath",
1412   .vector_size = sizeof (u32),
1413   .sibling_of = "nat-default",
1414   .format_trace = format_nat44_ed_out2in_trace,
1415   .type = VLIB_NODE_TYPE_INTERNAL,
1416   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1417   .error_strings = nat_out2in_ed_error_strings,
1418   .runtime_data_bytes = sizeof (snat_runtime_t),
1419 };
1420 /* *INDENT-ON* */
1421
1422 static u8 *
1423 format_nat_pre_trace (u8 * s, va_list * args)
1424 {
1425   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1426   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1427   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1428   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1429                  t->arc_next_index);
1430 }
1431
1432 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1433                                     vlib_node_runtime_t * node,
1434                                     vlib_frame_t * frame)
1435 {
1436   return nat_pre_node_fn_inline (vm, node, frame,
1437                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1438 }
1439
1440 /* *INDENT-OFF* */
1441 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1442   .name = "nat-pre-out2in",
1443   .vector_size = sizeof (u32),
1444   .sibling_of = "nat-default",
1445   .format_trace = format_nat_pre_trace,
1446   .type = VLIB_NODE_TYPE_INTERNAL,
1447   .n_errors = 0,
1448  };
1449 /* *INDENT-ON* */
1450
1451 /*
1452  * fd.io coding-style-patch-verification: ON
1453  *
1454  * Local Variables:
1455  * eval: (c-set-style "gnu")
1456  * End:
1457  */