src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/lib/ipfix_logging.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat44/inlines.h>
  31 #include <nat/lib/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33 #include <nat/nat44/ed_inlines.h>
  34
  35 static char *nat_out2in_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_out2in_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat44_ed_out2in_trace_t;
  48
  49 static u8 *
  50 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  59     "NAT44_OUT2IN_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 static inline u32
  68 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  69                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  70                           u32 sw_if_index0, u32 rx_fib_index0,
  71                           vlib_node_runtime_t * node, u32 next0, f64 now,
  72                           u32 thread_index, snat_session_t ** p_s0)
  73 {
  74   vlib_main_t *vm = vlib_get_main ();
  75
  76   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  77                        next0, thread_index, p_s0, 0);
  78   snat_session_t *s0 = *p_s0;
  79   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  80     {
  81       /* Accounting */
  82       nat44_session_update_counters (s0, now,
  83                                      vlib_buffer_length_in_chain
  84                                      (vm, b0), thread_index);
  85       /* Per-user LRU list maintenance */
  86       nat44_session_update_lru (sm, s0, thread_index);
  87     }
  88   return next0;
  89 }
  90
  91 #ifndef CLIB_MARCH_VARIANT
  92 int
  93 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  94 {
  95   snat_main_t *sm = &snat_main;
  96   nat44_is_idle_session_ctx_t *ctx = arg;
  97   snat_session_t *s;
  98   u64 sess_timeout_time;
  99   u8 proto;
 100   u16 r_port, l_port;
 101   ip4_address_t *l_addr, *r_addr;
 102   u32 fib_index;
 103   clib_bihash_kv_16_8_t ed_kv;
 104   int i;
 105   //snat_address_t *a;
 106   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 107                                                        ctx->thread_index);
 108
 109   s = pool_elt_at_index (tsm->sessions, kv->value);
 110   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 111   if (ctx->now >= sess_timeout_time)
 112     {
 113       l_addr = &s->in2out.addr;
 114       r_addr = &s->ext_host_addr;
 115       fib_index = s->in2out.fib_index;
 116       if (snat_is_unk_proto_session (s))
 117         {
 118           proto = s->in2out.port;
 119           r_port = 0;
 120           l_port = 0;
 121         }
 122       else
 123         {
 124           proto = nat_proto_to_ip_proto (s->nat_proto);
 125           l_port = s->in2out.port;
 126           r_port = s->ext_host_port;
 127         }
 128       if (is_twice_nat_session (s))
 129         {
 130           r_addr = &s->ext_host_nat_addr;
 131           r_port = s->ext_host_nat_port;
 132         }
 133       init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
 134       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 135         nat_elog_warn ("in2out_ed key del failed");
 136
 137       if (snat_is_unk_proto_session (s))
 138         goto delete;
 139
 140       nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 141                                           s->in2out.addr.as_u32,
 142                                           s->out2in.addr.as_u32,
 143                                           s->nat_proto,
 144                                           s->in2out.port,
 145                                           s->out2in.port,
 146                                           s->in2out.fib_index);
 147
 148       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 149                              &s->in2out.addr, s->in2out.port,
 150                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 151                              &s->out2in.addr, s->out2in.port,
 152                              &s->ext_host_addr, s->ext_host_port,
 153                              s->nat_proto, is_twice_nat_session (s));
 154
 155       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 156                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 157                    ctx->thread_index);
 158
 159       if (is_twice_nat_session (s))
 160         {
 161           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 162             {
 163               // FIXME TODO this is obviously wrong code ... needs fix!
 164               //       key.protocol = s->nat_proto;
 165               //       key.port = s->ext_host_nat_port;
 166               //       a = sm->twice_nat_addresses + i;
 167               //       if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 168               //      {
 169               //        snat_free_outside_address_and_port (sm->twice_nat_addresses,
 170               //                                            ctx->thread_index,
 171               //                                            &key);
 172               //        break;
 173               //      }
 174             }
 175         }
 176
 177       if (snat_is_session_static (s))
 178         goto delete;
 179
 180       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 181                                           &s->out2in.addr, s->out2in.port,
 182                                           s->nat_proto);
 183     delete:
 184       nat_ed_session_delete (sm, s, ctx->thread_index, 1);
 185       return 1;
 186     }
 187
 188   return 0;
 189 }
 190 #endif
 191
 192 // allocate exact address based on preference
 193 static_always_inline int
 194 nat_alloc_addr_and_port_exact (snat_address_t * a,
 195                                u32 thread_index,
 196                                nat_protocol_t proto,
 197                                ip4_address_t * addr,
 198                                u16 * port,
 199                                u16 port_per_thread, u32 snat_thread_index)
 200 {
 201   u32 portnum;
 202
 203   switch (proto)
 204     {
 205 #define _(N, j, n, s) \
 206     case NAT_PROTOCOL_##N: \
 207       if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
 208         { \
 209           while (1) \
 210             { \
 211               portnum = (port_per_thread * \
 212                 snat_thread_index) + \
 213                 snat_random_port(0, port_per_thread - 1) + 1024; \
 214               if (a->busy_##n##_port_refcounts[portnum]) \
 215                 continue; \
 216               --a->busy_##n##_port_refcounts[portnum]; \
 217               a->busy_##n##_ports_per_thread[thread_index]++; \
 218               a->busy_##n##_ports++; \
 219               *addr = a->addr; \
 220               *port = clib_host_to_net_u16(portnum); \
 221               return 0; \
 222             } \
 223         } \
 224       break;
 225       foreach_nat_protocol
 226 #undef _
 227     default:
 228       nat_elog_info ("unknown protocol");
 229       return 1;
 230     }
 231
 232   /* Totally out of translations to use... */
 233   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 234   return 1;
 235 }
 236
 237
 238 static snat_session_t *
 239 create_session_for_static_mapping_ed (snat_main_t * sm,
 240                                       vlib_buffer_t * b,
 241                                       ip4_address_t i2o_addr,
 242                                       u16 i2o_port,
 243                                       u32 i2o_fib_index,
 244                                       ip4_address_t o2i_addr,
 245                                       u16 o2i_port,
 246                                       u32 o2i_fib_index,
 247                                       nat_protocol_t nat_proto,
 248                                       vlib_node_runtime_t * node,
 249                                       u32 rx_fib_index,
 250                                       u32 thread_index,
 251                                       twice_nat_type_t twice_nat,
 252                                       lb_nat_type_t lb_nat, f64 now,
 253                                       snat_static_mapping_t * mapping)
 254 {
 255   snat_session_t *s;
 256   ip4_header_t *ip;
 257   udp_header_t *udp;
 258   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 259   clib_bihash_kv_16_8_t kv;
 260   nat44_is_idle_session_ctx_t ctx;
 261
 262   if (PREDICT_FALSE
 263       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 264     {
 265       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 266       nat_elog_notice ("maximum sessions exceeded");
 267       return 0;
 268     }
 269
 270   s = nat_ed_session_alloc (sm, thread_index, now, nat_proto);
 271   if (!s)
 272     {
 273       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 274       nat_elog_warn ("create NAT session failed");
 275       return 0;
 276     }
 277
 278   ip = vlib_buffer_get_current (b);
 279   udp = ip4_next_header (ip);
 280
 281   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 282   s->ext_host_port = nat_proto == NAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 283   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 284   if (lb_nat)
 285     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 286   if (lb_nat == AFFINITY_LB_NAT)
 287     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 288   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 289   s->out2in.addr = o2i_addr;
 290   s->out2in.port = o2i_port;
 291   s->out2in.fib_index = o2i_fib_index;
 292   s->in2out.addr = i2o_addr;
 293   s->in2out.port = i2o_port;
 294   s->in2out.fib_index = i2o_fib_index;
 295   s->nat_proto = nat_proto;
 296
 297   /* Add to lookup tables */
 298   init_ed_kv (&kv, o2i_addr, o2i_port, s->ext_host_addr, s->ext_host_port,
 299               o2i_fib_index, ip->protocol, thread_index, s - tsm->sessions);
 300   ctx.now = now;
 301   ctx.thread_index = thread_index;
 302   if (clib_bihash_add_or_overwrite_stale_16_8 (&sm->out2in_ed, &kv,
 303                                                nat44_o2i_ed_is_idle_session_cb,
 304                                                &ctx))
 305     nat_elog_notice ("out2in-ed key add failed");
 306
 307   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 308                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 309     {
 310       int rc = 0;
 311       snat_address_t *filter = 0;
 312
 313       // if exact address is specified use this address
 314       if (is_exact_address (mapping))
 315         {
 316           snat_address_t *ap;
 317           vec_foreach (ap, sm->twice_nat_addresses)
 318           {
 319             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 320               {
 321                 filter = ap;
 322                 break;
 323               }
 324           }
 325         }
 326
 327       if (filter)
 328         {
 329           rc = nat_alloc_addr_and_port_exact (filter,
 330                                               thread_index,
 331                                               nat_proto,
 332                                               &s->ext_host_nat_addr,
 333                                               &s->ext_host_nat_port,
 334                                               sm->port_per_thread,
 335                                               tsm->snat_thread_index);
 336           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 337         }
 338       else
 339         {
 340           rc =
 341             snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 342                                                  thread_index, nat_proto,
 343                                                  &s->ext_host_nat_addr,
 344                                                  &s->ext_host_nat_port,
 345                                                  sm->port_per_thread,
 346                                                  tsm->snat_thread_index);
 347         }
 348
 349       if (rc)
 350         {
 351           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 352           nat_ed_session_delete (sm, s, thread_index, 1);
 353           if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 0))
 354             nat_elog_notice ("out2in-ed key del failed");
 355           return 0;
 356         }
 357
 358       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 359       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_nat_addr,
 360                   s->ext_host_nat_port, i2o_fib_index, ip->protocol,
 361                   thread_index, s - tsm->sessions);
 362     }
 363   else
 364     {
 365       init_ed_kv (&kv, i2o_addr, i2o_port, s->ext_host_addr,
 366                   s->ext_host_port, i2o_fib_index, ip->protocol,
 367                   thread_index, s - tsm->sessions);
 368     }
 369   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 370                                                nat44_i2o_ed_is_idle_session_cb,
 371                                                &ctx))
 372     nat_elog_notice ("in2out-ed key add failed");
 373
 374   nat_ipfix_logging_nat44_ses_create (thread_index,
 375                                       s->in2out.addr.as_u32,
 376                                       s->out2in.addr.as_u32,
 377                                       s->nat_proto,
 378                                       s->in2out.port,
 379                                       s->out2in.port, s->in2out.fib_index);
 380
 381   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 382                          &s->in2out.addr, s->in2out.port,
 383                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 384                          &s->out2in.addr, s->out2in.port,
 385                          &s->ext_host_addr, s->ext_host_port,
 386                          s->nat_proto, is_twice_nat_session (s));
 387
 388   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 389                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 390                &s->ext_host_nat_addr, s->ext_host_nat_port,
 391                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 392
 393   per_vrf_sessions_register_session (s, thread_index);
 394
 395   return s;
 396 }
 397
 398 static int
 399 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
 400               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 401 {
 402   clib_bihash_kv_16_8_t kv, value;
 403   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 404
 405   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 406              rx_fib_index, ip->protocol);
 407   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 408     return 1;
 409
 410   return 0;
 411 }
 412
 413 static void
 414 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
 415                        u32 rx_fib_index, u32 thread_index)
 416 {
 417   clib_bihash_kv_16_8_t kv, value;
 418   udp_header_t *udp;
 419   snat_session_t *s = 0;
 420   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 421   vlib_main_t *vm = vlib_get_main ();
 422   f64 now = vlib_time_now (vm);
 423   u16 l_port, r_port;
 424
 425   if (ip->protocol == IP_PROTOCOL_ICMP)
 426     {
 427       if (get_icmp_o2i_ed_key
 428           (b, ip, rx_fib_index, ~0, ~0, 0, &l_port, &r_port, &kv))
 429         return;
 430     }
 431   else
 432     {
 433       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 434         {
 435           udp = ip4_next_header (ip);
 436           l_port = udp->dst_port;
 437           r_port = udp->src_port;
 438         }
 439       else
 440         {
 441           l_port = 0;
 442           r_port = 0;
 443         }
 444       init_ed_k (&kv, ip->dst_address, l_port, ip->src_address, r_port,
 445                  rx_fib_index, ip->protocol);
 446     }
 447
 448   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 449     {
 450       ASSERT (thread_index == ed_value_get_thread_index (&value));
 451       s =
 452         pool_elt_at_index (tsm->sessions,
 453                            ed_value_get_session_index (&value));
 454     }
 455   else if (ip->protocol == IP_PROTOCOL_ICMP &&
 456            icmp_type_is_error_message
 457            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 458     {
 459       return;
 460     }
 461   else
 462     {
 463       u32 proto;
 464
 465       if (PREDICT_FALSE
 466           (nat44_ed_maximum_sessions_exceeded
 467            (sm, rx_fib_index, thread_index)))
 468         return;
 469
 470       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 471       if (!s)
 472         {
 473           nat_elog_warn ("create NAT session failed");
 474           return;
 475         }
 476
 477       proto = ip_proto_to_nat_proto (ip->protocol);
 478
 479       s->ext_host_addr = ip->src_address;
 480       s->ext_host_port = r_port;
 481       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 482       s->out2in.addr = ip->dst_address;
 483       s->out2in.port = l_port;
 484       s->nat_proto = proto;
 485       if (proto == NAT_PROTOCOL_OTHER)
 486         {
 487           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 488           s->out2in.port = ip->protocol;
 489         }
 490       s->out2in.fib_index = rx_fib_index;
 491       s->in2out.addr = s->out2in.addr;
 492       s->in2out.port = s->out2in.port;
 493       s->in2out.fib_index = s->out2in.fib_index;
 494
 495       kv.value = s - tsm->sessions;
 496       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 497         nat_elog_notice ("in2out_ed key add failed");
 498
 499       per_vrf_sessions_register_session (s, thread_index);
 500     }
 501
 502   if (ip->protocol == IP_PROTOCOL_TCP)
 503     {
 504       tcp_header_t *tcp = ip4_next_header (ip);
 505       nat44_set_tcp_session_state_o2i (sm, now, s, tcp->flags,
 506                                        tcp->ack_number, tcp->seq_number,
 507                                        thread_index);
 508     }
 509
 510   /* Accounting */
 511   nat44_session_update_counters (s, now, 0, thread_index);
 512   /* Per-user LRU list maintenance */
 513   nat44_session_update_lru (sm, s, thread_index);
 514 }
 515
 516 static inline void
 517 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
 518                               ip4_header_t * ip, u32 rx_fib_index)
 519 {
 520   ip4_header_t ip_wkr = {
 521     .src_address = ip->dst_address,
 522   };
 523   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 524
 525   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 526 }
 527
 528 #ifndef CLIB_MARCH_VARIANT
 529 u32
 530 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 531                       u32 thread_index, vlib_buffer_t * b,
 532                       ip4_header_t * ip, ip4_address_t * addr,
 533                       u16 * port, u32 * fib_index, nat_protocol_t * proto,
 534                       void *d, void *e, u8 * dont_translate)
 535 {
 536   u32 next = ~0, sw_if_index, rx_fib_index;
 537   clib_bihash_kv_16_8_t kv, value;
 538   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 539   snat_session_t *s = 0;
 540   u8 is_addr_only, identity_nat;
 541   u16 l_port, r_port;
 542   vlib_main_t *vm = vlib_get_main ();
 543   ip4_address_t sm_addr;
 544   u16 sm_port;
 545   u32 sm_fib_index;
 546   *dont_translate = 0;
 547   snat_static_mapping_t *m;
 548
 549   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 550   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 551
 552   if (get_icmp_o2i_ed_key
 553       (b, ip, rx_fib_index, ~0, ~0, proto, &l_port, &r_port, &kv))
 554     {
 555       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 556       next = NAT_NEXT_DROP;
 557       goto out;
 558     }
 559
 560   if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
 561     {
 562       if (snat_static_mapping_match
 563           (sm, ip->dst_address, l_port, rx_fib_index,
 564            ip_proto_to_nat_proto (ip->protocol), &sm_addr, &sm_port,
 565            &sm_fib_index, 1, &is_addr_only, 0, 0, 0, &identity_nat, &m))
 566         {
 567           // static mapping not matched
 568           if (!sm->forwarding_enabled)
 569             {
 570               /* Don't NAT packet aimed at the intfc address */
 571               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 572                                                     ip->dst_address.as_u32)))
 573                 {
 574                   *dont_translate = 1;
 575                 }
 576               else
 577                 {
 578                   b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 579                   next = NAT_NEXT_DROP;
 580                 }
 581             }
 582           else
 583             {
 584               *dont_translate = 1;
 585               if (next_src_nat (sm, ip, l_port, r_port,
 586                                 thread_index, rx_fib_index))
 587                 {
 588                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 589                 }
 590               else
 591                 {
 592                   if (sm->num_workers > 1)
 593                     create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
 594                   else
 595                     create_bypass_for_fwd (sm, b, ip, rx_fib_index,
 596                                            thread_index);
 597                 }
 598             }
 599           goto out;
 600         }
 601
 602       if (PREDICT_FALSE
 603           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 604            ICMP4_echo_reply
 605            && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 606                ICMP4_echo_request || !is_addr_only)))
 607         {
 608           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 609           next = NAT_NEXT_DROP;
 610           goto out;
 611         }
 612
 613       if (PREDICT_FALSE (identity_nat))
 614         {
 615           *dont_translate = 1;
 616           goto out;
 617         }
 618
 619       /* Create session initiated by host from external network */
 620       s =
 621         create_session_for_static_mapping_ed (sm, b, sm_addr, sm_port,
 622                                               sm_fib_index, ip->dst_address,
 623                                               l_port, rx_fib_index, *proto,
 624                                               node, rx_fib_index,
 625                                               thread_index, 0, 0,
 626                                               vlib_time_now (vm), m);
 627       if (!s)
 628         next = NAT_NEXT_DROP;
 629     }
 630   else
 631     {
 632       if (PREDICT_FALSE
 633           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 634            ICMP4_echo_reply
 635            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 636            ICMP4_echo_request
 637            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 638                                            reass.icmp_type_or_tcp_flags)))
 639         {
 640           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 641           next = NAT_NEXT_DROP;
 642           goto out;
 643         }
 644
 645       ASSERT (thread_index == ed_value_get_thread_index (&value));
 646       s =
 647         pool_elt_at_index (tsm->sessions,
 648                            ed_value_get_session_index (&value));
 649     }
 650 out:
 651   if (s)
 652     {
 653       *addr = s->in2out.addr;
 654       *port = s->in2out.port;
 655       *fib_index = s->in2out.fib_index;
 656     }
 657   if (d)
 658     *(snat_session_t **) d = s;
 659   return next;
 660 }
 661 #endif
 662
 663 static snat_session_t *
 664 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 665                                vlib_buffer_t * b,
 666                                ip4_header_t * ip,
 667                                u32 rx_fib_index,
 668                                u32 thread_index,
 669                                f64 now,
 670                                vlib_main_t * vm, vlib_node_runtime_t * node)
 671 {
 672   clib_bihash_kv_8_8_t kv, value;
 673   clib_bihash_kv_16_8_t s_kv, s_value;
 674   snat_static_mapping_t *m;
 675   u32 old_addr, new_addr;
 676   ip_csum_t sum;
 677   snat_session_t *s;
 678   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 679
 680   old_addr = ip->dst_address.as_u32;
 681
 682   init_ed_k (&s_kv, ip->dst_address, 0, ip->src_address, 0, rx_fib_index,
 683              ip->protocol);
 684
 685   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 686     {
 687       ASSERT (thread_index == ed_value_get_thread_index (&s_value));
 688       s =
 689         pool_elt_at_index (tsm->sessions,
 690                            ed_value_get_session_index (&s_value));
 691       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 692     }
 693   else
 694     {
 695       if (PREDICT_FALSE
 696           (nat44_ed_maximum_sessions_exceeded
 697            (sm, rx_fib_index, thread_index)))
 698         {
 699           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 700           nat_elog_notice ("maximum sessions exceeded");
 701           return 0;
 702         }
 703
 704       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 705       if (clib_bihash_search_8_8
 706           (&sm->static_mapping_by_external, &kv, &value))
 707         {
 708           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 709           return 0;
 710         }
 711
 712       m = pool_elt_at_index (sm->static_mappings, value.value);
 713
 714       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 715
 716       /* Create a new session */
 717       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 718       if (!s)
 719         {
 720           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 721           nat_elog_warn ("create NAT session failed");
 722           return 0;
 723         }
 724
 725       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 726       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 727       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 728       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 729       s->out2in.addr.as_u32 = old_addr;
 730       s->out2in.fib_index = rx_fib_index;
 731       s->in2out.addr.as_u32 = new_addr;
 732       s->in2out.fib_index = m->fib_index;
 733       s->in2out.port = s->out2in.port = ip->protocol;
 734
 735       /* Add to lookup tables */
 736       s_kv.value = s - tsm->sessions;
 737       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 738         nat_elog_notice ("out2in key add failed");
 739
 740       init_ed_kv (&s_kv, ip->dst_address, 0, ip->src_address, 0, m->fib_index,
 741                   ip->protocol, thread_index, s - tsm->sessions);
 742       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 743         nat_elog_notice ("in2out key add failed");
 744
 745       per_vrf_sessions_register_session (s, thread_index);
 746     }
 747
 748   /* Update IP checksum */
 749   sum = ip->checksum;
 750   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 751   ip->checksum = ip_csum_fold (sum);
 752
 753   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 754
 755   /* Accounting */
 756   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 757                                  thread_index);
 758   /* Per-user LRU list maintenance */
 759   nat44_session_update_lru (sm, s, thread_index);
 760
 761   return s;
 762 }
 763
 764 static inline uword
 765 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 766                                           vlib_node_runtime_t * node,
 767                                           vlib_frame_t * frame,
 768                                           int is_multi_worker)
 769 {
 770   u32 n_left_from, *from;
 771   snat_main_t *sm = &snat_main;
 772   f64 now = vlib_time_now (vm);
 773   u32 thread_index = vm->thread_index;
 774   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 775
 776   from = vlib_frame_vector_args (frame);
 777   n_left_from = frame->n_vectors;
 778
 779   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 780   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 781   vlib_get_buffers (vm, from, b, n_left_from);
 782
 783   while (n_left_from > 0)
 784     {
 785       vlib_buffer_t *b0;
 786       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
 787       u16 old_port0, new_port0;
 788       ip4_header_t *ip0;
 789       udp_header_t *udp0;
 790       tcp_header_t *tcp0;
 791       snat_session_t *s0 = 0;
 792       clib_bihash_kv_16_8_t kv0, value0;
 793       ip_csum_t sum0;
 794
 795       b0 = *b;
 796       b++;
 797
 798       /* Prefetch next iteration. */
 799       if (PREDICT_TRUE (n_left_from >= 2))
 800         {
 801           vlib_buffer_t *p2;
 802
 803           p2 = *b;
 804
 805           vlib_prefetch_buffer_header (p2, LOAD);
 806
 807           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 808         }
 809
 810       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 811
 812       vnet_buffer (b0)->snat.flags = 0;
 813       ip0 = vlib_buffer_get_current (b0);
 814
 815       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 816       rx_fib_index0 =
 817         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 818
 819       if (PREDICT_FALSE (ip0->ttl == 1))
 820         {
 821           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 822           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 823                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 824                                        0);
 825           next[0] = NAT_NEXT_ICMP_ERROR;
 826           goto trace0;
 827         }
 828
 829       udp0 = ip4_next_header (ip0);
 830       tcp0 = (tcp_header_t *) udp0;
 831       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 832
 833       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 834         {
 835           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 836           goto trace0;
 837         }
 838
 839       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 840         {
 841           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 842           goto trace0;
 843         }
 844
 845       init_ed_k (&kv0, ip0->dst_address,
 846                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
 847                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
 848                  ip0->protocol);
 849
 850       /* there is a stashed index in vnet_buffer2 from handoff node,
 851        * see if we can use it */
 852       if (is_multi_worker
 853           &&
 854           PREDICT_TRUE (!pool_is_free_index
 855                         (tsm->sessions,
 856                          vnet_buffer2 (b0)->nat.ed_out2in_nat_session_index)))
 857         {
 858           s0 = pool_elt_at_index (tsm->sessions,
 859                                   vnet_buffer2 (b0)->
 860                                   nat.ed_out2in_nat_session_index);
 861           if (PREDICT_TRUE
 862               (s0->out2in.addr.as_u32 == ip0->dst_address.as_u32
 863                && s0->out2in.port == vnet_buffer (b0)->ip.reass.l4_dst_port
 864                && s0->nat_proto == ip_proto_to_nat_proto (ip0->protocol)
 865                && s0->out2in.fib_index == rx_fib_index0
 866                && s0->ext_host_addr.as_u32 == ip0->src_address.as_u32
 867                && s0->ext_host_port ==
 868                vnet_buffer (b0)->ip.reass.l4_src_port))
 869             {
 870               /* yes, this is the droid we're looking for */
 871               goto skip_lookup;
 872             }
 873         }
 874
 875       // lookup for session
 876       if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
 877         {
 878           // session does not exist go slow path
 879           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 880           goto trace0;
 881         }
 882       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 883       s0 =
 884         pool_elt_at_index (tsm->sessions,
 885                            ed_value_get_session_index (&value0));
 886
 887     skip_lookup:
 888
 889       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 890         {
 891           // session is closed, go slow path
 892           nat_free_session_data (sm, s0, thread_index, 0);
 893           nat_ed_session_delete (sm, s0, thread_index, 1);
 894           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 895           goto trace0;
 896         }
 897
 898       if (s0->tcp_closed_timestamp)
 899         {
 900           if (now >= s0->tcp_closed_timestamp)
 901             {
 902               // session is closed, go slow path, freed in slow path
 903               next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 904             }
 905           else
 906             {
 907               // session in transitory timeout, drop
 908               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 909               next[0] = NAT_NEXT_DROP;
 910             }
 911           goto trace0;
 912         }
 913
 914       // drop if session expired
 915       u64 sess_timeout_time;
 916       sess_timeout_time =
 917         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 918       if (now >= sess_timeout_time)
 919         {
 920           // session is closed, go slow path
 921           nat_free_session_data (sm, s0, thread_index, 0);
 922           nat_ed_session_delete (sm, s0, thread_index, 1);
 923           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 924           goto trace0;
 925         }
 926
 927       old_addr0 = ip0->dst_address.as_u32;
 928       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 929       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 930
 931       sum0 = ip0->checksum;
 932       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 933                              dst_address);
 934       if (PREDICT_FALSE (is_twice_nat_session (s0)))
 935         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 936                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
 937                                src_address);
 938       ip0->checksum = ip_csum_fold (sum0);
 939
 940       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 941
 942       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 943         {
 944           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 945             {
 946               new_port0 = udp0->dst_port = s0->in2out.port;
 947               sum0 = tcp0->checksum;
 948               sum0 =
 949                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 950                                 dst_address);
 951               sum0 =
 952                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 953                                 length);
 954               if (is_twice_nat_session (s0))
 955                 {
 956                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 957                                          s0->ext_host_nat_addr.as_u32,
 958                                          ip4_header_t, dst_address);
 959                   sum0 =
 960                     ip_csum_update (sum0,
 961                                     vnet_buffer (b0)->ip.reass.l4_src_port,
 962                                     s0->ext_host_nat_port, ip4_header_t,
 963                                     length);
 964                   tcp0->src_port = s0->ext_host_nat_port;
 965                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 966                 }
 967               tcp0->checksum = ip_csum_fold (sum0);
 968             }
 969           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.tcp,
 970                                          thread_index, sw_if_index0, 1);
 971           nat44_set_tcp_session_state_o2i (sm, now, s0,
 972                                            vnet_buffer (b0)->ip.
 973                                            reass.icmp_type_or_tcp_flags,
 974                                            vnet_buffer (b0)->ip.
 975                                            reass.tcp_ack_number,
 976                                            vnet_buffer (b0)->ip.
 977                                            reass.tcp_seq_number,
 978                                            thread_index);
 979         }
 980       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
 981                && udp0->checksum)
 982         {
 983           new_port0 = udp0->dst_port = s0->in2out.port;
 984           sum0 = udp0->checksum;
 985           sum0 =
 986             ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 987                             dst_address);
 988           sum0 =
 989             ip_csum_update (sum0, old_port0, new_port0, ip4_header_t, length);
 990           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 991             {
 992               sum0 =
 993                 ip_csum_update (sum0, ip0->src_address.as_u32,
 994                                 s0->ext_host_nat_addr.as_u32, ip4_header_t,
 995                                 dst_address);
 996               sum0 =
 997                 ip_csum_update (sum0, vnet_buffer (b0)->ip.reass.l4_src_port,
 998                                 s0->ext_host_nat_port, ip4_header_t, length);
 999               udp0->src_port = s0->ext_host_nat_port;
1000               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1001             }
1002           udp0->checksum = ip_csum_fold (sum0);
1003           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
1004                                          thread_index, sw_if_index0, 1);
1005         }
1006       else
1007         {
1008           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1009             {
1010               new_port0 = udp0->dst_port = s0->in2out.port;
1011               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1012                 {
1013                   udp0->src_port = s0->ext_host_nat_port;
1014                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1015                 }
1016             }
1017           vlib_increment_simple_counter (&sm->counters.fastpath.out2in_ed.udp,
1018                                          thread_index, sw_if_index0, 1);
1019         }
1020
1021       /* Accounting */
1022       nat44_session_update_counters (s0, now,
1023                                      vlib_buffer_length_in_chain (vm, b0),
1024                                      thread_index);
1025       /* Per-user LRU list maintenance */
1026       nat44_session_update_lru (sm, s0, thread_index);
1027
1028     trace0:
1029       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1030                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1031         {
1032           nat44_ed_out2in_trace_t *t =
1033             vlib_add_trace (vm, node, b0, sizeof (*t));
1034           t->sw_if_index = sw_if_index0;
1035           t->next_index = next[0];
1036           t->is_slow_path = 0;
1037
1038           if (s0)
1039             t->session_index = s0 - tsm->sessions;
1040           else
1041             t->session_index = ~0;
1042         }
1043
1044       if (next[0] == NAT_NEXT_DROP)
1045         {
1046           vlib_increment_simple_counter (&sm->counters.fastpath.
1047                                          out2in_ed.drops, thread_index,
1048                                          sw_if_index0, 1);
1049         }
1050
1051       n_left_from--;
1052       next++;
1053     }
1054
1055   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1056                                frame->n_vectors);
1057   return frame->n_vectors;
1058 }
1059
1060 static inline uword
1061 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1062                                           vlib_node_runtime_t * node,
1063                                           vlib_frame_t * frame)
1064 {
1065   u32 n_left_from, *from;
1066   snat_main_t *sm = &snat_main;
1067   f64 now = vlib_time_now (vm);
1068   u32 thread_index = vm->thread_index;
1069   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1070   snat_static_mapping_t *m;
1071
1072   from = vlib_frame_vector_args (frame);
1073   n_left_from = frame->n_vectors;
1074
1075   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1076   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1077   vlib_get_buffers (vm, from, b, n_left_from);
1078
1079   while (n_left_from > 0)
1080     {
1081       vlib_buffer_t *b0;
1082       u32 sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
1083       u16 old_port0, new_port0;
1084       ip4_header_t *ip0;
1085       udp_header_t *udp0;
1086       tcp_header_t *tcp0;
1087       icmp46_header_t *icmp0;
1088       snat_session_t *s0 = 0;
1089       clib_bihash_kv_16_8_t kv0, value0;
1090       ip_csum_t sum0;
1091       lb_nat_type_t lb_nat0;
1092       twice_nat_type_t twice_nat0;
1093       u8 identity_nat0;
1094       ip4_address_t sm_addr;
1095       u16 sm_port;
1096       u32 sm_fib_index;
1097
1098       b0 = *b;
1099       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1100
1101       vnet_buffer (b0)->snat.flags = 0;
1102       ip0 = vlib_buffer_get_current (b0);
1103
1104       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1105       rx_fib_index0 =
1106         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1107
1108       if (PREDICT_FALSE (ip0->ttl == 1))
1109         {
1110           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1111           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1112                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1113                                        0);
1114           next[0] = NAT_NEXT_ICMP_ERROR;
1115           goto trace0;
1116         }
1117
1118       udp0 = ip4_next_header (ip0);
1119       tcp0 = (tcp_header_t *) udp0;
1120       icmp0 = (icmp46_header_t *) udp0;
1121       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1122
1123       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1124         {
1125           s0 =
1126             nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1127                                            thread_index, now, vm, node);
1128           if (!sm->forwarding_enabled)
1129             {
1130               if (!s0)
1131                 next[0] = NAT_NEXT_DROP;
1132             }
1133           vlib_increment_simple_counter (&sm->counters.slowpath.
1134                                          out2in_ed.other, thread_index,
1135                                          sw_if_index0, 1);
1136           goto trace0;
1137         }
1138
1139       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1140         {
1141           next[0] = icmp_out2in_ed_slow_path
1142             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1143              next[0], now, thread_index, &s0);
1144           vlib_increment_simple_counter (&sm->counters.slowpath.
1145                                          out2in_ed.icmp, thread_index,
1146                                          sw_if_index0, 1);
1147           goto trace0;
1148         }
1149
1150       init_ed_k (&kv0, ip0->dst_address,
1151                  vnet_buffer (b0)->ip.reass.l4_dst_port, ip0->src_address,
1152                  vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1153                  ip0->protocol);
1154
1155       s0 = NULL;
1156       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
1157         {
1158           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1159           s0 =
1160             pool_elt_at_index (tsm->sessions,
1161                                ed_value_get_session_index (&value0));
1162
1163           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1164             {
1165               nat_free_session_data (sm, s0, thread_index, 0);
1166               nat_ed_session_delete (sm, s0, thread_index, 1);
1167               s0 = NULL;
1168             }
1169         }
1170
1171       if (!s0)
1172         {
1173           /* Try to match static mapping by external address and port,
1174              destination address and port in packet */
1175
1176           if (snat_static_mapping_match
1177               (sm, ip0->dst_address,
1178                vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1179                proto0, &sm_addr, &sm_port, &sm_fib_index, 1, 0,
1180                &twice_nat0, &lb_nat0, &ip0->src_address, &identity_nat0, &m))
1181             {
1182               /*
1183                * Send DHCP packets to the ipv4 stack, or we won't
1184                * be able to use dhcp client on the outside interface
1185                */
1186               if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_UDP
1187                                  && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1188                                      clib_host_to_net_u16
1189                                      (UDP_DST_PORT_dhcp_to_client))))
1190                 {
1191                   goto trace0;
1192                 }
1193
1194               if (!sm->forwarding_enabled)
1195                 {
1196                   b0->error =
1197                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1198                   next[0] = NAT_NEXT_DROP;
1199                 }
1200               else
1201                 {
1202                   if (next_src_nat
1203                       (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1204                        vnet_buffer (b0)->ip.reass.l4_dst_port,
1205                        thread_index, rx_fib_index0))
1206                     {
1207                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1208                     }
1209                   else
1210                     {
1211                       if (sm->num_workers > 1)
1212                         create_bypass_for_fwd_worker (sm, b0, ip0,
1213                                                       rx_fib_index0);
1214                       else
1215                         create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1216                                                thread_index);
1217                     }
1218                 }
1219               goto trace0;
1220             }
1221
1222           if (PREDICT_FALSE (identity_nat0))
1223             goto trace0;
1224
1225           if ((proto0 == NAT_PROTOCOL_TCP)
1226               && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1227                                      reass.icmp_type_or_tcp_flags))
1228             {
1229               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1230               next[0] = NAT_NEXT_DROP;
1231               goto trace0;
1232             }
1233
1234           /* Create session initiated by host from external network */
1235           s0 = create_session_for_static_mapping_ed (sm, b0,
1236                                                      sm_addr, sm_port,
1237                                                      sm_fib_index,
1238                                                      ip0->dst_address,
1239                                                      vnet_buffer (b0)->
1240                                                      ip.reass.l4_dst_port,
1241                                                      rx_fib_index0, proto0,
1242                                                      node, rx_fib_index0,
1243                                                      thread_index, twice_nat0,
1244                                                      lb_nat0, now, m);
1245           if (!s0)
1246             {
1247               next[0] = NAT_NEXT_DROP;
1248               goto trace0;
1249             }
1250         }
1251
1252       old_addr0 = ip0->dst_address.as_u32;
1253       new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1254       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1255
1256       sum0 = ip0->checksum;
1257       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1258                              dst_address);
1259       if (PREDICT_FALSE (is_twice_nat_session (s0)))
1260         sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1261                                s0->ext_host_nat_addr.as_u32, ip4_header_t,
1262                                src_address);
1263       ip0->checksum = ip_csum_fold (sum0);
1264
1265       old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1266
1267       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1268         {
1269           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1270             {
1271               new_port0 = udp0->dst_port = s0->in2out.port;
1272               sum0 = tcp0->checksum;
1273               sum0 =
1274                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1275                                 dst_address);
1276               sum0 =
1277                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1278                                 length);
1279               if (is_twice_nat_session (s0))
1280                 {
1281                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1282                                          s0->ext_host_nat_addr.as_u32,
1283                                          ip4_header_t, dst_address);
1284                   sum0 =
1285                     ip_csum_update (sum0,
1286                                     vnet_buffer (b0)->ip.reass.l4_src_port,
1287                                     s0->ext_host_nat_port, ip4_header_t,
1288                                     length);
1289                   tcp0->src_port = s0->ext_host_nat_port;
1290                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1291                 }
1292               tcp0->checksum = ip_csum_fold (sum0);
1293             }
1294           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.tcp,
1295                                          thread_index, sw_if_index0, 1);
1296           nat44_set_tcp_session_state_o2i (sm, now, s0,
1297                                            vnet_buffer (b0)->ip.
1298                                            reass.icmp_type_or_tcp_flags,
1299                                            vnet_buffer (b0)->ip.
1300                                            reass.tcp_ack_number,
1301                                            vnet_buffer (b0)->ip.
1302                                            reass.tcp_seq_number,
1303                                            thread_index);
1304         }
1305       else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1306                && udp0->checksum)
1307         {
1308           new_port0 = udp0->dst_port = s0->in2out.port;
1309           sum0 = udp0->checksum;
1310           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1311                                  dst_address);
1312           sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1313                                  length);
1314           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1315             {
1316               sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1317                                      s0->ext_host_nat_addr.as_u32,
1318                                      ip4_header_t, dst_address);
1319               sum0 =
1320                 ip_csum_update (sum0,
1321                                 vnet_buffer (b0)->ip.reass.l4_src_port,
1322                                 s0->ext_host_nat_port, ip4_header_t, length);
1323               udp0->src_port = s0->ext_host_nat_port;
1324               ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1325             }
1326           udp0->checksum = ip_csum_fold (sum0);
1327           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1328                                          thread_index, sw_if_index0, 1);
1329         }
1330       else
1331         {
1332           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1333             {
1334               new_port0 = udp0->dst_port = s0->in2out.port;
1335               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1336                 {
1337                   udp0->src_port = s0->ext_host_nat_port;
1338                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1339                 }
1340             }
1341           vlib_increment_simple_counter (&sm->counters.slowpath.out2in_ed.udp,
1342                                          thread_index, sw_if_index0, 1);
1343         }
1344
1345       /* Accounting */
1346       nat44_session_update_counters (s0, now,
1347                                      vlib_buffer_length_in_chain (vm, b0),
1348                                      thread_index);
1349       /* Per-user LRU list maintenance */
1350       nat44_session_update_lru (sm, s0, thread_index);
1351
1352     trace0:
1353       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1354                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1355         {
1356           nat44_ed_out2in_trace_t *t =
1357             vlib_add_trace (vm, node, b0, sizeof (*t));
1358           t->sw_if_index = sw_if_index0;
1359           t->next_index = next[0];
1360           t->is_slow_path = 1;
1361
1362           if (s0)
1363             t->session_index = s0 - tsm->sessions;
1364           else
1365             t->session_index = ~0;
1366         }
1367
1368       if (next[0] == NAT_NEXT_DROP)
1369         {
1370           vlib_increment_simple_counter (&sm->counters.slowpath.
1371                                          out2in_ed.drops, thread_index,
1372                                          sw_if_index0, 1);
1373         }
1374
1375       n_left_from--;
1376       next++;
1377       b++;
1378     }
1379
1380   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1381                                frame->n_vectors);
1382
1383   return frame->n_vectors;
1384 }
1385
1386 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1387                                      vlib_node_runtime_t * node,
1388                                      vlib_frame_t * frame)
1389 {
1390   if (snat_main.num_workers > 1)
1391     {
1392       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1393     }
1394   else
1395     {
1396       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1397     }
1398 }
1399
1400 /* *INDENT-OFF* */
1401 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1402   .name = "nat44-ed-out2in",
1403   .vector_size = sizeof (u32),
1404   .sibling_of = "nat-default",
1405   .format_trace = format_nat44_ed_out2in_trace,
1406   .type = VLIB_NODE_TYPE_INTERNAL,
1407   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1408   .error_strings = nat_out2in_ed_error_strings,
1409   .runtime_data_bytes = sizeof (snat_runtime_t),
1410 };
1411 /* *INDENT-ON* */
1412
1413 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1414                                               vlib_node_runtime_t * node,
1415                                               vlib_frame_t * frame)
1416 {
1417   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1418 }
1419
1420 /* *INDENT-OFF* */
1421 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1422   .name = "nat44-ed-out2in-slowpath",
1423   .vector_size = sizeof (u32),
1424   .sibling_of = "nat-default",
1425   .format_trace = format_nat44_ed_out2in_trace,
1426   .type = VLIB_NODE_TYPE_INTERNAL,
1427   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1428   .error_strings = nat_out2in_ed_error_strings,
1429   .runtime_data_bytes = sizeof (snat_runtime_t),
1430 };
1431 /* *INDENT-ON* */
1432
1433 static u8 *
1434 format_nat_pre_trace (u8 * s, va_list * args)
1435 {
1436   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1437   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1438   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1439   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1440                  t->arc_next_index);
1441 }
1442
1443 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1444                                     vlib_node_runtime_t * node,
1445                                     vlib_frame_t * frame)
1446 {
1447   return nat_pre_node_fn_inline (vm, node, frame,
1448                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1449 }
1450
1451 /* *INDENT-OFF* */
1452 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1453   .name = "nat-pre-out2in",
1454   .vector_size = sizeof (u32),
1455   .sibling_of = "nat-default",
1456   .format_trace = format_nat_pre_trace,
1457   .type = VLIB_NODE_TYPE_INTERNAL,
1458   .n_errors = 0,
1459  };
1460 /* *INDENT-ON* */
1461
1462 /*
1463  * fd.io coding-style-patch-verification: ON
1464  *
1465  * Local Variables:
1466  * eval: (c-set-style "gnu")
1467  * End:
1468  */