src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 static char *nat_out2in_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_out2in_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat44_ed_out2in_trace_t;
  48
  49 static u8 *
  50 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  59     "NAT44_OUT2IN_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 static inline u32
  68 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  69                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  70                           u32 sw_if_index0, u32 rx_fib_index0,
  71                           vlib_node_runtime_t * node, u32 next0, f64 now,
  72                           u32 thread_index, snat_session_t ** p_s0)
  73 {
  74   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
  75   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  76                        next0, thread_index, p_s0, 0);
  77   snat_session_t *s0 = *p_s0;
  78   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  79     {
  80       /* Accounting */
  81       nat44_session_update_counters (s0, now,
  82                                      vlib_buffer_length_in_chain
  83                                      (tsm->vlib_main, b0), thread_index);
  84       /* Per-user LRU list maintenance */
  85       nat44_session_update_lru (sm, s0, thread_index);
  86     }
  87   return next0;
  88 }
  89
  90 #ifndef CLIB_MARCH_VARIANT
  91 int
  92 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  93 {
  94   snat_main_t *sm = &snat_main;
  95   nat44_is_idle_session_ctx_t *ctx = arg;
  96   snat_session_t *s;
  97   u64 sess_timeout_time;
  98   u8 proto;
  99   u16 r_port, l_port;
 100   ip4_address_t *l_addr, *r_addr;
 101   u32 fib_index;
 102   clib_bihash_kv_16_8_t ed_kv;
 103   int i;
 104   snat_address_t *a;
 105   snat_session_key_t key;
 106   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 107                                                        ctx->thread_index);
 108
 109   s = pool_elt_at_index (tsm->sessions, kv->value);
 110   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 111   if (ctx->now >= sess_timeout_time)
 112     {
 113       l_addr = &s->in2out.addr;
 114       r_addr = &s->ext_host_addr;
 115       fib_index = s->in2out.fib_index;
 116       if (snat_is_unk_proto_session (s))
 117         {
 118           proto = s->in2out.port;
 119           r_port = 0;
 120           l_port = 0;
 121         }
 122       else
 123         {
 124           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 125           l_port = s->in2out.port;
 126           r_port = s->ext_host_port;
 127         }
 128       if (is_twice_nat_session (s))
 129         {
 130           r_addr = &s->ext_host_nat_addr;
 131           r_port = s->ext_host_nat_port;
 132         }
 133       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 134                   &ed_kv);
 135       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 136         nat_elog_warn ("in2out_ed key del failed");
 137
 138       if (snat_is_unk_proto_session (s))
 139         goto delete;
 140
 141       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 142                                            s->in2out.addr.as_u32,
 143                                            s->out2in.addr.as_u32,
 144                                            s->in2out.protocol,
 145                                            s->in2out.port,
 146                                            s->out2in.port,
 147                                            s->in2out.fib_index);
 148
 149       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 150                              &s->in2out.addr, s->in2out.port,
 151                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 152                              &s->out2in.addr, s->out2in.port,
 153                              &s->ext_host_addr, s->ext_host_port,
 154                              s->in2out.protocol, is_twice_nat_session (s));
 155
 156       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 157                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 158                    ctx->thread_index);
 159
 160       if (is_twice_nat_session (s))
 161         {
 162           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 163             {
 164               key.protocol = s->in2out.protocol;
 165               key.port = s->ext_host_nat_port;
 166               a = sm->twice_nat_addresses + i;
 167               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 168                 {
 169                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 170                                                       ctx->thread_index,
 171                                                       &key);
 172                   break;
 173                 }
 174             }
 175         }
 176
 177       if (snat_is_session_static (s))
 178         goto delete;
 179
 180       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 181                                           &s->out2in);
 182     delete:
 183       nat44_ed_delete_session (sm, s, ctx->thread_index, 1);
 184       return 1;
 185     }
 186
 187   return 0;
 188 }
 189 #endif
 190
 191 static snat_session_t *
 192 create_session_for_static_mapping_ed (snat_main_t * sm,
 193                                       vlib_buffer_t * b,
 194                                       snat_session_key_t l_key,
 195                                       snat_session_key_t e_key,
 196                                       vlib_node_runtime_t * node,
 197                                       u32 rx_fib_index,
 198                                       u32 thread_index,
 199                                       twice_nat_type_t twice_nat,
 200                                       lb_nat_type_t lb_nat, f64 now)
 201 {
 202   snat_session_t *s;
 203   ip4_header_t *ip;
 204   udp_header_t *udp;
 205   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 206   clib_bihash_kv_16_8_t kv;
 207   snat_session_key_t eh_key;
 208   nat44_is_idle_session_ctx_t ctx;
 209
 210   if (PREDICT_FALSE
 211       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 212     {
 213       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 214       nat_elog_notice ("maximum sessions exceeded");
 215       return 0;
 216     }
 217
 218   s = nat_ed_session_alloc (sm, thread_index, now);
 219   if (!s)
 220     {
 221       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 222       nat_elog_warn ("create NAT session failed");
 223       return 0;
 224     }
 225
 226   ip = vlib_buffer_get_current (b);
 227   udp = ip4_next_header (ip);
 228
 229   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 230   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 231   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 232   if (lb_nat)
 233     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 234   if (lb_nat == AFFINITY_LB_NAT)
 235     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 236   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 237   s->out2in = e_key;
 238   s->in2out = l_key;
 239   s->in2out.protocol = s->out2in.protocol;
 240
 241   /* Add to lookup tables */
 242   make_ed_kv (&e_key.addr, &s->ext_host_addr, ip->protocol,
 243               e_key.fib_index, e_key.port, s->ext_host_port,
 244               s - tsm->sessions, &kv);
 245   ctx.now = now;
 246   ctx.thread_index = thread_index;
 247   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
 248                                                nat44_o2i_ed_is_idle_session_cb,
 249                                                &ctx))
 250     nat_elog_notice ("out2in-ed key add failed");
 251
 252   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 253                                  ip->src_address.as_u32 == l_key.addr.as_u32))
 254     {
 255       eh_key.protocol = e_key.protocol;
 256       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 257                                                thread_index, &eh_key,
 258                                                sm->port_per_thread,
 259                                                tsm->snat_thread_index))
 260         {
 261           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 262           nat44_ed_delete_session (sm, s, thread_index, 1);
 263           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
 264             nat_elog_notice ("out2in-ed key del failed");
 265           return 0;
 266         }
 267       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 268       s->ext_host_nat_port = eh_key.port;
 269       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 270       make_ed_kv (&l_key.addr, &s->ext_host_nat_addr, ip->protocol,
 271                   l_key.fib_index, l_key.port, s->ext_host_nat_port,
 272                   s - tsm->sessions, &kv);
 273     }
 274   else
 275     {
 276       make_ed_kv (&l_key.addr, &s->ext_host_addr, ip->protocol,
 277                   l_key.fib_index, l_key.port, s->ext_host_port,
 278                   s - tsm->sessions, &kv);
 279     }
 280   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 281                                                nat44_i2o_ed_is_idle_session_cb,
 282                                                &ctx))
 283     nat_elog_notice ("in2out-ed key add failed");
 284
 285   snat_ipfix_logging_nat44_ses_create (thread_index,
 286                                        s->in2out.addr.as_u32,
 287                                        s->out2in.addr.as_u32,
 288                                        s->in2out.protocol,
 289                                        s->in2out.port,
 290                                        s->out2in.port, s->in2out.fib_index);
 291
 292   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 293                          &s->in2out.addr, s->in2out.port,
 294                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 295                          &s->out2in.addr, s->out2in.port,
 296                          &s->ext_host_addr, s->ext_host_port,
 297                          s->in2out.protocol, is_twice_nat_session (s));
 298
 299   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 300                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 301                &s->ext_host_nat_addr, s->ext_host_nat_port,
 302                s->in2out.protocol, s->in2out.fib_index, s->flags,
 303                thread_index, 0);
 304
 305   return s;
 306 }
 307
 308 static int
 309 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
 310               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 311 {
 312   clib_bihash_kv_16_8_t kv, value;
 313   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 314
 315   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 316               rx_fib_index, src_port, dst_port, ~0ULL, &kv);
 317   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 318     return 1;
 319
 320   return 0;
 321 }
 322
 323 static void
 324 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
 325                        u32 rx_fib_index, u32 thread_index)
 326 {
 327   clib_bihash_kv_16_8_t kv, value;
 328   udp_header_t *udp;
 329   snat_session_t *s = 0;
 330   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 331   f64 now = vlib_time_now (vlib_mains[thread_index]);
 332   u16 l_port, r_port;
 333
 334   if (ip->protocol == IP_PROTOCOL_ICMP)
 335     {
 336       if (get_icmp_o2i_ed_key
 337           (b, ip, rx_fib_index, ~0ULL, 0, &l_port, &r_port, &kv))
 338         return;
 339     }
 340   else
 341     {
 342       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 343         {
 344           udp = ip4_next_header (ip);
 345           l_port = udp->dst_port;
 346           r_port = udp->src_port;
 347         }
 348       else
 349         {
 350           l_port = 0;
 351           r_port = 0;
 352         }
 353       make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 354                   rx_fib_index, l_port, r_port, ~0ULL, &kv);
 355     }
 356
 357   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 358     {
 359       s = pool_elt_at_index (tsm->sessions, value.value);
 360     }
 361   else
 362     {
 363       u32 proto;
 364
 365       if (PREDICT_FALSE
 366           (nat44_ed_maximum_sessions_exceeded
 367            (sm, rx_fib_index, thread_index)))
 368         return;
 369
 370       s = nat_ed_session_alloc (sm, thread_index, now);
 371       if (!s)
 372         {
 373           nat_elog_warn ("create NAT session failed");
 374           return;
 375         }
 376
 377       proto = ip_proto_to_snat_proto (ip->protocol);
 378
 379       s->ext_host_addr = ip->src_address;
 380       s->ext_host_port = r_port;
 381       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 382       s->out2in.addr = ip->dst_address;
 383       s->out2in.port = l_port;
 384       s->out2in.protocol = proto;
 385       if (proto == ~0)
 386         {
 387           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 388           s->out2in.port = ip->protocol;
 389         }
 390       s->out2in.fib_index = 0;
 391       s->in2out = s->out2in;
 392
 393       kv.value = s - tsm->sessions;
 394       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 395         nat_elog_notice ("in2out_ed key add failed");
 396     }
 397
 398   if (ip->protocol == IP_PROTOCOL_TCP)
 399     {
 400       tcp_header_t *tcp = ip4_next_header (ip);
 401       if (nat44_set_tcp_session_state_o2i
 402           (sm, now, s, tcp->flags, tcp->ack_number, tcp->seq_number,
 403            thread_index))
 404         return;
 405     }
 406
 407   /* Accounting */
 408   nat44_session_update_counters (s, now, 0, thread_index);
 409   /* Per-user LRU list maintenance */
 410   nat44_session_update_lru (sm, s, thread_index);
 411 }
 412
 413 static inline void
 414 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
 415                               ip4_header_t * ip, u32 rx_fib_index)
 416 {
 417   ip4_header_t ip_wkr = {
 418     .src_address = ip->dst_address,
 419   };
 420   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 421
 422   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 423 }
 424
 425 #ifndef CLIB_MARCH_VARIANT
 426 u32
 427 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 428                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 429                       u8 * p_proto, snat_session_key_t * p_value,
 430                       u8 * p_dont_translate, void *d, void *e)
 431 {
 432   u32 next = ~0, sw_if_index, rx_fib_index;
 433   clib_bihash_kv_16_8_t kv, value;
 434   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 435   snat_session_t *s = 0;
 436   u8 dont_translate = 0, is_addr_only, identity_nat;
 437   snat_session_key_t e_key, l_key;
 438   u16 l_port, r_port;
 439
 440   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 441   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 442
 443   if (get_icmp_o2i_ed_key
 444       (b, ip, rx_fib_index, ~0ULL, p_proto, &l_port, &r_port, &kv))
 445     {
 446       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 447       next = NAT_NEXT_DROP;
 448       goto out;
 449     }
 450
 451   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 452     {
 453       /* Try to match static mapping */
 454       e_key.addr = ip->dst_address;
 455       e_key.port = l_port;
 456       e_key.protocol = ip_proto_to_snat_proto (ip->protocol);
 457       e_key.fib_index = rx_fib_index;
 458       if (snat_static_mapping_match
 459           (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
 460         {
 461           if (!sm->forwarding_enabled)
 462             {
 463               /* Don't NAT packet aimed at the intfc address */
 464               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 465                                                     ip->dst_address.as_u32)))
 466                 {
 467                   dont_translate = 1;
 468                   goto out;
 469                 }
 470               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 471               next = NAT_NEXT_DROP;
 472               goto out;
 473             }
 474           else
 475             {
 476               dont_translate = 1;
 477               if (next_src_nat (sm, ip, l_port, r_port,
 478                                 thread_index, rx_fib_index))
 479                 {
 480                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 481                   goto out;
 482                 }
 483               if (sm->num_workers > 1)
 484                 create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
 485               else
 486                 create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 487               goto out;
 488             }
 489         }
 490
 491       if (PREDICT_FALSE
 492           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 493            ICMP4_echo_reply
 494            && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 495                ICMP4_echo_request || !is_addr_only)))
 496         {
 497           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 498           next = NAT_NEXT_DROP;
 499           goto out;
 500         }
 501
 502       if (PREDICT_FALSE (identity_nat))
 503         {
 504           dont_translate = 1;
 505           goto out;
 506         }
 507
 508       /* Create session initiated by host from external network */
 509       s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
 510                                                 rx_fib_index, thread_index, 0,
 511                                                 0,
 512                                                 vlib_time_now
 513                                                 (tsm->vlib_main));
 514
 515       if (!s)
 516         {
 517           next = NAT_NEXT_DROP;
 518           goto out;
 519         }
 520     }
 521   else
 522     {
 523       if (PREDICT_FALSE
 524           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 525            ICMP4_echo_reply
 526            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 527            ICMP4_echo_request
 528            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 529                                            reass.icmp_type_or_tcp_flags)))
 530         {
 531           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 532           next = NAT_NEXT_DROP;
 533           goto out;
 534         }
 535
 536       s = pool_elt_at_index (tsm->sessions, value.value);
 537     }
 538 out:
 539   if (s)
 540     *p_value = s->in2out;
 541   *p_dont_translate = dont_translate;
 542   if (d)
 543     *(snat_session_t **) d = s;
 544   return next;
 545 }
 546 #endif
 547
 548 static snat_session_t *
 549 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 550                                vlib_buffer_t * b,
 551                                ip4_header_t * ip,
 552                                u32 rx_fib_index,
 553                                u32 thread_index,
 554                                f64 now,
 555                                vlib_main_t * vm, vlib_node_runtime_t * node)
 556 {
 557   clib_bihash_kv_8_8_t kv, value;
 558   clib_bihash_kv_16_8_t s_kv, s_value;
 559   snat_static_mapping_t *m;
 560   u32 old_addr, new_addr;
 561   ip_csum_t sum;
 562   snat_session_t *s;
 563   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 564
 565   old_addr = ip->dst_address.as_u32;
 566
 567   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol, rx_fib_index,
 568               0, 0, ~0ULL, &s_kv);
 569
 570   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 571     {
 572       s = pool_elt_at_index (tsm->sessions, s_value.value);
 573       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 574     }
 575   else
 576     {
 577       if (PREDICT_FALSE
 578           (nat44_ed_maximum_sessions_exceeded
 579            (sm, rx_fib_index, thread_index)))
 580         {
 581           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 582           nat_elog_notice ("maximum sessions exceeded");
 583           return 0;
 584         }
 585
 586       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 587       if (clib_bihash_search_8_8
 588           (&sm->static_mapping_by_external, &kv, &value))
 589         {
 590           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 591           return 0;
 592         }
 593
 594       m = pool_elt_at_index (sm->static_mappings, value.value);
 595
 596       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 597
 598       /* Create a new session */
 599       s = nat_ed_session_alloc (sm, thread_index, now);
 600       if (!s)
 601         {
 602           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 603           nat_elog_warn ("create NAT session failed");
 604           return 0;
 605         }
 606
 607       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 608       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 609       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 610       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 611       s->out2in.addr.as_u32 = old_addr;
 612       s->out2in.fib_index = rx_fib_index;
 613       s->in2out.addr.as_u32 = new_addr;
 614       s->in2out.fib_index = m->fib_index;
 615       s->in2out.port = s->out2in.port = ip->protocol;
 616
 617       /* Add to lookup tables */
 618       s_kv.value = s - tsm->sessions;
 619       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 620         nat_elog_notice ("out2in key add failed");
 621
 622       make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 623                   m->fib_index, 0, 0, s - tsm->sessions, &s_kv);
 624       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 625         nat_elog_notice ("in2out key add failed");
 626     }
 627
 628   /* Update IP checksum */
 629   sum = ip->checksum;
 630   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 631   ip->checksum = ip_csum_fold (sum);
 632
 633   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 634
 635   /* Accounting */
 636   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 637                                  thread_index);
 638   /* Per-user LRU list maintenance */
 639   nat44_session_update_lru (sm, s, thread_index);
 640
 641   return s;
 642 }
 643
 644 static inline uword
 645 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 646                                           vlib_node_runtime_t * node,
 647                                           vlib_frame_t * frame)
 648 {
 649   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 650   nat_next_t next_index;
 651   snat_main_t *sm = &snat_main;
 652   f64 now = vlib_time_now (vm);
 653   u32 thread_index = vm->thread_index;
 654   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 655   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 656     0, fragments = 0;
 657
 658   stats_node_index = sm->ed_out2in_node_index;
 659
 660   from = vlib_frame_vector_args (frame);
 661   n_left_from = frame->n_vectors;
 662   next_index = node->cached_next_index;
 663
 664   while (n_left_from > 0)
 665     {
 666       u32 n_left_to_next;
 667
 668       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 669
 670       while (n_left_from > 0 && n_left_to_next > 0)
 671         {
 672           u32 bi0;
 673           vlib_buffer_t *b0;
 674           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 675             new_addr0;
 676           u16 old_port0, new_port0;
 677           ip4_header_t *ip0;
 678           udp_header_t *udp0;
 679           tcp_header_t *tcp0;
 680           snat_session_t *s0 = 0;
 681           clib_bihash_kv_16_8_t kv0, value0;
 682           ip_csum_t sum0;
 683
 684           /* speculatively enqueue b0 to the current next frame */
 685           bi0 = from[0];
 686           to_next[0] = bi0;
 687           from += 1;
 688           to_next += 1;
 689           n_left_from -= 1;
 690           n_left_to_next -= 1;
 691
 692           b0 = vlib_get_buffer (vm, bi0);
 693           next0 = vnet_buffer2 (b0)->nat.arc_next;
 694
 695           vnet_buffer (b0)->snat.flags = 0;
 696           ip0 = vlib_buffer_get_current (b0);
 697
 698           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 699           rx_fib_index0 =
 700             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 701                                                  sw_if_index0);
 702
 703           if (PREDICT_FALSE (ip0->ttl == 1))
 704             {
 705               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 706               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 707                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 708                                            0);
 709               next0 = NAT_NEXT_ICMP_ERROR;
 710               goto trace0;
 711             }
 712
 713           udp0 = ip4_next_header (ip0);
 714           tcp0 = (tcp_header_t *) udp0;
 715           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 716
 717           if (PREDICT_FALSE (proto0 == ~0))
 718             {
 719               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 720               goto trace0;
 721             }
 722
 723           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 724             {
 725               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 726               goto trace0;
 727             }
 728
 729           make_ed_kv (&ip0->dst_address, &ip0->src_address,
 730                       ip0->protocol, rx_fib_index0,
 731                       vnet_buffer (b0)->ip.reass.l4_dst_port,
 732                       vnet_buffer (b0)->ip.reass.l4_src_port, ~0ULL, &kv0);
 733
 734           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
 735             {
 736               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 737               goto trace0;
 738             }
 739           s0 = pool_elt_at_index (tsm->sessions, value0.value);
 740
 741           if (s0->tcp_close_timestamp)
 742             {
 743               if (now >= s0->tcp_close_timestamp)
 744                 {
 745                   // session is closed, go slow path
 746                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 747                 }
 748               else
 749                 {
 750                   // session in transitory timeout, drop
 751                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 752                   next0 = NAT_NEXT_DROP;
 753                 }
 754               goto trace0;
 755             }
 756
 757           // drop if session expired
 758           u64 sess_timeout_time;
 759           sess_timeout_time = s0->last_heard +
 760             (f64) nat44_session_get_timeout (sm, s0);
 761           if (now >= sess_timeout_time)
 762             {
 763               // session is closed, go slow path
 764               nat_free_session_data (sm, s0, thread_index, 0);
 765               nat44_ed_delete_session (sm, s0, thread_index, 1);
 766               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 767               goto trace0;
 768             }
 769           //
 770
 771           old_addr0 = ip0->dst_address.as_u32;
 772           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 773           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 774
 775           sum0 = ip0->checksum;
 776           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 777                                  dst_address);
 778           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 779             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 780                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
 781                                    src_address);
 782           ip0->checksum = ip_csum_fold (sum0);
 783
 784           old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 785
 786           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 787             {
 788               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 789                 {
 790                   new_port0 = udp0->dst_port = s0->in2out.port;
 791                   sum0 = tcp0->checksum;
 792                   sum0 =
 793                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 794                                     dst_address);
 795                   sum0 =
 796                     ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 797                                     length);
 798                   if (is_twice_nat_session (s0))
 799                     {
 800                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 801                                              s0->ext_host_nat_addr.as_u32,
 802                                              ip4_header_t, dst_address);
 803                       sum0 =
 804                         ip_csum_update (sum0,
 805                                         vnet_buffer (b0)->ip.
 806                                         reass.l4_src_port,
 807                                         s0->ext_host_nat_port, ip4_header_t,
 808                                         length);
 809                       tcp0->src_port = s0->ext_host_nat_port;
 810                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 811                     }
 812                   tcp0->checksum = ip_csum_fold (sum0);
 813                 }
 814               tcp_packets++;
 815               if (nat44_set_tcp_session_state_o2i
 816                   (sm, now, s0,
 817                    vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
 818                    vnet_buffer (b0)->ip.reass.tcp_ack_number,
 819                    vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
 820                 goto trace0;
 821             }
 822           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
 823                    && udp0->checksum)
 824             {
 825               new_port0 = udp0->dst_port = s0->in2out.port;
 826               sum0 = udp0->checksum;
 827               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 828                                      dst_address);
 829               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 830                                      length);
 831               if (PREDICT_FALSE (is_twice_nat_session (s0)))
 832                 {
 833                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 834                                          s0->ext_host_nat_addr.as_u32,
 835                                          ip4_header_t, dst_address);
 836                   sum0 =
 837                     ip_csum_update (sum0,
 838                                     vnet_buffer (b0)->ip.reass.l4_src_port,
 839                                     s0->ext_host_nat_port, ip4_header_t,
 840                                     length);
 841                   udp0->src_port = s0->ext_host_nat_port;
 842                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 843                 }
 844               udp0->checksum = ip_csum_fold (sum0);
 845               udp_packets++;
 846             }
 847           else
 848             {
 849               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 850                 {
 851                   new_port0 = udp0->dst_port = s0->in2out.port;
 852                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
 853                     {
 854                       udp0->src_port = s0->ext_host_nat_port;
 855                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 856                     }
 857                 }
 858               udp_packets++;
 859             }
 860
 861           /* Accounting */
 862           nat44_session_update_counters (s0, now,
 863                                          vlib_buffer_length_in_chain (vm, b0),
 864                                          thread_index);
 865           /* Per-user LRU list maintenance */
 866           nat44_session_update_lru (sm, s0, thread_index);
 867
 868         trace0:
 869           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 870                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 871             {
 872               nat44_ed_out2in_trace_t *t =
 873                 vlib_add_trace (vm, node, b0, sizeof (*t));
 874               t->sw_if_index = sw_if_index0;
 875               t->next_index = next0;
 876               t->is_slow_path = 0;
 877
 878               if (s0)
 879                 t->session_index = s0 - tsm->sessions;
 880               else
 881                 t->session_index = ~0;
 882             }
 883
 884           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
 885           /* verify speculative enqueue, maybe switch current next frame */
 886           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 887                                            to_next, n_left_to_next,
 888                                            bi0, next0);
 889         }
 890
 891       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 892     }
 893
 894   vlib_node_increment_counter (vm, stats_node_index,
 895                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
 896                                pkts_processed);
 897   vlib_node_increment_counter (vm, stats_node_index,
 898                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
 899   vlib_node_increment_counter (vm, stats_node_index,
 900                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
 901   vlib_node_increment_counter (vm, stats_node_index,
 902                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
 903                                icmp_packets);
 904   vlib_node_increment_counter (vm, stats_node_index,
 905                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
 906                                other_packets);
 907   vlib_node_increment_counter (vm, stats_node_index,
 908                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
 909   return frame->n_vectors;
 910 }
 911
 912 static inline uword
 913 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
 914                                           vlib_node_runtime_t * node,
 915                                           vlib_frame_t * frame)
 916 {
 917   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 918   nat_next_t next_index;
 919   snat_main_t *sm = &snat_main;
 920   // HERE
 921   f64 now = vlib_time_now (vm);
 922   u32 thread_index = vm->thread_index;
 923   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 924   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 925     0, fragments = 0;
 926
 927   stats_node_index = sm->ed_out2in_slowpath_node_index;
 928
 929   from = vlib_frame_vector_args (frame);
 930   n_left_from = frame->n_vectors;
 931   next_index = node->cached_next_index;
 932
 933   while (n_left_from > 0)
 934     {
 935       u32 n_left_to_next;
 936
 937       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 938
 939       while (n_left_from > 0 && n_left_to_next > 0)
 940         {
 941           u32 bi0;
 942           vlib_buffer_t *b0;
 943           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 944             new_addr0;
 945           u16 old_port0, new_port0;
 946           ip4_header_t *ip0;
 947           udp_header_t *udp0;
 948           tcp_header_t *tcp0;
 949           icmp46_header_t *icmp0;
 950           snat_session_t *s0 = 0;
 951           clib_bihash_kv_16_8_t kv0, value0;
 952           ip_csum_t sum0;
 953           snat_session_key_t e_key0, l_key0;
 954           lb_nat_type_t lb_nat0;
 955           twice_nat_type_t twice_nat0;
 956           u8 identity_nat0;
 957
 958           /* speculatively enqueue b0 to the current next frame */
 959           bi0 = from[0];
 960           to_next[0] = bi0;
 961           from += 1;
 962           to_next += 1;
 963           n_left_from -= 1;
 964           n_left_to_next -= 1;
 965
 966           b0 = vlib_get_buffer (vm, bi0);
 967           next0 = vnet_buffer2 (b0)->nat.arc_next;
 968
 969           vnet_buffer (b0)->snat.flags = 0;
 970           ip0 = vlib_buffer_get_current (b0);
 971
 972           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 973           rx_fib_index0 =
 974             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 975                                                  sw_if_index0);
 976
 977           if (PREDICT_FALSE (ip0->ttl == 1))
 978             {
 979               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 980               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 981                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 982                                            0);
 983               next0 = NAT_NEXT_ICMP_ERROR;
 984               goto trace0;
 985             }
 986
 987           udp0 = ip4_next_header (ip0);
 988           tcp0 = (tcp_header_t *) udp0;
 989           icmp0 = (icmp46_header_t *) udp0;
 990           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 991
 992           if (PREDICT_FALSE (proto0 == ~0))
 993             {
 994               s0 =
 995                 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
 996                                                thread_index, now, vm, node);
 997               if (!sm->forwarding_enabled)
 998                 {
 999                   if (!s0)
1000                     next0 = NAT_NEXT_DROP;
1001                 }
1002               other_packets++;
1003               goto trace0;
1004             }
1005
1006           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1007             {
1008               next0 = icmp_out2in_ed_slow_path
1009                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1010                  next0, now, thread_index, &s0);
1011               icmp_packets++;
1012               goto trace0;
1013             }
1014
1015           make_ed_kv (&ip0->dst_address, &ip0->src_address,
1016                       ip0->protocol, rx_fib_index0,
1017                       vnet_buffer (b0)->ip.reass.l4_dst_port,
1018                       vnet_buffer (b0)->ip.reass.l4_src_port, ~0ULL, &kv0);
1019
1020           s0 = NULL;
1021           if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1022             {
1023               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1024
1025               if (s0->tcp_close_timestamp && now >= s0->tcp_close_timestamp)
1026                 {
1027                   nat_free_session_data (sm, s0, thread_index, 0);
1028                   nat44_ed_delete_session (sm, s0, thread_index, 1);
1029                   s0 = NULL;
1030                 }
1031             }
1032
1033           if (!s0)
1034             {
1035               /* Try to match static mapping by external address and port,
1036                  destination address and port in packet */
1037               e_key0.addr = ip0->dst_address;
1038               e_key0.port = vnet_buffer (b0)->ip.reass.l4_dst_port;
1039               e_key0.protocol = proto0;
1040               e_key0.fib_index = rx_fib_index0;
1041
1042               if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1043                                              &twice_nat0, &lb_nat0,
1044                                              &ip0->src_address,
1045                                              &identity_nat0))
1046                 {
1047                   /*
1048                    * Send DHCP packets to the ipv4 stack, or we won't
1049                    * be able to use dhcp client on the outside interface
1050                    */
1051                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1052                                      && (vnet_buffer (b0)->ip.
1053                                          reass.l4_dst_port ==
1054                                          clib_host_to_net_u16
1055                                          (UDP_DST_PORT_dhcp_to_client))))
1056                     {
1057                       goto trace0;
1058                     }
1059
1060                   if (!sm->forwarding_enabled)
1061                     {
1062                       b0->error =
1063                         node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1064                       next0 = NAT_NEXT_DROP;
1065                     }
1066                   else
1067                     {
1068                       if (next_src_nat
1069                           (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1070                            vnet_buffer (b0)->ip.reass.l4_dst_port,
1071                            thread_index, rx_fib_index0))
1072                         {
1073                           next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1074                           goto trace0;
1075                         }
1076                       // TEST:
1077                       if (sm->num_workers > 1)
1078                         create_bypass_for_fwd_worker (sm, b0, ip0,
1079                                                       rx_fib_index0);
1080                       else
1081                         create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1082                                                thread_index);
1083                     }
1084                   goto trace0;
1085                 }
1086
1087               if (PREDICT_FALSE (identity_nat0))
1088                 goto trace0;
1089
1090               if ((proto0 == SNAT_PROTOCOL_TCP)
1091                   && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1092                                          reass.icmp_type_or_tcp_flags))
1093                 {
1094                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1095                   next0 = NAT_NEXT_DROP;
1096                   goto trace0;
1097                 }
1098
1099               /* Create session initiated by host from external network */
1100               s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1101                                                          e_key0, node,
1102                                                          rx_fib_index0,
1103                                                          thread_index,
1104                                                          twice_nat0,
1105                                                          lb_nat0, now);
1106               if (!s0)
1107                 {
1108                   next0 = NAT_NEXT_DROP;
1109                   goto trace0;
1110                 }
1111             }
1112
1113           old_addr0 = ip0->dst_address.as_u32;
1114           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1115           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1116
1117           sum0 = ip0->checksum;
1118           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1119                                  dst_address);
1120           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1121             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1122                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1123                                    src_address);
1124           ip0->checksum = ip_csum_fold (sum0);
1125
1126           old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1127
1128           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1129             {
1130               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1131                 {
1132                   new_port0 = udp0->dst_port = s0->in2out.port;
1133                   sum0 = tcp0->checksum;
1134                   sum0 =
1135                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1136                                     dst_address);
1137                   sum0 =
1138                     ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1139                                     length);
1140                   if (is_twice_nat_session (s0))
1141                     {
1142                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1143                                              s0->ext_host_nat_addr.as_u32,
1144                                              ip4_header_t, dst_address);
1145                       sum0 =
1146                         ip_csum_update (sum0,
1147                                         vnet_buffer (b0)->ip.
1148                                         reass.l4_src_port,
1149                                         s0->ext_host_nat_port, ip4_header_t,
1150                                         length);
1151                       tcp0->src_port = s0->ext_host_nat_port;
1152                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1153                     }
1154                   tcp0->checksum = ip_csum_fold (sum0);
1155                 }
1156               tcp_packets++;
1157               if (nat44_set_tcp_session_state_o2i
1158                   (sm, now, s0,
1159                    vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
1160                    vnet_buffer (b0)->ip.reass.tcp_ack_number,
1161                    vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
1162                 goto trace0;
1163             }
1164           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1165                    && udp0->checksum)
1166             {
1167               new_port0 = udp0->dst_port = s0->in2out.port;
1168               sum0 = udp0->checksum;
1169               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1170                                      dst_address);
1171               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1172                                      length);
1173               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1174                 {
1175                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1176                                          s0->ext_host_nat_addr.as_u32,
1177                                          ip4_header_t, dst_address);
1178                   sum0 =
1179                     ip_csum_update (sum0,
1180                                     vnet_buffer (b0)->ip.reass.l4_src_port,
1181                                     s0->ext_host_nat_port, ip4_header_t,
1182                                     length);
1183                   udp0->src_port = s0->ext_host_nat_port;
1184                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1185                 }
1186               udp0->checksum = ip_csum_fold (sum0);
1187               udp_packets++;
1188             }
1189           else
1190             {
1191               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1192                 {
1193                   new_port0 = udp0->dst_port = s0->in2out.port;
1194                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1195                     {
1196                       udp0->src_port = s0->ext_host_nat_port;
1197                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1198                     }
1199                 }
1200               udp_packets++;
1201             }
1202
1203           /* Accounting */
1204           nat44_session_update_counters (s0, now,
1205                                          vlib_buffer_length_in_chain (vm, b0),
1206                                          thread_index);
1207           /* Per-user LRU list maintenance */
1208           nat44_session_update_lru (sm, s0, thread_index);
1209
1210         trace0:
1211           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1212                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1213             {
1214               nat44_ed_out2in_trace_t *t =
1215                 vlib_add_trace (vm, node, b0, sizeof (*t));
1216               t->sw_if_index = sw_if_index0;
1217               t->next_index = next0;
1218               t->is_slow_path = 1;
1219
1220               if (s0)
1221                 t->session_index = s0 - tsm->sessions;
1222               else
1223                 t->session_index = ~0;
1224             }
1225
1226           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1227           /* verify speculative enqueue, maybe switch current next frame */
1228           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1229                                            to_next, n_left_to_next,
1230                                            bi0, next0);
1231         }
1232
1233       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1234     }
1235
1236   vlib_node_increment_counter (vm, stats_node_index,
1237                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1238                                pkts_processed);
1239   vlib_node_increment_counter (vm, stats_node_index,
1240                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1241   vlib_node_increment_counter (vm, stats_node_index,
1242                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1243   vlib_node_increment_counter (vm, stats_node_index,
1244                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1245                                icmp_packets);
1246   vlib_node_increment_counter (vm, stats_node_index,
1247                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1248                                other_packets);
1249   vlib_node_increment_counter (vm, stats_node_index,
1250                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1251   return frame->n_vectors;
1252 }
1253
1254 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1255                                      vlib_node_runtime_t * node,
1256                                      vlib_frame_t * frame)
1257 {
1258   return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame);
1259 }
1260
1261 /* *INDENT-OFF* */
1262 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1263   .name = "nat44-ed-out2in",
1264   .vector_size = sizeof (u32),
1265   .sibling_of = "nat-default",
1266   .format_trace = format_nat44_ed_out2in_trace,
1267   .type = VLIB_NODE_TYPE_INTERNAL,
1268   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1269   .error_strings = nat_out2in_ed_error_strings,
1270   .runtime_data_bytes = sizeof (snat_runtime_t),
1271 };
1272 /* *INDENT-ON* */
1273
1274 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1275                                               vlib_node_runtime_t * node,
1276                                               vlib_frame_t * frame)
1277 {
1278   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1279 }
1280
1281 /* *INDENT-OFF* */
1282 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1283   .name = "nat44-ed-out2in-slowpath",
1284   .vector_size = sizeof (u32),
1285   .sibling_of = "nat-default",
1286   .format_trace = format_nat44_ed_out2in_trace,
1287   .type = VLIB_NODE_TYPE_INTERNAL,
1288   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1289   .error_strings = nat_out2in_ed_error_strings,
1290   .runtime_data_bytes = sizeof (snat_runtime_t),
1291 };
1292 /* *INDENT-ON* */
1293
1294 static u8 *
1295 format_nat_pre_trace (u8 * s, va_list * args)
1296 {
1297   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1298   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1299   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1300   return format (s, "out2in next_index %d", t->next_index);
1301 }
1302
1303 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1304                                     vlib_node_runtime_t * node,
1305                                     vlib_frame_t * frame)
1306 {
1307   return nat_pre_node_fn_inline (vm, node, frame,
1308                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1309 }
1310
1311 /* *INDENT-OFF* */
1312 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1313   .name = "nat-pre-out2in",
1314   .vector_size = sizeof (u32),
1315   .sibling_of = "nat-default",
1316   .format_trace = format_nat_pre_trace,
1317   .type = VLIB_NODE_TYPE_INTERNAL,
1318   .n_errors = 0,
1319  };
1320 /* *INDENT-ON* */
1321
1322 /*
1323  * fd.io coding-style-patch-verification: ON
1324  *
1325  * Local Variables:
1326  * eval: (c-set-style "gnu")
1327  * End:
1328  */