src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 static char *nat_out2in_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_out2in_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat44_ed_out2in_trace_t;
  48
  49 static u8 *
  50 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  59     "NAT44_OUT2IN_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 static inline u32
  68 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  69                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  70                           u32 sw_if_index0, u32 rx_fib_index0,
  71                           vlib_node_runtime_t * node, u32 next0, f64 now,
  72                           u32 thread_index, snat_session_t ** p_s0)
  73 {
  74   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  75                        next0, thread_index, p_s0, 0);
  76   snat_session_t *s0 = *p_s0;
  77   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  78     {
  79       /* Accounting */
  80       nat44_session_update_counters (s0, now,
  81                                      vlib_buffer_length_in_chain
  82                                      (sm->vlib_main, b0), thread_index);
  83       /* Per-user LRU list maintenance */
  84       nat44_session_update_lru (sm, s0, thread_index);
  85     }
  86   return next0;
  87 }
  88
  89 #ifndef CLIB_MARCH_VARIANT
  90 int
  91 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  92 {
  93   snat_main_t *sm = &snat_main;
  94   nat44_is_idle_session_ctx_t *ctx = arg;
  95   snat_session_t *s;
  96   u64 sess_timeout_time;
  97   u8 proto;
  98   u16 r_port, l_port;
  99   ip4_address_t *l_addr, *r_addr;
 100   u32 fib_index;
 101   clib_bihash_kv_16_8_t ed_kv;
 102   int i;
 103   snat_address_t *a;
 104   snat_session_key_t key;
 105   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 106                                                        ctx->thread_index);
 107
 108   s = pool_elt_at_index (tsm->sessions, kv->value);
 109   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 110   if (ctx->now >= sess_timeout_time)
 111     {
 112       l_addr = &s->in2out.addr;
 113       r_addr = &s->ext_host_addr;
 114       fib_index = s->in2out.fib_index;
 115       if (snat_is_unk_proto_session (s))
 116         {
 117           proto = s->in2out.port;
 118           r_port = 0;
 119           l_port = 0;
 120         }
 121       else
 122         {
 123           proto = snat_proto_to_ip_proto (s->in2out.protocol);
 124           l_port = s->in2out.port;
 125           r_port = s->ext_host_port;
 126         }
 127       if (is_twice_nat_session (s))
 128         {
 129           r_addr = &s->ext_host_nat_addr;
 130           r_port = s->ext_host_nat_port;
 131         }
 132       make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0ULL,
 133                   &ed_kv);
 134       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 135         nat_elog_warn ("in2out_ed key del failed");
 136
 137       if (snat_is_unk_proto_session (s))
 138         goto delete;
 139
 140       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 141                                            s->in2out.addr.as_u32,
 142                                            s->out2in.addr.as_u32,
 143                                            s->in2out.protocol,
 144                                            s->in2out.port,
 145                                            s->out2in.port,
 146                                            s->in2out.fib_index);
 147
 148       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 149                              &s->in2out.addr, s->in2out.port,
 150                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 151                              &s->out2in.addr, s->out2in.port,
 152                              &s->ext_host_addr, s->ext_host_port,
 153                              s->in2out.protocol, is_twice_nat_session (s));
 154
 155       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 156                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 157                    ctx->thread_index);
 158
 159       if (is_twice_nat_session (s))
 160         {
 161           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 162             {
 163               key.protocol = s->in2out.protocol;
 164               key.port = s->ext_host_nat_port;
 165               a = sm->twice_nat_addresses + i;
 166               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 167                 {
 168                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 169                                                       ctx->thread_index,
 170                                                       &key);
 171                   break;
 172                 }
 173             }
 174         }
 175
 176       if (snat_is_session_static (s))
 177         goto delete;
 178
 179       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 180                                           &s->out2in);
 181     delete:
 182       nat44_ed_delete_session (sm, s, ctx->thread_index, 1);
 183       return 1;
 184     }
 185
 186   return 0;
 187 }
 188 #endif
 189
 190 static snat_session_t *
 191 create_session_for_static_mapping_ed (snat_main_t * sm,
 192                                       vlib_buffer_t * b,
 193                                       snat_session_key_t l_key,
 194                                       snat_session_key_t e_key,
 195                                       vlib_node_runtime_t * node,
 196                                       u32 rx_fib_index,
 197                                       u32 thread_index,
 198                                       twice_nat_type_t twice_nat,
 199                                       lb_nat_type_t lb_nat, f64 now)
 200 {
 201   snat_session_t *s;
 202   ip4_header_t *ip;
 203   udp_header_t *udp;
 204   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 205   clib_bihash_kv_16_8_t kv;
 206   snat_session_key_t eh_key;
 207   nat44_is_idle_session_ctx_t ctx;
 208
 209   if (PREDICT_FALSE
 210       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 211     {
 212       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 213       nat_elog_notice ("maximum sessions exceeded");
 214       return 0;
 215     }
 216
 217   s = nat_ed_session_alloc (sm, thread_index, now);
 218   if (!s)
 219     {
 220       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 221       nat_elog_warn ("create NAT session failed");
 222       return 0;
 223     }
 224
 225   ip = vlib_buffer_get_current (b);
 226   udp = ip4_next_header (ip);
 227
 228   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 229   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 230   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 231   if (lb_nat)
 232     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 233   if (lb_nat == AFFINITY_LB_NAT)
 234     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 235   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 236   s->out2in = e_key;
 237   s->in2out = l_key;
 238   s->in2out.protocol = s->out2in.protocol;
 239
 240   /* Add to lookup tables */
 241   make_ed_kv (&e_key.addr, &s->ext_host_addr, ip->protocol,
 242               e_key.fib_index, e_key.port, s->ext_host_port,
 243               s - tsm->sessions, &kv);
 244   ctx.now = now;
 245   ctx.thread_index = thread_index;
 246   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
 247                                                nat44_o2i_ed_is_idle_session_cb,
 248                                                &ctx))
 249     nat_elog_notice ("out2in-ed key add failed");
 250
 251   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 252                                  ip->src_address.as_u32 == l_key.addr.as_u32))
 253     {
 254       eh_key.protocol = e_key.protocol;
 255       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 256                                                thread_index, &eh_key,
 257                                                sm->port_per_thread,
 258                                                tsm->snat_thread_index))
 259         {
 260           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 261           nat44_ed_delete_session (sm, s, thread_index, 1);
 262           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
 263             nat_elog_notice ("out2in-ed key del failed");
 264           return 0;
 265         }
 266       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 267       s->ext_host_nat_port = eh_key.port;
 268       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 269       make_ed_kv (&l_key.addr, &s->ext_host_nat_addr, ip->protocol,
 270                   l_key.fib_index, l_key.port, s->ext_host_nat_port,
 271                   s - tsm->sessions, &kv);
 272     }
 273   else
 274     {
 275       make_ed_kv (&l_key.addr, &s->ext_host_addr, ip->protocol,
 276                   l_key.fib_index, l_key.port, s->ext_host_port,
 277                   s - tsm->sessions, &kv);
 278     }
 279   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 280                                                nat44_i2o_ed_is_idle_session_cb,
 281                                                &ctx))
 282     nat_elog_notice ("in2out-ed key add failed");
 283
 284   snat_ipfix_logging_nat44_ses_create (thread_index,
 285                                        s->in2out.addr.as_u32,
 286                                        s->out2in.addr.as_u32,
 287                                        s->in2out.protocol,
 288                                        s->in2out.port,
 289                                        s->out2in.port, s->in2out.fib_index);
 290
 291   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 292                          &s->in2out.addr, s->in2out.port,
 293                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 294                          &s->out2in.addr, s->out2in.port,
 295                          &s->ext_host_addr, s->ext_host_port,
 296                          s->in2out.protocol, is_twice_nat_session (s));
 297
 298   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 299                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 300                &s->ext_host_nat_addr, s->ext_host_nat_port,
 301                s->in2out.protocol, s->in2out.fib_index, s->flags,
 302                thread_index, 0);
 303
 304   return s;
 305 }
 306
 307 static int
 308 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
 309               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 310 {
 311   clib_bihash_kv_16_8_t kv, value;
 312   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 313
 314   make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
 315               rx_fib_index, src_port, dst_port, ~0ULL, &kv);
 316   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 317     return 1;
 318
 319   return 0;
 320 }
 321
 322 static void
 323 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
 324                        u32 rx_fib_index, u32 thread_index)
 325 {
 326   clib_bihash_kv_16_8_t kv, value;
 327   udp_header_t *udp;
 328   snat_session_t *s = 0;
 329   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 330   f64 now = vlib_time_now (sm->vlib_main);
 331   u16 l_port, r_port;
 332
 333   if (ip->protocol == IP_PROTOCOL_ICMP)
 334     {
 335       if (get_icmp_o2i_ed_key
 336           (b, ip, rx_fib_index, ~0ULL, 0, &l_port, &r_port, &kv))
 337         return;
 338     }
 339   else
 340     {
 341       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 342         {
 343           udp = ip4_next_header (ip);
 344           l_port = udp->dst_port;
 345           r_port = udp->src_port;
 346         }
 347       else
 348         {
 349           l_port = 0;
 350           r_port = 0;
 351         }
 352       make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 353                   rx_fib_index, l_port, r_port, ~0ULL, &kv);
 354     }
 355
 356   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 357     {
 358       s = pool_elt_at_index (tsm->sessions, value.value);
 359     }
 360   else
 361     {
 362       u32 proto;
 363
 364       if (PREDICT_FALSE
 365           (nat44_ed_maximum_sessions_exceeded
 366            (sm, rx_fib_index, thread_index)))
 367         return;
 368
 369       s = nat_ed_session_alloc (sm, thread_index, now);
 370       if (!s)
 371         {
 372           nat_elog_warn ("create NAT session failed");
 373           return;
 374         }
 375
 376       proto = ip_proto_to_snat_proto (ip->protocol);
 377
 378       s->ext_host_addr = ip->src_address;
 379       s->ext_host_port = r_port;
 380       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 381       s->out2in.addr = ip->dst_address;
 382       s->out2in.port = l_port;
 383       s->out2in.protocol = proto;
 384       if (proto == ~0)
 385         {
 386           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 387           s->out2in.port = ip->protocol;
 388         }
 389       s->out2in.fib_index = 0;
 390       s->in2out = s->out2in;
 391
 392       kv.value = s - tsm->sessions;
 393       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 394         nat_elog_notice ("in2out_ed key add failed");
 395     }
 396
 397   if (ip->protocol == IP_PROTOCOL_TCP)
 398     {
 399       tcp_header_t *tcp = ip4_next_header (ip);
 400       if (nat44_set_tcp_session_state_o2i
 401           (sm, now, s, tcp->flags, tcp->ack_number, tcp->seq_number,
 402            thread_index))
 403         return;
 404     }
 405
 406   /* Accounting */
 407   nat44_session_update_counters (s, now, 0, thread_index);
 408   /* Per-user LRU list maintenance */
 409   nat44_session_update_lru (sm, s, thread_index);
 410 }
 411
 412 static inline void
 413 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
 414                               ip4_header_t * ip, u32 rx_fib_index)
 415 {
 416   ip4_header_t ip_wkr = {
 417     .src_address = ip->dst_address,
 418   };
 419   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 420
 421   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 422 }
 423
 424 #ifndef CLIB_MARCH_VARIANT
 425 u32
 426 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 427                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 428                       u8 * p_proto, snat_session_key_t * p_value,
 429                       u8 * p_dont_translate, void *d, void *e)
 430 {
 431   u32 next = ~0, sw_if_index, rx_fib_index;
 432   clib_bihash_kv_16_8_t kv, value;
 433   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 434   snat_session_t *s = 0;
 435   u8 dont_translate = 0, is_addr_only, identity_nat;
 436   snat_session_key_t e_key, l_key;
 437   u16 l_port, r_port;
 438
 439   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 440   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 441
 442   if (get_icmp_o2i_ed_key
 443       (b, ip, rx_fib_index, ~0ULL, p_proto, &l_port, &r_port, &kv))
 444     {
 445       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 446       next = NAT_NEXT_DROP;
 447       goto out;
 448     }
 449
 450   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 451     {
 452       /* Try to match static mapping */
 453       e_key.addr = ip->dst_address;
 454       e_key.port = l_port;
 455       e_key.protocol = ip_proto_to_snat_proto (ip->protocol);
 456       e_key.fib_index = rx_fib_index;
 457       if (snat_static_mapping_match
 458           (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
 459         {
 460           if (!sm->forwarding_enabled)
 461             {
 462               /* Don't NAT packet aimed at the intfc address */
 463               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 464                                                     ip->dst_address.as_u32)))
 465                 {
 466                   dont_translate = 1;
 467                   goto out;
 468                 }
 469               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 470               next = NAT_NEXT_DROP;
 471               goto out;
 472             }
 473           else
 474             {
 475               dont_translate = 1;
 476               if (next_src_nat (sm, ip, l_port, r_port,
 477                                 thread_index, rx_fib_index))
 478                 {
 479                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 480                   goto out;
 481                 }
 482               if (sm->num_workers > 1)
 483                 create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
 484               else
 485                 create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
 486               goto out;
 487             }
 488         }
 489
 490       if (PREDICT_FALSE
 491           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 492            ICMP4_echo_reply
 493            && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 494                ICMP4_echo_request || !is_addr_only)))
 495         {
 496           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 497           next = NAT_NEXT_DROP;
 498           goto out;
 499         }
 500
 501       if (PREDICT_FALSE (identity_nat))
 502         {
 503           dont_translate = 1;
 504           goto out;
 505         }
 506
 507       /* Create session initiated by host from external network */
 508       s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
 509                                                 rx_fib_index, thread_index, 0,
 510                                                 0,
 511                                                 vlib_time_now
 512                                                 (sm->vlib_main));
 513
 514       if (!s)
 515         {
 516           next = NAT_NEXT_DROP;
 517           goto out;
 518         }
 519     }
 520   else
 521     {
 522       if (PREDICT_FALSE
 523           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 524            ICMP4_echo_reply
 525            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 526            ICMP4_echo_request
 527            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 528                                            reass.icmp_type_or_tcp_flags)))
 529         {
 530           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 531           next = NAT_NEXT_DROP;
 532           goto out;
 533         }
 534
 535       s = pool_elt_at_index (tsm->sessions, value.value);
 536     }
 537 out:
 538   if (s)
 539     *p_value = s->in2out;
 540   *p_dont_translate = dont_translate;
 541   if (d)
 542     *(snat_session_t **) d = s;
 543   return next;
 544 }
 545 #endif
 546
 547 static snat_session_t *
 548 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 549                                vlib_buffer_t * b,
 550                                ip4_header_t * ip,
 551                                u32 rx_fib_index,
 552                                u32 thread_index,
 553                                f64 now,
 554                                vlib_main_t * vm, vlib_node_runtime_t * node)
 555 {
 556   clib_bihash_kv_8_8_t kv, value;
 557   clib_bihash_kv_16_8_t s_kv, s_value;
 558   snat_static_mapping_t *m;
 559   u32 old_addr, new_addr;
 560   ip_csum_t sum;
 561   snat_session_t *s;
 562   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 563
 564   old_addr = ip->dst_address.as_u32;
 565
 566   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol, rx_fib_index,
 567               0, 0, ~0ULL, &s_kv);
 568
 569   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 570     {
 571       s = pool_elt_at_index (tsm->sessions, s_value.value);
 572       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 573     }
 574   else
 575     {
 576       if (PREDICT_FALSE
 577           (nat44_ed_maximum_sessions_exceeded
 578            (sm, rx_fib_index, thread_index)))
 579         {
 580           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 581           nat_elog_notice ("maximum sessions exceeded");
 582           return 0;
 583         }
 584
 585       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 586       if (clib_bihash_search_8_8
 587           (&sm->static_mapping_by_external, &kv, &value))
 588         {
 589           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 590           return 0;
 591         }
 592
 593       m = pool_elt_at_index (sm->static_mappings, value.value);
 594
 595       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 596
 597       /* Create a new session */
 598       s = nat_ed_session_alloc (sm, thread_index, now);
 599       if (!s)
 600         {
 601           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 602           nat_elog_warn ("create NAT session failed");
 603           return 0;
 604         }
 605
 606       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 607       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 608       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 609       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 610       s->out2in.addr.as_u32 = old_addr;
 611       s->out2in.fib_index = rx_fib_index;
 612       s->in2out.addr.as_u32 = new_addr;
 613       s->in2out.fib_index = m->fib_index;
 614       s->in2out.port = s->out2in.port = ip->protocol;
 615
 616       /* Add to lookup tables */
 617       s_kv.value = s - tsm->sessions;
 618       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 619         nat_elog_notice ("out2in key add failed");
 620
 621       make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 622                   m->fib_index, 0, 0, s - tsm->sessions, &s_kv);
 623       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 624         nat_elog_notice ("in2out key add failed");
 625     }
 626
 627   /* Update IP checksum */
 628   sum = ip->checksum;
 629   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 630   ip->checksum = ip_csum_fold (sum);
 631
 632   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 633
 634   /* Accounting */
 635   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 636                                  thread_index);
 637   /* Per-user LRU list maintenance */
 638   nat44_session_update_lru (sm, s, thread_index);
 639
 640   return s;
 641 }
 642
 643 static inline uword
 644 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 645                                           vlib_node_runtime_t * node,
 646                                           vlib_frame_t * frame)
 647 {
 648   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 649   nat_next_t next_index;
 650   snat_main_t *sm = &snat_main;
 651   f64 now = vlib_time_now (vm);
 652   u32 thread_index = vm->thread_index;
 653   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 654   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 655     0, fragments = 0;
 656
 657   stats_node_index = sm->ed_out2in_node_index;
 658
 659   from = vlib_frame_vector_args (frame);
 660   n_left_from = frame->n_vectors;
 661   next_index = node->cached_next_index;
 662
 663   while (n_left_from > 0)
 664     {
 665       u32 n_left_to_next;
 666
 667       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 668
 669       while (n_left_from > 0 && n_left_to_next > 0)
 670         {
 671           u32 bi0;
 672           vlib_buffer_t *b0;
 673           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 674             new_addr0;
 675           u16 old_port0, new_port0;
 676           ip4_header_t *ip0;
 677           udp_header_t *udp0;
 678           tcp_header_t *tcp0;
 679           snat_session_t *s0 = 0;
 680           clib_bihash_kv_16_8_t kv0, value0;
 681           ip_csum_t sum0;
 682
 683           /* speculatively enqueue b0 to the current next frame */
 684           bi0 = from[0];
 685           to_next[0] = bi0;
 686           from += 1;
 687           to_next += 1;
 688           n_left_from -= 1;
 689           n_left_to_next -= 1;
 690
 691           b0 = vlib_get_buffer (vm, bi0);
 692           next0 = vnet_buffer2 (b0)->nat.arc_next;
 693
 694           vnet_buffer (b0)->snat.flags = 0;
 695           ip0 = vlib_buffer_get_current (b0);
 696
 697           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 698           rx_fib_index0 =
 699             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 700                                                  sw_if_index0);
 701
 702           if (PREDICT_FALSE (ip0->ttl == 1))
 703             {
 704               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 705               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 706                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 707                                            0);
 708               next0 = NAT_NEXT_ICMP_ERROR;
 709               goto trace0;
 710             }
 711
 712           udp0 = ip4_next_header (ip0);
 713           tcp0 = (tcp_header_t *) udp0;
 714           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 715
 716           if (PREDICT_FALSE (proto0 == ~0))
 717             {
 718               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 719               goto trace0;
 720             }
 721
 722           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 723             {
 724               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 725               goto trace0;
 726             }
 727
 728           make_ed_kv (&ip0->dst_address, &ip0->src_address,
 729                       ip0->protocol, rx_fib_index0,
 730                       vnet_buffer (b0)->ip.reass.l4_dst_port,
 731                       vnet_buffer (b0)->ip.reass.l4_src_port, ~0ULL, &kv0);
 732
 733           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
 734             {
 735               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 736               goto trace0;
 737             }
 738           s0 = pool_elt_at_index (tsm->sessions, value0.value);
 739
 740           if (s0->tcp_close_timestamp)
 741             {
 742               if (now >= s0->tcp_close_timestamp)
 743                 {
 744                   // session is closed, go slow path
 745                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 746                 }
 747               else
 748                 {
 749                   // session in transitory timeout, drop
 750                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 751                   next0 = NAT_NEXT_DROP;
 752                 }
 753               goto trace0;
 754             }
 755
 756           // drop if session expired
 757           u64 sess_timeout_time;
 758           sess_timeout_time = s0->last_heard +
 759             (f64) nat44_session_get_timeout (sm, s0);
 760           if (now >= sess_timeout_time)
 761             {
 762               // session is closed, go slow path
 763               nat_free_session_data (sm, s0, thread_index, 0);
 764               nat44_ed_delete_session (sm, s0, thread_index, 1);
 765               next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 766               goto trace0;
 767             }
 768           //
 769
 770           old_addr0 = ip0->dst_address.as_u32;
 771           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 772           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 773
 774           sum0 = ip0->checksum;
 775           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 776                                  dst_address);
 777           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 778             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 779                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
 780                                    src_address);
 781           ip0->checksum = ip_csum_fold (sum0);
 782
 783           old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 784
 785           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 786             {
 787               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 788                 {
 789                   new_port0 = udp0->dst_port = s0->in2out.port;
 790                   sum0 = tcp0->checksum;
 791                   sum0 =
 792                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 793                                     dst_address);
 794                   sum0 =
 795                     ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 796                                     length);
 797                   if (is_twice_nat_session (s0))
 798                     {
 799                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 800                                              s0->ext_host_nat_addr.as_u32,
 801                                              ip4_header_t, dst_address);
 802                       sum0 =
 803                         ip_csum_update (sum0,
 804                                         vnet_buffer (b0)->ip.
 805                                         reass.l4_src_port,
 806                                         s0->ext_host_nat_port, ip4_header_t,
 807                                         length);
 808                       tcp0->src_port = s0->ext_host_nat_port;
 809                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 810                     }
 811                   tcp0->checksum = ip_csum_fold (sum0);
 812                 }
 813               tcp_packets++;
 814               if (nat44_set_tcp_session_state_o2i
 815                   (sm, now, s0,
 816                    vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
 817                    vnet_buffer (b0)->ip.reass.tcp_ack_number,
 818                    vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
 819                 goto trace0;
 820             }
 821           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
 822                    && udp0->checksum)
 823             {
 824               new_port0 = udp0->dst_port = s0->in2out.port;
 825               sum0 = udp0->checksum;
 826               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 827                                      dst_address);
 828               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 829                                      length);
 830               if (PREDICT_FALSE (is_twice_nat_session (s0)))
 831                 {
 832                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 833                                          s0->ext_host_nat_addr.as_u32,
 834                                          ip4_header_t, dst_address);
 835                   sum0 =
 836                     ip_csum_update (sum0,
 837                                     vnet_buffer (b0)->ip.reass.l4_src_port,
 838                                     s0->ext_host_nat_port, ip4_header_t,
 839                                     length);
 840                   udp0->src_port = s0->ext_host_nat_port;
 841                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 842                 }
 843               udp0->checksum = ip_csum_fold (sum0);
 844               udp_packets++;
 845             }
 846           else
 847             {
 848               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 849                 {
 850                   new_port0 = udp0->dst_port = s0->in2out.port;
 851                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
 852                     {
 853                       udp0->src_port = s0->ext_host_nat_port;
 854                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 855                     }
 856                 }
 857               udp_packets++;
 858             }
 859
 860           /* Accounting */
 861           nat44_session_update_counters (s0, now,
 862                                          vlib_buffer_length_in_chain (vm, b0),
 863                                          thread_index);
 864           /* Per-user LRU list maintenance */
 865           nat44_session_update_lru (sm, s0, thread_index);
 866
 867         trace0:
 868           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 869                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 870             {
 871               nat44_ed_out2in_trace_t *t =
 872                 vlib_add_trace (vm, node, b0, sizeof (*t));
 873               t->sw_if_index = sw_if_index0;
 874               t->next_index = next0;
 875               t->is_slow_path = 0;
 876
 877               if (s0)
 878                 t->session_index = s0 - tsm->sessions;
 879               else
 880                 t->session_index = ~0;
 881             }
 882
 883           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
 884           /* verify speculative enqueue, maybe switch current next frame */
 885           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 886                                            to_next, n_left_to_next,
 887                                            bi0, next0);
 888         }
 889
 890       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 891     }
 892
 893   vlib_node_increment_counter (vm, stats_node_index,
 894                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
 895                                pkts_processed);
 896   vlib_node_increment_counter (vm, stats_node_index,
 897                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
 898   vlib_node_increment_counter (vm, stats_node_index,
 899                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
 900   vlib_node_increment_counter (vm, stats_node_index,
 901                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
 902                                icmp_packets);
 903   vlib_node_increment_counter (vm, stats_node_index,
 904                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
 905                                other_packets);
 906   vlib_node_increment_counter (vm, stats_node_index,
 907                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
 908   return frame->n_vectors;
 909 }
 910
 911 static inline uword
 912 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
 913                                           vlib_node_runtime_t * node,
 914                                           vlib_frame_t * frame)
 915 {
 916   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 917   nat_next_t next_index;
 918   snat_main_t *sm = &snat_main;
 919   f64 now = vlib_time_now (vm);
 920   u32 thread_index = vm->thread_index;
 921   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 922   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 923     0, fragments = 0;
 924
 925   stats_node_index = sm->ed_out2in_slowpath_node_index;
 926
 927   from = vlib_frame_vector_args (frame);
 928   n_left_from = frame->n_vectors;
 929   next_index = node->cached_next_index;
 930
 931   while (n_left_from > 0)
 932     {
 933       u32 n_left_to_next;
 934
 935       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 936
 937       while (n_left_from > 0 && n_left_to_next > 0)
 938         {
 939           u32 bi0;
 940           vlib_buffer_t *b0;
 941           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 942             new_addr0;
 943           u16 old_port0, new_port0;
 944           ip4_header_t *ip0;
 945           udp_header_t *udp0;
 946           tcp_header_t *tcp0;
 947           icmp46_header_t *icmp0;
 948           snat_session_t *s0 = 0;
 949           clib_bihash_kv_16_8_t kv0, value0;
 950           ip_csum_t sum0;
 951           snat_session_key_t e_key0, l_key0;
 952           lb_nat_type_t lb_nat0;
 953           twice_nat_type_t twice_nat0;
 954           u8 identity_nat0;
 955
 956           /* speculatively enqueue b0 to the current next frame */
 957           bi0 = from[0];
 958           to_next[0] = bi0;
 959           from += 1;
 960           to_next += 1;
 961           n_left_from -= 1;
 962           n_left_to_next -= 1;
 963
 964           b0 = vlib_get_buffer (vm, bi0);
 965           next0 = vnet_buffer2 (b0)->nat.arc_next;
 966
 967           vnet_buffer (b0)->snat.flags = 0;
 968           ip0 = vlib_buffer_get_current (b0);
 969
 970           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 971           rx_fib_index0 =
 972             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 973                                                  sw_if_index0);
 974
 975           if (PREDICT_FALSE (ip0->ttl == 1))
 976             {
 977               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 978               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 979                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 980                                            0);
 981               next0 = NAT_NEXT_ICMP_ERROR;
 982               goto trace0;
 983             }
 984
 985           udp0 = ip4_next_header (ip0);
 986           tcp0 = (tcp_header_t *) udp0;
 987           icmp0 = (icmp46_header_t *) udp0;
 988           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 989
 990           if (PREDICT_FALSE (proto0 == ~0))
 991             {
 992               s0 =
 993                 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
 994                                                thread_index, now, vm, node);
 995               if (!sm->forwarding_enabled)
 996                 {
 997                   if (!s0)
 998                     next0 = NAT_NEXT_DROP;
 999                 }
1000               other_packets++;
1001               goto trace0;
1002             }
1003
1004           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1005             {
1006               next0 = icmp_out2in_ed_slow_path
1007                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1008                  next0, now, thread_index, &s0);
1009               icmp_packets++;
1010               goto trace0;
1011             }
1012
1013           make_ed_kv (&ip0->dst_address, &ip0->src_address,
1014                       ip0->protocol, rx_fib_index0,
1015                       vnet_buffer (b0)->ip.reass.l4_dst_port,
1016                       vnet_buffer (b0)->ip.reass.l4_src_port, ~0ULL, &kv0);
1017
1018           s0 = NULL;
1019           if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1020             {
1021               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1022
1023               if (s0->tcp_close_timestamp && now >= s0->tcp_close_timestamp)
1024                 {
1025                   nat_free_session_data (sm, s0, thread_index, 0);
1026                   nat44_ed_delete_session (sm, s0, thread_index, 1);
1027                   s0 = NULL;
1028                 }
1029             }
1030
1031           if (!s0)
1032             {
1033               /* Try to match static mapping by external address and port,
1034                  destination address and port in packet */
1035               e_key0.addr = ip0->dst_address;
1036               e_key0.port = vnet_buffer (b0)->ip.reass.l4_dst_port;
1037               e_key0.protocol = proto0;
1038               e_key0.fib_index = rx_fib_index0;
1039
1040               if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1041                                              &twice_nat0, &lb_nat0,
1042                                              &ip0->src_address,
1043                                              &identity_nat0))
1044                 {
1045                   /*
1046                    * Send DHCP packets to the ipv4 stack, or we won't
1047                    * be able to use dhcp client on the outside interface
1048                    */
1049                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1050                                      && (vnet_buffer (b0)->ip.
1051                                          reass.l4_dst_port ==
1052                                          clib_host_to_net_u16
1053                                          (UDP_DST_PORT_dhcp_to_client))))
1054                     {
1055                       goto trace0;
1056                     }
1057
1058                   if (!sm->forwarding_enabled)
1059                     {
1060                       b0->error =
1061                         node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1062                       next0 = NAT_NEXT_DROP;
1063                     }
1064                   else
1065                     {
1066                       if (next_src_nat
1067                           (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1068                            vnet_buffer (b0)->ip.reass.l4_dst_port,
1069                            thread_index, rx_fib_index0))
1070                         {
1071                           next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1072                           goto trace0;
1073                         }
1074                       if (sm->num_workers > 1)
1075                         create_bypass_for_fwd_worker (sm, b0, ip0,
1076                                                       rx_fib_index0);
1077                       else
1078                         create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1079                                                thread_index);
1080                     }
1081                   goto trace0;
1082                 }
1083
1084               if (PREDICT_FALSE (identity_nat0))
1085                 goto trace0;
1086
1087               if ((proto0 == SNAT_PROTOCOL_TCP)
1088                   && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1089                                          reass.icmp_type_or_tcp_flags))
1090                 {
1091                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1092                   next0 = NAT_NEXT_DROP;
1093                   goto trace0;
1094                 }
1095
1096               /* Create session initiated by host from external network */
1097               s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1098                                                          e_key0, node,
1099                                                          rx_fib_index0,
1100                                                          thread_index,
1101                                                          twice_nat0,
1102                                                          lb_nat0, now);
1103               if (!s0)
1104                 {
1105                   next0 = NAT_NEXT_DROP;
1106                   goto trace0;
1107                 }
1108             }
1109
1110           old_addr0 = ip0->dst_address.as_u32;
1111           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1112           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1113
1114           sum0 = ip0->checksum;
1115           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1116                                  dst_address);
1117           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1118             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1119                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1120                                    src_address);
1121           ip0->checksum = ip_csum_fold (sum0);
1122
1123           old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1124
1125           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1126             {
1127               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1128                 {
1129                   new_port0 = udp0->dst_port = s0->in2out.port;
1130                   sum0 = tcp0->checksum;
1131                   sum0 =
1132                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1133                                     dst_address);
1134                   sum0 =
1135                     ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1136                                     length);
1137                   if (is_twice_nat_session (s0))
1138                     {
1139                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1140                                              s0->ext_host_nat_addr.as_u32,
1141                                              ip4_header_t, dst_address);
1142                       sum0 =
1143                         ip_csum_update (sum0,
1144                                         vnet_buffer (b0)->ip.
1145                                         reass.l4_src_port,
1146                                         s0->ext_host_nat_port, ip4_header_t,
1147                                         length);
1148                       tcp0->src_port = s0->ext_host_nat_port;
1149                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1150                     }
1151                   tcp0->checksum = ip_csum_fold (sum0);
1152                 }
1153               tcp_packets++;
1154               if (nat44_set_tcp_session_state_o2i
1155                   (sm, now, s0,
1156                    vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
1157                    vnet_buffer (b0)->ip.reass.tcp_ack_number,
1158                    vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
1159                 goto trace0;
1160             }
1161           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1162                    && udp0->checksum)
1163             {
1164               new_port0 = udp0->dst_port = s0->in2out.port;
1165               sum0 = udp0->checksum;
1166               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1167                                      dst_address);
1168               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1169                                      length);
1170               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1171                 {
1172                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1173                                          s0->ext_host_nat_addr.as_u32,
1174                                          ip4_header_t, dst_address);
1175                   sum0 =
1176                     ip_csum_update (sum0,
1177                                     vnet_buffer (b0)->ip.reass.l4_src_port,
1178                                     s0->ext_host_nat_port, ip4_header_t,
1179                                     length);
1180                   udp0->src_port = s0->ext_host_nat_port;
1181                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1182                 }
1183               udp0->checksum = ip_csum_fold (sum0);
1184               udp_packets++;
1185             }
1186           else
1187             {
1188               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1189                 {
1190                   new_port0 = udp0->dst_port = s0->in2out.port;
1191                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1192                     {
1193                       udp0->src_port = s0->ext_host_nat_port;
1194                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1195                     }
1196                 }
1197               udp_packets++;
1198             }
1199
1200           /* Accounting */
1201           nat44_session_update_counters (s0, now,
1202                                          vlib_buffer_length_in_chain (vm, b0),
1203                                          thread_index);
1204           /* Per-user LRU list maintenance */
1205           nat44_session_update_lru (sm, s0, thread_index);
1206
1207         trace0:
1208           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1209                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1210             {
1211               nat44_ed_out2in_trace_t *t =
1212                 vlib_add_trace (vm, node, b0, sizeof (*t));
1213               t->sw_if_index = sw_if_index0;
1214               t->next_index = next0;
1215               t->is_slow_path = 1;
1216
1217               if (s0)
1218                 t->session_index = s0 - tsm->sessions;
1219               else
1220                 t->session_index = ~0;
1221             }
1222
1223           pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1224           /* verify speculative enqueue, maybe switch current next frame */
1225           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1226                                            to_next, n_left_to_next,
1227                                            bi0, next0);
1228         }
1229
1230       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1231     }
1232
1233   vlib_node_increment_counter (vm, stats_node_index,
1234                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1235                                pkts_processed);
1236   vlib_node_increment_counter (vm, stats_node_index,
1237                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1238   vlib_node_increment_counter (vm, stats_node_index,
1239                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1240   vlib_node_increment_counter (vm, stats_node_index,
1241                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1242                                icmp_packets);
1243   vlib_node_increment_counter (vm, stats_node_index,
1244                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1245                                other_packets);
1246   vlib_node_increment_counter (vm, stats_node_index,
1247                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1248   return frame->n_vectors;
1249 }
1250
1251 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1252                                      vlib_node_runtime_t * node,
1253                                      vlib_frame_t * frame)
1254 {
1255   return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame);
1256 }
1257
1258 /* *INDENT-OFF* */
1259 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1260   .name = "nat44-ed-out2in",
1261   .vector_size = sizeof (u32),
1262   .sibling_of = "nat-default",
1263   .format_trace = format_nat44_ed_out2in_trace,
1264   .type = VLIB_NODE_TYPE_INTERNAL,
1265   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1266   .error_strings = nat_out2in_ed_error_strings,
1267   .runtime_data_bytes = sizeof (snat_runtime_t),
1268 };
1269 /* *INDENT-ON* */
1270
1271 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1272                                               vlib_node_runtime_t * node,
1273                                               vlib_frame_t * frame)
1274 {
1275   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1276 }
1277
1278 /* *INDENT-OFF* */
1279 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1280   .name = "nat44-ed-out2in-slowpath",
1281   .vector_size = sizeof (u32),
1282   .sibling_of = "nat-default",
1283   .format_trace = format_nat44_ed_out2in_trace,
1284   .type = VLIB_NODE_TYPE_INTERNAL,
1285   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1286   .error_strings = nat_out2in_ed_error_strings,
1287   .runtime_data_bytes = sizeof (snat_runtime_t),
1288 };
1289 /* *INDENT-ON* */
1290
1291 static u8 *
1292 format_nat_pre_trace (u8 * s, va_list * args)
1293 {
1294   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1295   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1296   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1297   return format (s, "out2in next_index %d", t->next_index);
1298 }
1299
1300 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1301                                     vlib_node_runtime_t * node,
1302                                     vlib_frame_t * frame)
1303 {
1304   return nat_pre_node_fn_inline (vm, node, frame,
1305                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1306 }
1307
1308 /* *INDENT-OFF* */
1309 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1310   .name = "nat-pre-out2in",
1311   .vector_size = sizeof (u32),
1312   .sibling_of = "nat-default",
1313   .format_trace = format_nat_pre_trace,
1314   .type = VLIB_NODE_TYPE_INTERNAL,
1315   .n_errors = 0,
1316  };
1317 /* *INDENT-ON* */
1318
1319 /*
1320  * fd.io coding-style-patch-verification: ON
1321  *
1322  * Local Variables:
1323  * eval: (c-set-style "gnu")
1324  * End:
1325  */