src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_reass.h>
  31 #include <nat/nat_inlines.h>
  32 #include <nat/nat_syslog.h>
  33
  34 #define foreach_nat_out2in_ed_error                     \
  35 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  36 _(OUT2IN_PACKETS, "good out2in packets processed")      \
  37 _(OUT_OF_PORTS, "out of ports")                         \
  38 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  39 _(NO_TRANSLATION, "no translation")                     \
  40 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  41 _(DROP_FRAGMENT, "drop fragment")                       \
  42 _(MAX_REASS, "maximum reassemblies exceeded")           \
  43 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  44 _(NON_SYN, "non-SYN packet try to create session")      \
  45 _(TCP_PACKETS, "TCP packets")                           \
  46 _(UDP_PACKETS, "UDP packets")                           \
  47 _(ICMP_PACKETS, "ICMP packets")                         \
  48 _(OTHER_PACKETS, "other protocol packets")              \
  49 _(FRAGMENTS, "fragments")                               \
  50 _(CACHED_FRAGMENTS, "cached fragments")                 \
  51 _(PROCESSED_FRAGMENTS, "processed fragments")
  52
  53 typedef enum
  54 {
  55 #define _(sym,str) NAT_OUT2IN_ED_ERROR_##sym,
  56   foreach_nat_out2in_ed_error
  57 #undef _
  58     NAT_OUT2IN_ED_N_ERROR,
  59 } nat_out2in_ed_error_t;
  60
  61 static char *nat_out2in_ed_error_strings[] = {
  62 #define _(sym,string) string,
  63   foreach_nat_out2in_ed_error
  64 #undef _
  65 };
  66
  67 typedef enum
  68 {
  69   NAT44_ED_OUT2IN_NEXT_DROP,
  70   NAT44_ED_OUT2IN_NEXT_LOOKUP,
  71   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
  72   NAT44_ED_OUT2IN_NEXT_IN2OUT,
  73   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
  74   NAT44_ED_OUT2IN_NEXT_REASS,
  75   NAT44_ED_OUT2IN_N_NEXT,
  76 } nat44_ed_out2in_next_t;
  77
  78 typedef struct
  79 {
  80   u32 sw_if_index;
  81   u32 next_index;
  82   u32 session_index;
  83   u32 is_slow_path;
  84 } nat44_ed_out2in_trace_t;
  85
  86 vlib_node_registration_t nat44_ed_out2in_node;
  87 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
  88 vlib_node_registration_t nat44_ed_out2in_reass_node;
  89
  90 static u8 *
  91 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  92 {
  93   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  94   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  95   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  96   char *tag;
  97
  98   tag =
  99     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
 100     "NAT44_OUT2IN_ED_FAST_PATH";
 101
 102   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
 103               t->sw_if_index, t->next_index, t->session_index);
 104
 105   return s;
 106 }
 107
 108 static inline u32
 109 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 110                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 111                           u32 sw_if_index0, u32 rx_fib_index0,
 112                           vlib_node_runtime_t * node, u32 next0, f64 now,
 113                           u32 thread_index, snat_session_t ** p_s0)
 114 {
 115   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 116                        next0, thread_index, p_s0, 0);
 117   snat_session_t *s0 = *p_s0;
 118   if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0))
 119     {
 120       /* Accounting */
 121       nat44_session_update_counters (s0, now,
 122                                      vlib_buffer_length_in_chain
 123                                      (sm->vlib_main, b0));
 124       /* Per-user LRU list maintenance */
 125       nat44_session_update_lru (sm, s0, thread_index);
 126     }
 127   return next0;
 128 }
 129
 130 int
 131 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 132 {
 133   snat_main_t *sm = &snat_main;
 134   nat44_is_idle_session_ctx_t *ctx = arg;
 135   snat_session_t *s;
 136   u64 sess_timeout_time;
 137   nat_ed_ses_key_t ed_key;
 138   clib_bihash_kv_16_8_t ed_kv;
 139   int i;
 140   snat_address_t *a;
 141   snat_session_key_t key;
 142   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 143                                                        ctx->thread_index);
 144
 145   s = pool_elt_at_index (tsm->sessions, kv->value);
 146   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 147   if (ctx->now >= sess_timeout_time)
 148     {
 149       ed_key.l_addr = s->in2out.addr;
 150       ed_key.r_addr = s->ext_host_addr;
 151       ed_key.fib_index = s->in2out.fib_index;
 152       if (snat_is_unk_proto_session (s))
 153         {
 154           ed_key.proto = s->in2out.port;
 155           ed_key.r_port = 0;
 156           ed_key.l_port = 0;
 157         }
 158       else
 159         {
 160           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 161           ed_key.l_port = s->in2out.port;
 162           ed_key.r_port = s->ext_host_port;
 163         }
 164       if (is_twice_nat_session (s))
 165         {
 166           ed_key.r_addr = s->ext_host_nat_addr;
 167           ed_key.r_port = s->ext_host_nat_port;
 168         }
 169       ed_kv.key[0] = ed_key.as_u64[0];
 170       ed_kv.key[1] = ed_key.as_u64[1];
 171       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 172         nat_log_warn ("in2out_ed key del failed");
 173
 174       if (snat_is_unk_proto_session (s))
 175         goto delete;
 176
 177       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 178                                            s->in2out.addr.as_u32,
 179                                            s->out2in.addr.as_u32,
 180                                            s->in2out.protocol,
 181                                            s->in2out.port,
 182                                            s->out2in.port,
 183                                            s->in2out.fib_index);
 184
 185       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 186                              &s->in2out.addr, s->in2out.port,
 187                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 188                              &s->out2in.addr, s->out2in.port,
 189                              &s->ext_host_addr, s->ext_host_port,
 190                              s->in2out.protocol, is_twice_nat_session (s));
 191
 192       if (is_twice_nat_session (s))
 193         {
 194           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 195             {
 196               key.protocol = s->in2out.protocol;
 197               key.port = s->ext_host_nat_port;
 198               a = sm->twice_nat_addresses + i;
 199               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 200                 {
 201                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 202                                                       ctx->thread_index,
 203                                                       &key);
 204                   break;
 205                 }
 206             }
 207         }
 208
 209       if (snat_is_session_static (s))
 210         goto delete;
 211
 212       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 213                                           &s->out2in);
 214     delete:
 215       nat44_delete_session (sm, s, ctx->thread_index);
 216       return 1;
 217     }
 218
 219   return 0;
 220 }
 221
 222 static snat_session_t *
 223 create_session_for_static_mapping_ed (snat_main_t * sm,
 224                                       vlib_buffer_t * b,
 225                                       snat_session_key_t l_key,
 226                                       snat_session_key_t e_key,
 227                                       vlib_node_runtime_t * node,
 228                                       u32 thread_index,
 229                                       twice_nat_type_t twice_nat,
 230                                       lb_nat_type_t lb_nat, f64 now)
 231 {
 232   snat_session_t *s;
 233   snat_user_t *u;
 234   ip4_header_t *ip;
 235   udp_header_t *udp;
 236   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 237   clib_bihash_kv_16_8_t kv;
 238   snat_session_key_t eh_key;
 239   nat44_is_idle_session_ctx_t ctx;
 240
 241   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 242     {
 243       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 244       nat_log_notice ("maximum sessions exceeded");
 245       return 0;
 246     }
 247
 248   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
 249   if (!u)
 250     {
 251       nat_log_warn ("create NAT user failed");
 252       return 0;
 253     }
 254
 255   s = nat_ed_session_alloc (sm, u, thread_index, now);
 256   if (!s)
 257     {
 258       nat44_delete_user_with_no_session (sm, u, thread_index);
 259       nat_log_warn ("create NAT session failed");
 260       return 0;
 261     }
 262
 263   ip = vlib_buffer_get_current (b);
 264   udp = ip4_next_header (ip);
 265
 266   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 267   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 268   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 269   if (lb_nat)
 270     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 271   if (lb_nat == AFFINITY_LB_NAT)
 272     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 273   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 274   s->out2in = e_key;
 275   s->in2out = l_key;
 276   s->in2out.protocol = s->out2in.protocol;
 277   user_session_increment (sm, u, 1);
 278
 279   /* Add to lookup tables */
 280   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
 281               e_key.fib_index, e_key.port, s->ext_host_port);
 282   kv.value = s - tsm->sessions;
 283   ctx.now = now;
 284   ctx.thread_index = thread_index;
 285   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
 286                                                nat44_o2i_ed_is_idle_session_cb,
 287                                                &ctx))
 288     nat_log_notice ("out2in-ed key add failed");
 289
 290   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 291                                  ip->src_address.as_u32 == l_key.addr.as_u32))
 292     {
 293       eh_key.protocol = e_key.protocol;
 294       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 295                                                thread_index, &eh_key,
 296                                                sm->port_per_thread,
 297                                                tsm->snat_thread_index))
 298         {
 299           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 300           nat44_delete_session (sm, s, thread_index);
 301           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
 302             nat_log_notice ("out2in-ed key del failed");
 303           return 0;
 304         }
 305       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 306       s->ext_host_nat_port = eh_key.port;
 307       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 308       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
 309                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
 310     }
 311   else
 312     {
 313       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
 314                   l_key.fib_index, l_key.port, s->ext_host_port);
 315     }
 316   kv.value = s - tsm->sessions;
 317   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 318                                                nat44_i2o_ed_is_idle_session_cb,
 319                                                &ctx))
 320     nat_log_notice ("in2out-ed key add failed");
 321
 322   snat_ipfix_logging_nat44_ses_create (thread_index,
 323                                        s->in2out.addr.as_u32,
 324                                        s->out2in.addr.as_u32,
 325                                        s->in2out.protocol,
 326                                        s->in2out.port,
 327                                        s->out2in.port, s->in2out.fib_index);
 328
 329   nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 330                          &s->in2out.addr, s->in2out.port,
 331                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 332                          &s->out2in.addr, s->out2in.port,
 333                          &s->ext_host_addr, s->ext_host_port,
 334                          s->in2out.protocol, is_twice_nat_session (s));
 335
 336   return s;
 337 }
 338
 339 static_always_inline int
 340 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 341 {
 342   icmp46_header_t *icmp0;
 343   nat_ed_ses_key_t key0;
 344   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 345   ip4_header_t *inner_ip0;
 346   void *l4_header = 0;
 347   icmp46_header_t *inner_icmp0;
 348
 349   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 350   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 351
 352   if (!icmp_is_error_message (icmp0))
 353     {
 354       key0.proto = IP_PROTOCOL_ICMP;
 355       key0.l_addr = ip0->dst_address;
 356       key0.r_addr = ip0->src_address;
 357       key0.l_port = echo0->identifier;
 358       key0.r_port = 0;
 359     }
 360   else
 361     {
 362       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 363       l4_header = ip4_next_header (inner_ip0);
 364       key0.proto = inner_ip0->protocol;
 365       key0.l_addr = inner_ip0->src_address;
 366       key0.r_addr = inner_ip0->dst_address;
 367       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 368         {
 369         case SNAT_PROTOCOL_ICMP:
 370           inner_icmp0 = (icmp46_header_t *) l4_header;
 371           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 372           key0.l_port = inner_echo0->identifier;
 373           key0.r_port = 0;
 374           break;
 375         case SNAT_PROTOCOL_UDP:
 376         case SNAT_PROTOCOL_TCP:
 377           key0.l_port = ((tcp_udp_header_t *) l4_header)->src_port;
 378           key0.r_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 379           break;
 380         default:
 381           return -1;
 382         }
 383     }
 384   *p_key0 = key0;
 385   return 0;
 386 }
 387
 388 static int
 389 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
 390               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 391 {
 392   clib_bihash_kv_16_8_t kv, value;
 393   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 394
 395   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
 396               rx_fib_index, src_port, dst_port);
 397   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 398     return 1;
 399
 400   return 0;
 401 }
 402
 403 static void
 404 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 405                        u32 thread_index)
 406 {
 407   nat_ed_ses_key_t key;
 408   clib_bihash_kv_16_8_t kv, value;
 409   udp_header_t *udp;
 410   snat_user_t *u;
 411   snat_session_t *s = 0;
 412   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 413   f64 now = vlib_time_now (sm->vlib_main);
 414
 415   if (ip->protocol == IP_PROTOCOL_ICMP)
 416     {
 417       if (icmp_get_ed_key (ip, &key))
 418         return;
 419     }
 420   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 421     {
 422       udp = ip4_next_header (ip);
 423       key.r_addr = ip->src_address;
 424       key.l_addr = ip->dst_address;
 425       key.proto = ip->protocol;
 426       key.l_port = udp->dst_port;
 427       key.r_port = udp->src_port;
 428     }
 429   else
 430     {
 431       key.r_addr = ip->src_address;
 432       key.l_addr = ip->dst_address;
 433       key.proto = ip->protocol;
 434       key.l_port = key.r_port = 0;
 435     }
 436   key.fib_index = 0;
 437   kv.key[0] = key.as_u64[0];
 438   kv.key[1] = key.as_u64[1];
 439
 440   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 441     {
 442       s = pool_elt_at_index (tsm->sessions, value.value);
 443     }
 444   else
 445     {
 446       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 447         return;
 448
 449       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
 450                                   thread_index);
 451       if (!u)
 452         {
 453           nat_log_warn ("create NAT user failed");
 454           return;
 455         }
 456
 457       s = nat_ed_session_alloc (sm, u, thread_index, now);
 458       if (!s)
 459         {
 460           nat44_delete_user_with_no_session (sm, u, thread_index);
 461           nat_log_warn ("create NAT session failed");
 462           return;
 463         }
 464
 465       s->ext_host_addr = key.r_addr;
 466       s->ext_host_port = key.r_port;
 467       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 468       s->out2in.addr = key.l_addr;
 469       s->out2in.port = key.l_port;
 470       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 471       s->out2in.fib_index = 0;
 472       s->in2out = s->out2in;
 473       user_session_increment (sm, u, 0);
 474
 475       kv.value = s - tsm->sessions;
 476       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 477         nat_log_notice ("in2out_ed key add failed");
 478     }
 479
 480   if (ip->protocol == IP_PROTOCOL_TCP)
 481     {
 482       tcp_header_t *tcp = ip4_next_header (ip);
 483       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
 484         return;
 485     }
 486
 487   /* Accounting */
 488   nat44_session_update_counters (s, now, 0);
 489   /* Per-user LRU list maintenance */
 490   nat44_session_update_lru (sm, s, thread_index);
 491 }
 492
 493 u32
 494 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 495                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 496                       u8 * p_proto, snat_session_key_t * p_value,
 497                       u8 * p_dont_translate, void *d, void *e)
 498 {
 499   u32 next = ~0, sw_if_index, rx_fib_index;
 500   icmp46_header_t *icmp;
 501   nat_ed_ses_key_t key;
 502   clib_bihash_kv_16_8_t kv, value;
 503   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 504   snat_session_t *s = 0;
 505   u8 dont_translate = 0, is_addr_only, identity_nat;
 506   snat_session_key_t e_key, l_key;
 507
 508   icmp = (icmp46_header_t *) ip4_next_header (ip);
 509   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 510   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 511
 512   if (icmp_get_ed_key (ip, &key))
 513     {
 514       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 515       next = NAT44_ED_OUT2IN_NEXT_DROP;
 516       goto out;
 517     }
 518   key.fib_index = rx_fib_index;
 519   kv.key[0] = key.as_u64[0];
 520   kv.key[1] = key.as_u64[1];
 521
 522   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 523     {
 524       /* Try to match static mapping */
 525       e_key.addr = ip->dst_address;
 526       e_key.port = key.l_port;
 527       e_key.protocol = ip_proto_to_snat_proto (key.proto);
 528       e_key.fib_index = rx_fib_index;
 529       if (snat_static_mapping_match
 530           (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
 531         {
 532           if (!sm->forwarding_enabled)
 533             {
 534               /* Don't NAT packet aimed at the intfc address */
 535               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 536                                                     ip->dst_address.as_u32)))
 537                 {
 538                   dont_translate = 1;
 539                   goto out;
 540                 }
 541               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 542               next = NAT44_ED_OUT2IN_NEXT_DROP;
 543               goto out;
 544             }
 545           else
 546             {
 547               dont_translate = 1;
 548               if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
 549                                 thread_index, rx_fib_index))
 550                 {
 551                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
 552                   goto out;
 553                 }
 554               create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
 555               goto out;
 556             }
 557         }
 558
 559       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 560                          (icmp->type != ICMP4_echo_request || !is_addr_only)))
 561         {
 562           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 563           next = NAT44_ED_OUT2IN_NEXT_DROP;
 564           goto out;
 565         }
 566
 567       if (PREDICT_FALSE (identity_nat))
 568         {
 569           dont_translate = 1;
 570           goto out;
 571         }
 572
 573       /* Create session initiated by host from external network */
 574       s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
 575                                                 thread_index, 0, 0,
 576                                                 vlib_time_now
 577                                                 (sm->vlib_main));
 578
 579       if (!s)
 580         {
 581           next = NAT44_ED_OUT2IN_NEXT_DROP;
 582           goto out;
 583         }
 584     }
 585   else
 586     {
 587       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 588                          icmp->type != ICMP4_echo_request &&
 589                          !icmp_is_error_message (icmp)))
 590         {
 591           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 592           next = NAT44_ED_OUT2IN_NEXT_DROP;
 593           goto out;
 594         }
 595
 596       s = pool_elt_at_index (tsm->sessions, value.value);
 597     }
 598
 599   *p_proto = ip_proto_to_snat_proto (key.proto);
 600 out:
 601   if (s)
 602     *p_value = s->in2out;
 603   *p_dont_translate = dont_translate;
 604   if (d)
 605     *(snat_session_t **) d = s;
 606   return next;
 607 }
 608
 609 static snat_session_t *
 610 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 611                                vlib_buffer_t * b,
 612                                ip4_header_t * ip,
 613                                u32 rx_fib_index,
 614                                u32 thread_index,
 615                                f64 now,
 616                                vlib_main_t * vm, vlib_node_runtime_t * node)
 617 {
 618   clib_bihash_kv_8_8_t kv, value;
 619   clib_bihash_kv_16_8_t s_kv, s_value;
 620   snat_static_mapping_t *m;
 621   u32 old_addr, new_addr;
 622   ip_csum_t sum;
 623   snat_session_t *s;
 624   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 625   snat_user_t *u;
 626
 627   old_addr = ip->dst_address.as_u32;
 628
 629   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 630               rx_fib_index, 0, 0);
 631
 632   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 633     {
 634       s = pool_elt_at_index (tsm->sessions, s_value.value);
 635       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 636     }
 637   else
 638     {
 639       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 640         {
 641           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 642           nat_log_notice ("maximum sessions exceeded");
 643           return 0;
 644         }
 645
 646       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 647       if (clib_bihash_search_8_8
 648           (&sm->static_mapping_by_external, &kv, &value))
 649         {
 650           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 651           return 0;
 652         }
 653
 654       m = pool_elt_at_index (sm->static_mappings, value.value);
 655
 656       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 657
 658       u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
 659                                   thread_index);
 660       if (!u)
 661         {
 662           nat_log_warn ("create NAT user failed");
 663           return 0;
 664         }
 665
 666       /* Create a new session */
 667       s = nat_ed_session_alloc (sm, u, thread_index, now);
 668       if (!s)
 669         {
 670           nat44_delete_user_with_no_session (sm, u, thread_index);
 671           nat_log_warn ("create NAT session failed");
 672           return 0;
 673         }
 674
 675       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 676       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 677       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 678       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 679       s->out2in.addr.as_u32 = old_addr;
 680       s->out2in.fib_index = rx_fib_index;
 681       s->in2out.addr.as_u32 = new_addr;
 682       s->in2out.fib_index = m->fib_index;
 683       s->in2out.port = s->out2in.port = ip->protocol;
 684       user_session_increment (sm, u, 1);
 685
 686       /* Add to lookup tables */
 687       s_kv.value = s - tsm->sessions;
 688       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 689         nat_log_notice ("out2in key add failed");
 690
 691       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 692                   m->fib_index, 0, 0);
 693       s_kv.value = s - tsm->sessions;
 694       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 695         nat_log_notice ("in2out key add failed");
 696     }
 697
 698   /* Update IP checksum */
 699   sum = ip->checksum;
 700   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 701   ip->checksum = ip_csum_fold (sum);
 702
 703   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 704
 705   /* Accounting */
 706   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 707   /* Per-user LRU list maintenance */
 708   nat44_session_update_lru (sm, s, thread_index);
 709
 710   return s;
 711 }
 712
 713 static inline uword
 714 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
 715                                 vlib_node_runtime_t * node,
 716                                 vlib_frame_t * frame, int is_slow_path)
 717 {
 718   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 719   nat44_ed_out2in_next_t next_index;
 720   snat_main_t *sm = &snat_main;
 721   f64 now = vlib_time_now (vm);
 722   u32 thread_index = vm->thread_index;
 723   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 724   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 725     0, fragments = 0;
 726
 727   stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
 728     nat44_ed_out2in_node.index;
 729
 730   from = vlib_frame_vector_args (frame);
 731   n_left_from = frame->n_vectors;
 732   next_index = node->cached_next_index;
 733
 734   while (n_left_from > 0)
 735     {
 736       u32 n_left_to_next;
 737
 738       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 739
 740       while (n_left_from >= 4 && n_left_to_next >= 2)
 741         {
 742           u32 bi0, bi1;
 743           vlib_buffer_t *b0, *b1;
 744           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 745             new_addr0;
 746           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
 747             new_addr1;
 748           u16 old_port0, new_port0, old_port1, new_port1;
 749           ip4_header_t *ip0, *ip1;
 750           udp_header_t *udp0, *udp1;
 751           tcp_header_t *tcp0, *tcp1;
 752           icmp46_header_t *icmp0, *icmp1;
 753           snat_session_t *s0 = 0, *s1 = 0;
 754           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 755           ip_csum_t sum0, sum1;
 756           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
 757           lb_nat_type_t lb_nat0, lb_nat1;
 758           twice_nat_type_t twice_nat0, twice_nat1;
 759           u8 identity_nat0, identity_nat1;
 760
 761           /* Prefetch next iteration. */
 762           {
 763             vlib_buffer_t *p2, *p3;
 764
 765             p2 = vlib_get_buffer (vm, from[2]);
 766             p3 = vlib_get_buffer (vm, from[3]);
 767
 768             vlib_prefetch_buffer_header (p2, LOAD);
 769             vlib_prefetch_buffer_header (p3, LOAD);
 770
 771             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 772             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 773           }
 774
 775           /* speculatively enqueue b0 and b1 to the current next frame */
 776           to_next[0] = bi0 = from[0];
 777           to_next[1] = bi1 = from[1];
 778           from += 2;
 779           to_next += 2;
 780           n_left_from -= 2;
 781           n_left_to_next -= 2;
 782
 783           b0 = vlib_get_buffer (vm, bi0);
 784           b1 = vlib_get_buffer (vm, bi1);
 785
 786           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
 787           vnet_buffer (b0)->snat.flags = 0;
 788           ip0 = vlib_buffer_get_current (b0);
 789
 790           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 791           rx_fib_index0 =
 792             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 793                                                  sw_if_index0);
 794
 795           if (PREDICT_FALSE (ip0->ttl == 1))
 796             {
 797               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 798               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 799                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 800                                            0);
 801               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
 802               goto trace00;
 803             }
 804
 805           udp0 = ip4_next_header (ip0);
 806           tcp0 = (tcp_header_t *) udp0;
 807           icmp0 = (icmp46_header_t *) udp0;
 808           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 809
 810           if (is_slow_path)
 811             {
 812               if (PREDICT_FALSE (proto0 == ~0))
 813                 {
 814                   s0 =
 815                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
 816                                                    thread_index, now, vm,
 817                                                    node);
 818                   other_packets++;
 819                   if (!sm->forwarding_enabled)
 820                     {
 821                       if (!s0)
 822                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 823                       goto trace00;
 824                     }
 825                 }
 826
 827               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 828                 {
 829                   next0 = icmp_out2in_ed_slow_path
 830                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 831                      next0, now, thread_index, &s0);
 832                   icmp_packets++;
 833                   goto trace00;
 834                 }
 835             }
 836           else
 837             {
 838               if (PREDICT_FALSE (proto0 == ~0))
 839                 {
 840                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 841                   goto trace00;
 842                 }
 843
 844               if (ip4_is_fragment (ip0))
 845                 {
 846                   next0 = NAT44_ED_OUT2IN_NEXT_REASS;
 847                   fragments++;
 848                   goto trace00;
 849                 }
 850
 851               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 852                 {
 853                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 854                   goto trace00;
 855                 }
 856             }
 857
 858           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
 859                       ip0->protocol, rx_fib_index0, udp0->dst_port,
 860                       udp0->src_port);
 861
 862           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
 863             {
 864               if (is_slow_path)
 865                 {
 866                   /* Try to match static mapping by external address and port,
 867                      destination address and port in packet */
 868                   e_key0.addr = ip0->dst_address;
 869                   e_key0.port = udp0->dst_port;
 870                   e_key0.protocol = proto0;
 871                   e_key0.fib_index = rx_fib_index0;
 872                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
 873                                                  &twice_nat0, &lb_nat0,
 874                                                  &ip0->src_address,
 875                                                  &identity_nat0))
 876                     {
 877                       /*
 878                        * Send DHCP packets to the ipv4 stack, or we won't
 879                        * be able to use dhcp client on the outside interface
 880                        */
 881                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 882                                          && (udp0->dst_port ==
 883                                              clib_host_to_net_u16
 884                                              (UDP_DST_PORT_dhcp_to_client))))
 885                         {
 886                           vnet_feature_next (&next0, b0);
 887                           goto trace00;
 888                         }
 889
 890                       if (!sm->forwarding_enabled)
 891                         {
 892                           b0->error =
 893                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 894                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 895                         }
 896                       else
 897                         {
 898                           if (next_src_nat (sm, ip0, ip0->protocol,
 899                                             udp0->src_port, udp0->dst_port,
 900                                             thread_index, rx_fib_index0))
 901                             {
 902                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
 903                               goto trace00;
 904                             }
 905                           create_bypass_for_fwd (sm, ip0, rx_fib_index0,
 906                                                  thread_index);
 907                         }
 908                       goto trace00;
 909                     }
 910
 911                   if (PREDICT_FALSE (identity_nat0))
 912                     goto trace00;
 913
 914                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
 915                     {
 916                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
 917                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 918                       goto trace00;
 919                     }
 920
 921                   /* Create session initiated by host from external network */
 922                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
 923                                                              e_key0, node,
 924                                                              thread_index,
 925                                                              twice_nat0,
 926                                                              lb_nat0, now);
 927
 928                   if (!s0)
 929                     {
 930                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 931                       goto trace00;
 932                     }
 933                 }
 934               else
 935                 {
 936                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 937                   goto trace00;
 938                 }
 939             }
 940           else
 941             {
 942               s0 = pool_elt_at_index (tsm->sessions, value0.value);
 943             }
 944
 945           old_addr0 = ip0->dst_address.as_u32;
 946           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 947           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 948
 949           sum0 = ip0->checksum;
 950           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 951                                  dst_address);
 952           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 953             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 954                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
 955                                    src_address);
 956           ip0->checksum = ip_csum_fold (sum0);
 957
 958           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 959             {
 960               old_port0 = tcp0->dst_port;
 961               new_port0 = tcp0->dst_port = s0->in2out.port;
 962
 963               sum0 = tcp0->checksum;
 964               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 965                                      dst_address);
 966               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 967                                      length);
 968               if (is_twice_nat_session (s0))
 969                 {
 970                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 971                                          s0->ext_host_nat_addr.as_u32,
 972                                          ip4_header_t, dst_address);
 973                   sum0 = ip_csum_update (sum0, tcp0->src_port,
 974                                          s0->ext_host_nat_port, ip4_header_t,
 975                                          length);
 976                   tcp0->src_port = s0->ext_host_nat_port;
 977                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 978                 }
 979               tcp0->checksum = ip_csum_fold (sum0);
 980               tcp_packets++;
 981               if (nat44_set_tcp_session_state_o2i
 982                   (sm, s0, tcp0, thread_index))
 983                 goto trace00;
 984             }
 985           else
 986             {
 987               udp0->dst_port = s0->in2out.port;
 988               if (is_twice_nat_session (s0))
 989                 {
 990                   udp0->src_port = s0->ext_host_nat_port;
 991                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 992                 }
 993               udp0->checksum = 0;
 994               udp_packets++;
 995             }
 996
 997           /* Accounting */
 998           nat44_session_update_counters (s0, now,
 999                                          vlib_buffer_length_in_chain (vm,
1000                                                                       b0));
1001           /* Per-user LRU list maintenance */
1002           nat44_session_update_lru (sm, s0, thread_index);
1003
1004         trace00:
1005           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1006                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1007             {
1008               nat44_ed_out2in_trace_t *t =
1009                 vlib_add_trace (vm, node, b0, sizeof (*t));
1010               t->is_slow_path = is_slow_path;
1011               t->sw_if_index = sw_if_index0;
1012               t->next_index = next0;
1013               t->session_index = ~0;
1014               if (s0)
1015                 t->session_index = s0 - tsm->sessions;
1016             }
1017
1018           pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1019
1020           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1021           vnet_buffer (b1)->snat.flags = 0;
1022           ip1 = vlib_buffer_get_current (b1);
1023
1024           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1025           rx_fib_index1 =
1026             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1027                                                  sw_if_index1);
1028
1029           if (PREDICT_FALSE (ip1->ttl == 1))
1030             {
1031               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1032               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1033                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1034                                            0);
1035               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1036               goto trace01;
1037             }
1038
1039           udp1 = ip4_next_header (ip1);
1040           tcp1 = (tcp_header_t *) udp1;
1041           icmp1 = (icmp46_header_t *) udp1;
1042           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1043
1044           if (is_slow_path)
1045             {
1046               if (PREDICT_FALSE (proto1 == ~0))
1047                 {
1048                   s1 =
1049                     nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1050                                                    thread_index, now, vm,
1051                                                    node);
1052                   other_packets++;
1053                   if (!sm->forwarding_enabled)
1054                     {
1055                       if (!s1)
1056                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1057                       goto trace01;
1058                     }
1059                 }
1060
1061               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1062                 {
1063                   next1 = icmp_out2in_ed_slow_path
1064                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1065                      next1, now, thread_index, &s1);
1066                   icmp_packets++;
1067                   goto trace01;
1068                 }
1069             }
1070           else
1071             {
1072               if (PREDICT_FALSE (proto1 == ~0))
1073                 {
1074                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1075                   goto trace01;
1076                 }
1077
1078               if (ip4_is_fragment (ip1))
1079                 {
1080                   next1 = NAT44_ED_OUT2IN_NEXT_REASS;
1081                   fragments++;
1082                   goto trace01;
1083                 }
1084
1085               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1086                 {
1087                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1088                   goto trace01;
1089                 }
1090             }
1091
1092           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1093                       ip1->protocol, rx_fib_index1, udp1->dst_port,
1094                       udp1->src_port);
1095
1096           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1097             {
1098               if (is_slow_path)
1099                 {
1100                   /* Try to match static mapping by external address and port,
1101                      destination address and port in packet */
1102                   e_key1.addr = ip1->dst_address;
1103                   e_key1.port = udp1->dst_port;
1104                   e_key1.protocol = proto1;
1105                   e_key1.fib_index = rx_fib_index1;
1106                   if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1107                                                  &twice_nat1, &lb_nat1,
1108                                                  &ip1->src_address,
1109                                                  &identity_nat1))
1110                     {
1111                       /*
1112                        * Send DHCP packets to the ipv4 stack, or we won't
1113                        * be able to use dhcp client on the outside interface
1114                        */
1115                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1116                                          && (udp1->dst_port ==
1117                                              clib_host_to_net_u16
1118                                              (UDP_DST_PORT_dhcp_to_client))))
1119                         {
1120                           vnet_feature_next (&next1, b1);
1121                           goto trace01;
1122                         }
1123
1124                       if (!sm->forwarding_enabled)
1125                         {
1126                           b1->error =
1127                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1128                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1129                         }
1130                       else
1131                         {
1132                           if (next_src_nat (sm, ip1, ip1->protocol,
1133                                             udp1->src_port, udp1->dst_port,
1134                                             thread_index, rx_fib_index1))
1135                             {
1136                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1137                               goto trace01;
1138                             }
1139                           create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1140                                                  thread_index);
1141                         }
1142                       goto trace01;
1143                     }
1144
1145                   if (PREDICT_FALSE (identity_nat1))
1146                     goto trace01;
1147
1148                   if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1149                     {
1150                       b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1151                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1152                       goto trace01;
1153                     }
1154
1155                   /* Create session initiated by host from external network */
1156                   s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1157                                                              e_key1, node,
1158                                                              thread_index,
1159                                                              twice_nat1,
1160                                                              lb_nat1, now);
1161
1162                   if (!s1)
1163                     {
1164                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1165                       goto trace01;
1166                     }
1167                 }
1168               else
1169                 {
1170                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1171                   goto trace01;
1172                 }
1173             }
1174           else
1175             {
1176               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1177             }
1178
1179           old_addr1 = ip1->dst_address.as_u32;
1180           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1181           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1182
1183           sum1 = ip1->checksum;
1184           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1185                                  dst_address);
1186           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1187             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1188                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
1189                                    src_address);
1190           ip1->checksum = ip_csum_fold (sum1);
1191
1192           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1193             {
1194               old_port1 = tcp1->dst_port;
1195               new_port1 = tcp1->dst_port = s1->in2out.port;
1196
1197               sum1 = tcp1->checksum;
1198               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1199                                      dst_address);
1200               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1201                                      length);
1202               if (is_twice_nat_session (s1))
1203                 {
1204                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1205                                          s1->ext_host_nat_addr.as_u32,
1206                                          ip4_header_t, dst_address);
1207                   sum1 = ip_csum_update (sum1, tcp1->src_port,
1208                                          s1->ext_host_nat_port, ip4_header_t,
1209                                          length);
1210                   tcp1->src_port = s1->ext_host_nat_port;
1211                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1212                 }
1213               tcp1->checksum = ip_csum_fold (sum1);
1214               tcp_packets++;
1215               if (nat44_set_tcp_session_state_o2i
1216                   (sm, s1, tcp1, thread_index))
1217                 goto trace01;
1218             }
1219           else
1220             {
1221               udp1->dst_port = s1->in2out.port;
1222               if (is_twice_nat_session (s1))
1223                 {
1224                   udp1->src_port = s1->ext_host_nat_port;
1225                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1226                 }
1227               udp1->checksum = 0;
1228               udp_packets++;
1229             }
1230
1231           /* Accounting */
1232           nat44_session_update_counters (s1, now,
1233                                          vlib_buffer_length_in_chain (vm,
1234                                                                       b1));
1235           /* Per-user LRU list maintenance */
1236           nat44_session_update_lru (sm, s1, thread_index);
1237
1238         trace01:
1239           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1240                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1241             {
1242               nat44_ed_out2in_trace_t *t =
1243                 vlib_add_trace (vm, node, b1, sizeof (*t));
1244               t->is_slow_path = is_slow_path;
1245               t->sw_if_index = sw_if_index1;
1246               t->next_index = next1;
1247               t->session_index = ~0;
1248               if (s1)
1249                 t->session_index = s1 - tsm->sessions;
1250             }
1251
1252           pkts_processed += next1 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1253
1254           /* verify speculative enqueues, maybe switch current next frame */
1255           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1256                                            to_next, n_left_to_next,
1257                                            bi0, bi1, next0, next1);
1258         }
1259
1260       while (n_left_from > 0 && n_left_to_next > 0)
1261         {
1262           u32 bi0;
1263           vlib_buffer_t *b0;
1264           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1265             new_addr0;
1266           u16 old_port0, new_port0;
1267           ip4_header_t *ip0;
1268           udp_header_t *udp0;
1269           tcp_header_t *tcp0;
1270           icmp46_header_t *icmp0;
1271           snat_session_t *s0 = 0;
1272           clib_bihash_kv_16_8_t kv0, value0;
1273           ip_csum_t sum0;
1274           snat_session_key_t e_key0, l_key0;
1275           lb_nat_type_t lb_nat0;
1276           twice_nat_type_t twice_nat0;
1277           u8 identity_nat0;
1278
1279           /* speculatively enqueue b0 to the current next frame */
1280           bi0 = from[0];
1281           to_next[0] = bi0;
1282           from += 1;
1283           to_next += 1;
1284           n_left_from -= 1;
1285           n_left_to_next -= 1;
1286
1287           b0 = vlib_get_buffer (vm, bi0);
1288           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1289           vnet_buffer (b0)->snat.flags = 0;
1290           ip0 = vlib_buffer_get_current (b0);
1291
1292           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1293           rx_fib_index0 =
1294             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1295                                                  sw_if_index0);
1296
1297           if (PREDICT_FALSE (ip0->ttl == 1))
1298             {
1299               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1300               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1301                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1302                                            0);
1303               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1304               goto trace0;
1305             }
1306
1307           udp0 = ip4_next_header (ip0);
1308           tcp0 = (tcp_header_t *) udp0;
1309           icmp0 = (icmp46_header_t *) udp0;
1310           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1311
1312           if (is_slow_path)
1313             {
1314               if (PREDICT_FALSE (proto0 == ~0))
1315                 {
1316                   s0 =
1317                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1318                                                    thread_index, now, vm,
1319                                                    node);
1320                   other_packets++;
1321                   if (!sm->forwarding_enabled)
1322                     {
1323                       if (!s0)
1324                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1325                       goto trace0;
1326                     }
1327                 }
1328
1329               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1330                 {
1331                   next0 = icmp_out2in_ed_slow_path
1332                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1333                      next0, now, thread_index, &s0);
1334                   icmp_packets++;
1335                   goto trace0;
1336                 }
1337             }
1338           else
1339             {
1340               if (PREDICT_FALSE (proto0 == ~0))
1341                 {
1342                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1343                   goto trace0;
1344                 }
1345
1346               if (ip4_is_fragment (ip0))
1347                 {
1348                   next0 = NAT44_ED_OUT2IN_NEXT_REASS;
1349                   fragments++;
1350                   goto trace0;
1351                 }
1352
1353               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1354                 {
1355                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1356                   goto trace0;
1357                 }
1358             }
1359
1360           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1361                       ip0->protocol, rx_fib_index0, udp0->dst_port,
1362                       udp0->src_port);
1363
1364           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1365             {
1366               if (is_slow_path)
1367                 {
1368                   /* Try to match static mapping by external address and port,
1369                      destination address and port in packet */
1370                   e_key0.addr = ip0->dst_address;
1371                   e_key0.port = udp0->dst_port;
1372                   e_key0.protocol = proto0;
1373                   e_key0.fib_index = rx_fib_index0;
1374                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1375                                                  &twice_nat0, &lb_nat0,
1376                                                  &ip0->src_address,
1377                                                  &identity_nat0))
1378                     {
1379                       /*
1380                        * Send DHCP packets to the ipv4 stack, or we won't
1381                        * be able to use dhcp client on the outside interface
1382                        */
1383                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1384                                          && (udp0->dst_port ==
1385                                              clib_host_to_net_u16
1386                                              (UDP_DST_PORT_dhcp_to_client))))
1387                         {
1388                           vnet_feature_next (&next0, b0);
1389                           goto trace0;
1390                         }
1391
1392                       if (!sm->forwarding_enabled)
1393                         {
1394                           b0->error =
1395                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1396                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1397                         }
1398                       else
1399                         {
1400                           if (next_src_nat (sm, ip0, ip0->protocol,
1401                                             udp0->src_port, udp0->dst_port,
1402                                             thread_index, rx_fib_index0))
1403                             {
1404                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1405                               goto trace0;
1406                             }
1407                           create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1408                                                  thread_index);
1409                         }
1410                       goto trace0;
1411                     }
1412
1413                   if (PREDICT_FALSE (identity_nat0))
1414                     goto trace0;
1415
1416                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1417                     {
1418                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1419                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1420                       goto trace0;
1421                     }
1422
1423                   /* Create session initiated by host from external network */
1424                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1425                                                              e_key0, node,
1426                                                              thread_index,
1427                                                              twice_nat0,
1428                                                              lb_nat0, now);
1429
1430                   if (!s0)
1431                     {
1432                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1433                       goto trace0;
1434                     }
1435                 }
1436               else
1437                 {
1438                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1439                   goto trace0;
1440                 }
1441             }
1442           else
1443             {
1444               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1445             }
1446
1447           old_addr0 = ip0->dst_address.as_u32;
1448           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1449           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1450
1451           sum0 = ip0->checksum;
1452           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1453                                  dst_address);
1454           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1455             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1456                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1457                                    src_address);
1458           ip0->checksum = ip_csum_fold (sum0);
1459
1460           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1461             {
1462               old_port0 = tcp0->dst_port;
1463               new_port0 = tcp0->dst_port = s0->in2out.port;
1464
1465               sum0 = tcp0->checksum;
1466               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1467                                      dst_address);
1468               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1469                                      length);
1470               if (is_twice_nat_session (s0))
1471                 {
1472                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1473                                          s0->ext_host_nat_addr.as_u32,
1474                                          ip4_header_t, dst_address);
1475                   sum0 = ip_csum_update (sum0, tcp0->src_port,
1476                                          s0->ext_host_nat_port, ip4_header_t,
1477                                          length);
1478                   tcp0->src_port = s0->ext_host_nat_port;
1479                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1480                 }
1481               tcp0->checksum = ip_csum_fold (sum0);
1482               tcp_packets++;
1483               if (nat44_set_tcp_session_state_o2i
1484                   (sm, s0, tcp0, thread_index))
1485                 goto trace0;
1486             }
1487           else
1488             {
1489               udp0->dst_port = s0->in2out.port;
1490               if (is_twice_nat_session (s0))
1491                 {
1492                   udp0->src_port = s0->ext_host_nat_port;
1493                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1494                 }
1495               udp0->checksum = 0;
1496               udp_packets++;
1497             }
1498
1499           /* Accounting */
1500           nat44_session_update_counters (s0, now,
1501                                          vlib_buffer_length_in_chain (vm,
1502                                                                       b0));
1503           /* Per-user LRU list maintenance */
1504           nat44_session_update_lru (sm, s0, thread_index);
1505
1506         trace0:
1507           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1508                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1509             {
1510               nat44_ed_out2in_trace_t *t =
1511                 vlib_add_trace (vm, node, b0, sizeof (*t));
1512               t->is_slow_path = is_slow_path;
1513               t->sw_if_index = sw_if_index0;
1514               t->next_index = next0;
1515               t->session_index = ~0;
1516               if (s0)
1517                 t->session_index = s0 - tsm->sessions;
1518             }
1519
1520           pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1521           /* verify speculative enqueue, maybe switch current next frame */
1522           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1523                                            to_next, n_left_to_next,
1524                                            bi0, next0);
1525         }
1526
1527       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1528     }
1529
1530   vlib_node_increment_counter (vm, stats_node_index,
1531                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1532                                pkts_processed);
1533   vlib_node_increment_counter (vm, stats_node_index,
1534                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1535   vlib_node_increment_counter (vm, stats_node_index,
1536                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1537   vlib_node_increment_counter (vm, stats_node_index,
1538                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1539                                icmp_packets);
1540   vlib_node_increment_counter (vm, stats_node_index,
1541                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1542                                other_packets);
1543   vlib_node_increment_counter (vm, stats_node_index,
1544                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1545   return frame->n_vectors;
1546 }
1547
1548 static uword
1549 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
1550                               vlib_node_runtime_t * node,
1551                               vlib_frame_t * frame)
1552 {
1553   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1554 }
1555
1556 /* *INDENT-OFF* */
1557 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1558   .function = nat44_ed_out2in_fast_path_fn,
1559   .name = "nat44-ed-out2in",
1560   .vector_size = sizeof (u32),
1561   .format_trace = format_nat44_ed_out2in_trace,
1562   .type = VLIB_NODE_TYPE_INTERNAL,
1563   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1564   .error_strings = nat_out2in_ed_error_strings,
1565   .runtime_data_bytes = sizeof (snat_runtime_t),
1566   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1567   .next_nodes = {
1568     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1569     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1570     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1571     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1572     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1573     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1574   },
1575 };
1576 /* *INDENT-ON* */
1577
1578 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node,
1579                               nat44_ed_out2in_fast_path_fn);
1580
1581 static uword
1582 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
1583                               vlib_node_runtime_t * node,
1584                               vlib_frame_t * frame)
1585 {
1586   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1587 }
1588
1589 /* *INDENT-OFF* */
1590 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1591   .function = nat44_ed_out2in_slow_path_fn,
1592   .name = "nat44-ed-out2in-slowpath",
1593   .vector_size = sizeof (u32),
1594   .format_trace = format_nat44_ed_out2in_trace,
1595   .type = VLIB_NODE_TYPE_INTERNAL,
1596   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1597   .error_strings = nat_out2in_ed_error_strings,
1598   .runtime_data_bytes = sizeof (snat_runtime_t),
1599   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1600   .next_nodes = {
1601     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1602     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1603     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1604     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1605     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1606     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1607   },
1608 };
1609 /* *INDENT-ON* */
1610
1611 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
1612                               nat44_ed_out2in_slow_path_fn);
1613
1614 static uword
1615 nat44_ed_out2in_reass_node_fn (vlib_main_t * vm,
1616                                vlib_node_runtime_t * node,
1617                                vlib_frame_t * frame)
1618 {
1619   u32 n_left_from, *from, *to_next;
1620   nat44_ed_out2in_next_t next_index;
1621   u32 pkts_processed = 0;
1622   snat_main_t *sm = &snat_main;
1623   f64 now = vlib_time_now (vm);
1624   u32 thread_index = vm->thread_index;
1625   snat_main_per_thread_data_t *per_thread_data =
1626     &sm->per_thread_data[thread_index];
1627   u32 *fragments_to_drop = 0;
1628   u32 *fragments_to_loopback = 0;
1629
1630   from = vlib_frame_vector_args (frame);
1631   n_left_from = frame->n_vectors;
1632   next_index = node->cached_next_index;
1633
1634   while (n_left_from > 0)
1635     {
1636       u32 n_left_to_next;
1637
1638       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1639
1640       while (n_left_from > 0 && n_left_to_next > 0)
1641         {
1642           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1643           vlib_buffer_t *b0;
1644           u32 next0;
1645           u8 cached0 = 0;
1646           ip4_header_t *ip0;
1647           nat_reass_ip4_t *reass0;
1648           udp_header_t *udp0;
1649           tcp_header_t *tcp0;
1650           icmp46_header_t *icmp0;
1651           clib_bihash_kv_16_8_t kv0, value0;
1652           snat_session_t *s0 = 0;
1653           u16 old_port0, new_port0;
1654           ip_csum_t sum0;
1655           snat_session_key_t e_key0, l_key0;
1656           lb_nat_type_t lb0;
1657           twice_nat_type_t twice_nat0;
1658           u8 identity_nat0;
1659
1660           /* speculatively enqueue b0 to the current next frame */
1661           bi0 = from[0];
1662           to_next[0] = bi0;
1663           from += 1;
1664           to_next += 1;
1665           n_left_from -= 1;
1666           n_left_to_next -= 1;
1667
1668           b0 = vlib_get_buffer (vm, bi0);
1669           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1670
1671           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1672           rx_fib_index0 =
1673             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1674                                                  sw_if_index0);
1675
1676           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1677             {
1678               next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1679               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1680               goto trace0;
1681             }
1682
1683           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1684           udp0 = ip4_next_header (ip0);
1685           tcp0 = (tcp_header_t *) udp0;
1686           icmp0 = (icmp46_header_t *) udp0;
1687           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1688
1689           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1690                                                  ip0->dst_address,
1691                                                  ip0->fragment_id,
1692                                                  ip0->protocol,
1693                                                  1, &fragments_to_drop);
1694
1695           if (PREDICT_FALSE (!reass0))
1696             {
1697               next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1698               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1699               nat_log_notice ("maximum reassemblies exceeded");
1700               goto trace0;
1701             }
1702
1703           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1704             {
1705               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1706                 {
1707                   next0 = icmp_out2in_ed_slow_path
1708                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1709                      next0, now, thread_index, &s0);
1710
1711                   if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP))
1712                     {
1713                       if (s0)
1714                         reass0->sess_index = s0 - per_thread_data->sessions;
1715                       else
1716                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1717                       reass0->thread_index = thread_index;
1718                       nat_ip4_reass_get_frags (reass0,
1719                                                &fragments_to_loopback);
1720                     }
1721
1722                   goto trace0;
1723                 }
1724
1725               make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1726                           ip0->protocol, rx_fib_index0, udp0->dst_port,
1727                           udp0->src_port);
1728
1729               if (clib_bihash_search_16_8
1730                   (&per_thread_data->out2in_ed, &kv0, &value0))
1731                 {
1732                   /* Try to match static mapping by external address and port,
1733                      destination address and port in packet */
1734                   e_key0.addr = ip0->dst_address;
1735                   e_key0.port = udp0->dst_port;
1736                   e_key0.protocol = proto0;
1737                   e_key0.fib_index = rx_fib_index0;
1738                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1739                                                  &twice_nat0, &lb0, 0,
1740                                                  &identity_nat0))
1741                     {
1742                       /*
1743                        * Send DHCP packets to the ipv4 stack, or we won't
1744                        * be able to use dhcp client on the outside interface
1745                        */
1746                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1747                                          && (udp0->dst_port
1748                                              ==
1749                                              clib_host_to_net_u16
1750                                              (UDP_DST_PORT_dhcp_to_client))))
1751                         {
1752                           vnet_feature_next (&next0, b0);
1753                           goto trace0;
1754                         }
1755
1756                       if (!sm->forwarding_enabled)
1757                         {
1758                           b0->error =
1759                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1760                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1761                         }
1762                       else
1763                         {
1764                           if (next_src_nat (sm, ip0, ip0->protocol,
1765                                             udp0->src_port, udp0->dst_port,
1766                                             thread_index, rx_fib_index0))
1767                             {
1768                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1769                               goto trace0;
1770                             }
1771                           create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1772                                                  thread_index);
1773                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1774                           nat_ip4_reass_get_frags (reass0,
1775                                                    &fragments_to_loopback);
1776                         }
1777                       goto trace0;
1778                     }
1779
1780                   if (PREDICT_FALSE (identity_nat0))
1781                     {
1782                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1783                       goto trace0;
1784                     }
1785
1786                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1787                     {
1788                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1789                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1790                       goto trace0;
1791                     }
1792
1793                   /* Create session initiated by host from external network */
1794                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1795                                                              e_key0, node,
1796                                                              thread_index,
1797                                                              twice_nat0, lb0,
1798                                                              now);
1799                   if (!s0)
1800                     {
1801                       b0->error =
1802                         node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1803                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1804                       goto trace0;
1805                     }
1806                   reass0->sess_index = s0 - per_thread_data->sessions;
1807                   reass0->thread_index = thread_index;
1808                 }
1809               else
1810                 {
1811                   s0 = pool_elt_at_index (per_thread_data->sessions,
1812                                           value0.value);
1813                   reass0->sess_index = value0.value;
1814                 }
1815               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1816             }
1817           else
1818             {
1819               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1820                 goto trace0;
1821               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1822                 {
1823                   if (nat_ip4_reass_add_fragment
1824                       (thread_index, reass0, bi0, &fragments_to_drop))
1825                     {
1826                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1827                       nat_log_notice
1828                         ("maximum fragments per reassembly exceeded");
1829                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1830                       goto trace0;
1831                     }
1832                   cached0 = 1;
1833                   goto trace0;
1834                 }
1835               s0 = pool_elt_at_index (per_thread_data->sessions,
1836                                       reass0->sess_index);
1837             }
1838
1839           old_addr0 = ip0->dst_address.as_u32;
1840           ip0->dst_address = s0->in2out.addr;
1841           new_addr0 = ip0->dst_address.as_u32;
1842           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1843
1844           sum0 = ip0->checksum;
1845           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1846                                  ip4_header_t,
1847                                  dst_address /* changed member */ );
1848           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1849             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1850                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1851                                    src_address);
1852           ip0->checksum = ip_csum_fold (sum0);
1853
1854           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1855             {
1856               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1857                 {
1858                   old_port0 = tcp0->dst_port;
1859                   tcp0->dst_port = s0->in2out.port;
1860                   new_port0 = tcp0->dst_port;
1861
1862                   sum0 = tcp0->checksum;
1863                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1864                                          ip4_header_t,
1865                                          dst_address /* changed member */ );
1866
1867                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1868                                          ip4_header_t /* cheat */ ,
1869                                          length /* changed member */ );
1870                   if (is_twice_nat_session (s0))
1871                     {
1872                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1873                                              s0->ext_host_nat_addr.as_u32,
1874                                              ip4_header_t, dst_address);
1875                       sum0 = ip_csum_update (sum0, tcp0->src_port,
1876                                              s0->ext_host_nat_port,
1877                                              ip4_header_t, length);
1878                       tcp0->src_port = s0->ext_host_nat_port;
1879                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1880                     }
1881                   tcp0->checksum = ip_csum_fold (sum0);
1882                 }
1883               else
1884                 {
1885                   old_port0 = udp0->dst_port;
1886                   udp0->dst_port = s0->in2out.port;
1887                   if (is_twice_nat_session (s0))
1888                     {
1889                       udp0->src_port = s0->ext_host_nat_port;
1890                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1891                     }
1892                   udp0->checksum = 0;
1893                 }
1894             }
1895
1896           /* Accounting */
1897           nat44_session_update_counters (s0, now,
1898                                          vlib_buffer_length_in_chain (vm,
1899                                                                       b0));
1900           /* Per-user LRU list maintenance */
1901           nat44_session_update_lru (sm, s0, thread_index);
1902
1903         trace0:
1904           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1905                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1906             {
1907               nat44_reass_trace_t *t =
1908                 vlib_add_trace (vm, node, b0, sizeof (*t));
1909               t->cached = cached0;
1910               t->sw_if_index = sw_if_index0;
1911               t->next_index = next0;
1912             }
1913
1914           if (cached0)
1915             {
1916               n_left_to_next++;
1917               to_next--;
1918             }
1919           else
1920             {
1921               pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1922
1923               /* verify speculative enqueue, maybe switch current next frame */
1924               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1925                                                to_next, n_left_to_next,
1926                                                bi0, next0);
1927             }
1928
1929           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1930             {
1931               from = vlib_frame_vector_args (frame);
1932               u32 len = vec_len (fragments_to_loopback);
1933               if (len <= VLIB_FRAME_SIZE)
1934                 {
1935                   clib_memcpy_fast (from, fragments_to_loopback,
1936                                     sizeof (u32) * len);
1937                   n_left_from = len;
1938                   vec_reset_length (fragments_to_loopback);
1939                 }
1940               else
1941                 {
1942                   clib_memcpy_fast (from, fragments_to_loopback +
1943                                     (len - VLIB_FRAME_SIZE),
1944                                     sizeof (u32) * VLIB_FRAME_SIZE);
1945                   n_left_from = VLIB_FRAME_SIZE;
1946                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1947                 }
1948             }
1949         }
1950
1951       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1952     }
1953
1954   vlib_node_increment_counter (vm, nat44_ed_out2in_reass_node.index,
1955                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1956                                pkts_processed);
1957
1958   nat_send_all_to_node (vm, fragments_to_drop, node,
1959                         &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
1960                         NAT44_ED_OUT2IN_NEXT_DROP);
1961
1962   vec_free (fragments_to_drop);
1963   vec_free (fragments_to_loopback);
1964   return frame->n_vectors;
1965 }
1966
1967 /* *INDENT-OFF* */
1968 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
1969   .function = nat44_ed_out2in_reass_node_fn,
1970   .name = "nat44-ed-out2in-reass",
1971   .vector_size = sizeof (u32),
1972   .format_trace = format_nat44_reass_trace,
1973   .type = VLIB_NODE_TYPE_INTERNAL,
1974   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1975   .error_strings = nat_out2in_ed_error_strings,
1976   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1977   .next_nodes = {
1978     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1979     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1980     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1981     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1982     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1983     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1984   },
1985 };
1986 /* *INDENT-ON* */
1987
1988 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_reass_node,
1989                               nat44_ed_out2in_reass_node_fn);
1990
1991 /*
1992  * fd.io coding-style-patch-verification: ON
1993  *
1994  * Local Variables:
1995  * eval: (c-set-style "gnu")
1996  * End:
1997  */