src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_reass.h>
  31 #include <nat/nat_inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 #define foreach_nat_out2in_ed_error                     \
  36 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  37 _(OUT2IN_PACKETS, "good out2in packets processed")      \
  38 _(OUT_OF_PORTS, "out of ports")                         \
  39 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  40 _(NO_TRANSLATION, "no translation")                     \
  41 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  42 _(DROP_FRAGMENT, "drop fragment")                       \
  43 _(MAX_REASS, "maximum reassemblies exceeded")           \
  44 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  45 _(NON_SYN, "non-SYN packet try to create session")      \
  46 _(TCP_PACKETS, "TCP packets")                           \
  47 _(UDP_PACKETS, "UDP packets")                           \
  48 _(ICMP_PACKETS, "ICMP packets")                         \
  49 _(OTHER_PACKETS, "other protocol packets")              \
  50 _(FRAGMENTS, "fragments")                               \
  51 _(CACHED_FRAGMENTS, "cached fragments")                 \
  52 _(PROCESSED_FRAGMENTS, "processed fragments")
  53
  54 typedef enum
  55 {
  56 #define _(sym,str) NAT_OUT2IN_ED_ERROR_##sym,
  57   foreach_nat_out2in_ed_error
  58 #undef _
  59     NAT_OUT2IN_ED_N_ERROR,
  60 } nat_out2in_ed_error_t;
  61
  62 static char *nat_out2in_ed_error_strings[] = {
  63 #define _(sym,string) string,
  64   foreach_nat_out2in_ed_error
  65 #undef _
  66 };
  67
  68 typedef enum
  69 {
  70   NAT44_ED_OUT2IN_NEXT_DROP,
  71   NAT44_ED_OUT2IN_NEXT_LOOKUP,
  72   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
  73   NAT44_ED_OUT2IN_NEXT_IN2OUT,
  74   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
  75   NAT44_ED_OUT2IN_NEXT_REASS,
  76   NAT44_ED_OUT2IN_N_NEXT,
  77 } nat44_ed_out2in_next_t;
  78
  79 typedef struct
  80 {
  81   u32 sw_if_index;
  82   u32 next_index;
  83   u32 session_index;
  84   u32 is_slow_path;
  85 } nat44_ed_out2in_trace_t;
  86
  87 static u8 *
  88 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  93   char *tag;
  94
  95   tag =
  96     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  97     "NAT44_OUT2IN_ED_FAST_PATH";
  98
  99   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
 100               t->sw_if_index, t->next_index, t->session_index);
 101
 102   return s;
 103 }
 104
 105 static inline u32
 106 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 107                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 108                           u32 sw_if_index0, u32 rx_fib_index0,
 109                           vlib_node_runtime_t * node, u32 next0, f64 now,
 110                           u32 thread_index, snat_session_t ** p_s0)
 111 {
 112   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 113                        next0, thread_index, p_s0, 0);
 114   snat_session_t *s0 = *p_s0;
 115   if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0))
 116     {
 117       /* Accounting */
 118       nat44_session_update_counters (s0, now,
 119                                      vlib_buffer_length_in_chain
 120                                      (sm->vlib_main, b0), thread_index);
 121       /* Per-user LRU list maintenance */
 122       nat44_session_update_lru (sm, s0, thread_index);
 123     }
 124   return next0;
 125 }
 126
 127 #ifndef CLIB_MARCH_VARIANT
 128 int
 129 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 130 {
 131   snat_main_t *sm = &snat_main;
 132   nat44_is_idle_session_ctx_t *ctx = arg;
 133   snat_session_t *s;
 134   u64 sess_timeout_time;
 135   nat_ed_ses_key_t ed_key;
 136   clib_bihash_kv_16_8_t ed_kv;
 137   int i;
 138   snat_address_t *a;
 139   snat_session_key_t key;
 140   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 141                                                        ctx->thread_index);
 142
 143   s = pool_elt_at_index (tsm->sessions, kv->value);
 144   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 145   if (ctx->now >= sess_timeout_time)
 146     {
 147       ed_key.l_addr = s->in2out.addr;
 148       ed_key.r_addr = s->ext_host_addr;
 149       ed_key.fib_index = s->in2out.fib_index;
 150       if (snat_is_unk_proto_session (s))
 151         {
 152           ed_key.proto = s->in2out.port;
 153           ed_key.r_port = 0;
 154           ed_key.l_port = 0;
 155         }
 156       else
 157         {
 158           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 159           ed_key.l_port = s->in2out.port;
 160           ed_key.r_port = s->ext_host_port;
 161         }
 162       if (is_twice_nat_session (s))
 163         {
 164           ed_key.r_addr = s->ext_host_nat_addr;
 165           ed_key.r_port = s->ext_host_nat_port;
 166         }
 167       ed_kv.key[0] = ed_key.as_u64[0];
 168       ed_kv.key[1] = ed_key.as_u64[1];
 169       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 170         nat_elog_warn ("in2out_ed key del failed");
 171
 172       if (snat_is_unk_proto_session (s))
 173         goto delete;
 174
 175       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 176                                            s->in2out.addr.as_u32,
 177                                            s->out2in.addr.as_u32,
 178                                            s->in2out.protocol,
 179                                            s->in2out.port,
 180                                            s->out2in.port,
 181                                            s->in2out.fib_index);
 182
 183       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 184                              &s->in2out.addr, s->in2out.port,
 185                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 186                              &s->out2in.addr, s->out2in.port,
 187                              &s->ext_host_addr, s->ext_host_port,
 188                              s->in2out.protocol, is_twice_nat_session (s));
 189
 190       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 191                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 192                    ctx->thread_index);
 193
 194       if (is_twice_nat_session (s))
 195         {
 196           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 197             {
 198               key.protocol = s->in2out.protocol;
 199               key.port = s->ext_host_nat_port;
 200               a = sm->twice_nat_addresses + i;
 201               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 202                 {
 203                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 204                                                       ctx->thread_index,
 205                                                       &key);
 206                   break;
 207                 }
 208             }
 209         }
 210
 211       if (snat_is_session_static (s))
 212         goto delete;
 213
 214       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 215                                           &s->out2in);
 216     delete:
 217       nat44_delete_session (sm, s, ctx->thread_index);
 218       return 1;
 219     }
 220
 221   return 0;
 222 }
 223 #endif
 224
 225 static snat_session_t *
 226 create_session_for_static_mapping_ed (snat_main_t * sm,
 227                                       vlib_buffer_t * b,
 228                                       snat_session_key_t l_key,
 229                                       snat_session_key_t e_key,
 230                                       vlib_node_runtime_t * node,
 231                                       u32 thread_index,
 232                                       twice_nat_type_t twice_nat,
 233                                       lb_nat_type_t lb_nat, f64 now)
 234 {
 235   snat_session_t *s;
 236   snat_user_t *u;
 237   ip4_header_t *ip;
 238   udp_header_t *udp;
 239   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 240   clib_bihash_kv_16_8_t kv;
 241   snat_session_key_t eh_key;
 242   nat44_is_idle_session_ctx_t ctx;
 243
 244   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 245     {
 246       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 247       nat_elog_notice ("maximum sessions exceeded");
 248       return 0;
 249     }
 250
 251   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
 252   if (!u)
 253     {
 254       nat_elog_warn ("create NAT user failed");
 255       return 0;
 256     }
 257
 258   s = nat_ed_session_alloc (sm, u, thread_index, now);
 259   if (!s)
 260     {
 261       nat44_delete_user_with_no_session (sm, u, thread_index);
 262       nat_elog_warn ("create NAT session failed");
 263       return 0;
 264     }
 265
 266   ip = vlib_buffer_get_current (b);
 267   udp = ip4_next_header (ip);
 268
 269   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 270   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 271   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 272   if (lb_nat)
 273     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 274   if (lb_nat == AFFINITY_LB_NAT)
 275     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 276   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 277   s->out2in = e_key;
 278   s->in2out = l_key;
 279   s->in2out.protocol = s->out2in.protocol;
 280   user_session_increment (sm, u, 1);
 281
 282   /* Add to lookup tables */
 283   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
 284               e_key.fib_index, e_key.port, s->ext_host_port);
 285   kv.value = s - tsm->sessions;
 286   ctx.now = now;
 287   ctx.thread_index = thread_index;
 288   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
 289                                                nat44_o2i_ed_is_idle_session_cb,
 290                                                &ctx))
 291     nat_elog_notice ("out2in-ed key add failed");
 292
 293   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 294                                  ip->src_address.as_u32 == l_key.addr.as_u32))
 295     {
 296       eh_key.protocol = e_key.protocol;
 297       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 298                                                thread_index, &eh_key,
 299                                                sm->port_per_thread,
 300                                                tsm->snat_thread_index))
 301         {
 302           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 303           nat44_delete_session (sm, s, thread_index);
 304           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
 305             nat_elog_notice ("out2in-ed key del failed");
 306           return 0;
 307         }
 308       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 309       s->ext_host_nat_port = eh_key.port;
 310       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 311       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
 312                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
 313     }
 314   else
 315     {
 316       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
 317                   l_key.fib_index, l_key.port, s->ext_host_port);
 318     }
 319   kv.value = s - tsm->sessions;
 320   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 321                                                nat44_i2o_ed_is_idle_session_cb,
 322                                                &ctx))
 323     nat_elog_notice ("in2out-ed key add failed");
 324
 325   snat_ipfix_logging_nat44_ses_create (thread_index,
 326                                        s->in2out.addr.as_u32,
 327                                        s->out2in.addr.as_u32,
 328                                        s->in2out.protocol,
 329                                        s->in2out.port,
 330                                        s->out2in.port, s->in2out.fib_index);
 331
 332   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 333                          &s->in2out.addr, s->in2out.port,
 334                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 335                          &s->out2in.addr, s->out2in.port,
 336                          &s->ext_host_addr, s->ext_host_port,
 337                          s->in2out.protocol, is_twice_nat_session (s));
 338
 339   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 340                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 341                &s->ext_host_nat_addr, s->ext_host_nat_port,
 342                s->in2out.protocol, s->in2out.fib_index, s->flags,
 343                thread_index, 0);
 344
 345   return s;
 346 }
 347
 348 static_always_inline int
 349 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 350 {
 351   icmp46_header_t *icmp0;
 352   nat_ed_ses_key_t key0;
 353   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 354   ip4_header_t *inner_ip0;
 355   void *l4_header = 0;
 356   icmp46_header_t *inner_icmp0;
 357
 358   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 359   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 360
 361   if (!icmp_is_error_message (icmp0))
 362     {
 363       key0.proto = IP_PROTOCOL_ICMP;
 364       key0.l_addr = ip0->dst_address;
 365       key0.r_addr = ip0->src_address;
 366       key0.l_port = echo0->identifier;
 367       key0.r_port = 0;
 368     }
 369   else
 370     {
 371       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 372       l4_header = ip4_next_header (inner_ip0);
 373       key0.proto = inner_ip0->protocol;
 374       key0.l_addr = inner_ip0->src_address;
 375       key0.r_addr = inner_ip0->dst_address;
 376       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 377         {
 378         case SNAT_PROTOCOL_ICMP:
 379           inner_icmp0 = (icmp46_header_t *) l4_header;
 380           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 381           key0.l_port = inner_echo0->identifier;
 382           key0.r_port = 0;
 383           break;
 384         case SNAT_PROTOCOL_UDP:
 385         case SNAT_PROTOCOL_TCP:
 386           key0.l_port = ((tcp_udp_header_t *) l4_header)->src_port;
 387           key0.r_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 388           break;
 389         default:
 390           return -1;
 391         }
 392     }
 393   *p_key0 = key0;
 394   return 0;
 395 }
 396
 397 static int
 398 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
 399               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 400 {
 401   clib_bihash_kv_16_8_t kv, value;
 402   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 403
 404   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
 405               rx_fib_index, src_port, dst_port);
 406   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 407     return 1;
 408
 409   return 0;
 410 }
 411
 412 static void
 413 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 414                        u32 thread_index)
 415 {
 416   nat_ed_ses_key_t key;
 417   clib_bihash_kv_16_8_t kv, value;
 418   udp_header_t *udp;
 419   snat_user_t *u;
 420   snat_session_t *s = 0;
 421   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 422   f64 now = vlib_time_now (sm->vlib_main);
 423
 424   if (ip->protocol == IP_PROTOCOL_ICMP)
 425     {
 426       if (icmp_get_ed_key (ip, &key))
 427         return;
 428     }
 429   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 430     {
 431       udp = ip4_next_header (ip);
 432       key.r_addr = ip->src_address;
 433       key.l_addr = ip->dst_address;
 434       key.proto = ip->protocol;
 435       key.l_port = udp->dst_port;
 436       key.r_port = udp->src_port;
 437     }
 438   else
 439     {
 440       key.r_addr = ip->src_address;
 441       key.l_addr = ip->dst_address;
 442       key.proto = ip->protocol;
 443       key.l_port = key.r_port = 0;
 444     }
 445   key.fib_index = 0;
 446   kv.key[0] = key.as_u64[0];
 447   kv.key[1] = key.as_u64[1];
 448
 449   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 450     {
 451       s = pool_elt_at_index (tsm->sessions, value.value);
 452     }
 453   else
 454     {
 455       u32 proto;
 456
 457       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 458         return;
 459
 460       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
 461                                   thread_index);
 462       if (!u)
 463         {
 464           nat_elog_warn ("create NAT user failed");
 465           return;
 466         }
 467
 468       s = nat_ed_session_alloc (sm, u, thread_index, now);
 469       if (!s)
 470         {
 471           nat44_delete_user_with_no_session (sm, u, thread_index);
 472           nat_elog_warn ("create NAT session failed");
 473           return;
 474         }
 475
 476       proto = ip_proto_to_snat_proto (key.proto);
 477
 478       s->ext_host_addr = key.r_addr;
 479       s->ext_host_port = key.r_port;
 480       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 481       s->out2in.addr = key.l_addr;
 482       s->out2in.port = key.l_port;
 483       s->out2in.protocol = proto;
 484       if (proto == ~0)
 485         {
 486           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 487           s->out2in.port = ip->protocol;
 488         }
 489       s->out2in.fib_index = 0;
 490       s->in2out = s->out2in;
 491       user_session_increment (sm, u, 0);
 492
 493       kv.value = s - tsm->sessions;
 494       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 495         nat_elog_notice ("in2out_ed key add failed");
 496     }
 497
 498   if (ip->protocol == IP_PROTOCOL_TCP)
 499     {
 500       tcp_header_t *tcp = ip4_next_header (ip);
 501       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
 502         return;
 503     }
 504
 505   /* Accounting */
 506   nat44_session_update_counters (s, now, 0, thread_index);
 507   /* Per-user LRU list maintenance */
 508   nat44_session_update_lru (sm, s, thread_index);
 509 }
 510
 511 static inline void
 512 create_bypass_for_fwd_worker (snat_main_t * sm, ip4_header_t * ip,
 513                               u32 rx_fib_index)
 514 {
 515   ip4_header_t ip_wkr = {
 516     .src_address = ip->dst_address,
 517   };
 518   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index);
 519
 520   create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
 521 }
 522
 523 #ifndef CLIB_MARCH_VARIANT
 524 u32
 525 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 526                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 527                       u8 * p_proto, snat_session_key_t * p_value,
 528                       u8 * p_dont_translate, void *d, void *e)
 529 {
 530   u32 next = ~0, sw_if_index, rx_fib_index;
 531   icmp46_header_t *icmp;
 532   nat_ed_ses_key_t key;
 533   clib_bihash_kv_16_8_t kv, value;
 534   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 535   snat_session_t *s = 0;
 536   u8 dont_translate = 0, is_addr_only, identity_nat;
 537   snat_session_key_t e_key, l_key;
 538
 539   icmp = (icmp46_header_t *) ip4_next_header (ip);
 540   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 541   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 542
 543   if (icmp_get_ed_key (ip, &key))
 544     {
 545       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 546       next = NAT44_ED_OUT2IN_NEXT_DROP;
 547       goto out;
 548     }
 549   key.fib_index = rx_fib_index;
 550   kv.key[0] = key.as_u64[0];
 551   kv.key[1] = key.as_u64[1];
 552
 553   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 554     {
 555       /* Try to match static mapping */
 556       e_key.addr = ip->dst_address;
 557       e_key.port = key.l_port;
 558       e_key.protocol = ip_proto_to_snat_proto (key.proto);
 559       e_key.fib_index = rx_fib_index;
 560       if (snat_static_mapping_match
 561           (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
 562         {
 563           if (!sm->forwarding_enabled)
 564             {
 565               /* Don't NAT packet aimed at the intfc address */
 566               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 567                                                     ip->dst_address.as_u32)))
 568                 {
 569                   dont_translate = 1;
 570                   goto out;
 571                 }
 572               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 573               next = NAT44_ED_OUT2IN_NEXT_DROP;
 574               goto out;
 575             }
 576           else
 577             {
 578               dont_translate = 1;
 579               if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
 580                                 thread_index, rx_fib_index))
 581                 {
 582                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
 583                   goto out;
 584                 }
 585               if (sm->num_workers > 1)
 586                 create_bypass_for_fwd_worker (sm, ip, rx_fib_index);
 587               else
 588                 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
 589               goto out;
 590             }
 591         }
 592
 593       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 594                          (icmp->type != ICMP4_echo_request || !is_addr_only)))
 595         {
 596           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 597           next = NAT44_ED_OUT2IN_NEXT_DROP;
 598           goto out;
 599         }
 600
 601       if (PREDICT_FALSE (identity_nat))
 602         {
 603           dont_translate = 1;
 604           goto out;
 605         }
 606
 607       /* Create session initiated by host from external network */
 608       s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
 609                                                 thread_index, 0, 0,
 610                                                 vlib_time_now
 611                                                 (sm->vlib_main));
 612
 613       if (!s)
 614         {
 615           next = NAT44_ED_OUT2IN_NEXT_DROP;
 616           goto out;
 617         }
 618     }
 619   else
 620     {
 621       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 622                          icmp->type != ICMP4_echo_request &&
 623                          !icmp_is_error_message (icmp)))
 624         {
 625           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 626           next = NAT44_ED_OUT2IN_NEXT_DROP;
 627           goto out;
 628         }
 629
 630       s = pool_elt_at_index (tsm->sessions, value.value);
 631     }
 632
 633   *p_proto = ip_proto_to_snat_proto (key.proto);
 634 out:
 635   if (s)
 636     *p_value = s->in2out;
 637   *p_dont_translate = dont_translate;
 638   if (d)
 639     *(snat_session_t **) d = s;
 640   return next;
 641 }
 642 #endif
 643
 644 static snat_session_t *
 645 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 646                                vlib_buffer_t * b,
 647                                ip4_header_t * ip,
 648                                u32 rx_fib_index,
 649                                u32 thread_index,
 650                                f64 now,
 651                                vlib_main_t * vm, vlib_node_runtime_t * node)
 652 {
 653   clib_bihash_kv_8_8_t kv, value;
 654   clib_bihash_kv_16_8_t s_kv, s_value;
 655   snat_static_mapping_t *m;
 656   u32 old_addr, new_addr;
 657   ip_csum_t sum;
 658   snat_session_t *s;
 659   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 660   snat_user_t *u;
 661
 662   old_addr = ip->dst_address.as_u32;
 663
 664   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 665               rx_fib_index, 0, 0);
 666
 667   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 668     {
 669       s = pool_elt_at_index (tsm->sessions, s_value.value);
 670       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 671     }
 672   else
 673     {
 674       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 675         {
 676           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 677           nat_elog_notice ("maximum sessions exceeded");
 678           return 0;
 679         }
 680
 681       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 682       if (clib_bihash_search_8_8
 683           (&sm->static_mapping_by_external, &kv, &value))
 684         {
 685           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 686           return 0;
 687         }
 688
 689       m = pool_elt_at_index (sm->static_mappings, value.value);
 690
 691       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 692
 693       u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
 694                                   thread_index);
 695       if (!u)
 696         {
 697           nat_elog_warn ("create NAT user failed");
 698           return 0;
 699         }
 700
 701       /* Create a new session */
 702       s = nat_ed_session_alloc (sm, u, thread_index, now);
 703       if (!s)
 704         {
 705           nat44_delete_user_with_no_session (sm, u, thread_index);
 706           nat_elog_warn ("create NAT session failed");
 707           return 0;
 708         }
 709
 710       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 711       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 712       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 713       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 714       s->out2in.addr.as_u32 = old_addr;
 715       s->out2in.fib_index = rx_fib_index;
 716       s->in2out.addr.as_u32 = new_addr;
 717       s->in2out.fib_index = m->fib_index;
 718       s->in2out.port = s->out2in.port = ip->protocol;
 719       user_session_increment (sm, u, 1);
 720
 721       /* Add to lookup tables */
 722       s_kv.value = s - tsm->sessions;
 723       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 724         nat_elog_notice ("out2in key add failed");
 725
 726       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 727                   m->fib_index, 0, 0);
 728       s_kv.value = s - tsm->sessions;
 729       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 730         nat_elog_notice ("in2out key add failed");
 731     }
 732
 733   /* Update IP checksum */
 734   sum = ip->checksum;
 735   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 736   ip->checksum = ip_csum_fold (sum);
 737
 738   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 739
 740   /* Accounting */
 741   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 742                                  thread_index);
 743   /* Per-user LRU list maintenance */
 744   nat44_session_update_lru (sm, s, thread_index);
 745
 746   return s;
 747 }
 748
 749 static inline uword
 750 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
 751                                 vlib_node_runtime_t * node,
 752                                 vlib_frame_t * frame, int is_slow_path)
 753 {
 754   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 755   nat44_ed_out2in_next_t next_index;
 756   snat_main_t *sm = &snat_main;
 757   f64 now = vlib_time_now (vm);
 758   u32 thread_index = vm->thread_index;
 759   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 760   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 761     0, fragments = 0;
 762
 763   stats_node_index = is_slow_path ? sm->ed_out2in_slowpath_node_index :
 764     sm->ed_out2in_node_index;
 765
 766   from = vlib_frame_vector_args (frame);
 767   n_left_from = frame->n_vectors;
 768   next_index = node->cached_next_index;
 769
 770   while (n_left_from > 0)
 771     {
 772       u32 n_left_to_next;
 773
 774       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 775
 776       while (n_left_from >= 4 && n_left_to_next >= 2)
 777         {
 778           u32 bi0, bi1;
 779           vlib_buffer_t *b0, *b1;
 780           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 781             new_addr0;
 782           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
 783             new_addr1;
 784           u16 old_port0, new_port0, old_port1, new_port1;
 785           ip4_header_t *ip0, *ip1;
 786           udp_header_t *udp0, *udp1;
 787           tcp_header_t *tcp0, *tcp1;
 788           icmp46_header_t *icmp0, *icmp1;
 789           snat_session_t *s0 = 0, *s1 = 0;
 790           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 791           ip_csum_t sum0, sum1;
 792           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
 793           lb_nat_type_t lb_nat0, lb_nat1;
 794           twice_nat_type_t twice_nat0, twice_nat1;
 795           u8 identity_nat0, identity_nat1;
 796
 797           /* Prefetch next iteration. */
 798           {
 799             vlib_buffer_t *p2, *p3;
 800
 801             p2 = vlib_get_buffer (vm, from[2]);
 802             p3 = vlib_get_buffer (vm, from[3]);
 803
 804             vlib_prefetch_buffer_header (p2, LOAD);
 805             vlib_prefetch_buffer_header (p3, LOAD);
 806
 807             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 808             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 809           }
 810
 811           /* speculatively enqueue b0 and b1 to the current next frame */
 812           to_next[0] = bi0 = from[0];
 813           to_next[1] = bi1 = from[1];
 814           from += 2;
 815           to_next += 2;
 816           n_left_from -= 2;
 817           n_left_to_next -= 2;
 818
 819           b0 = vlib_get_buffer (vm, bi0);
 820           b1 = vlib_get_buffer (vm, bi1);
 821
 822           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
 823           vnet_buffer (b0)->snat.flags = 0;
 824           ip0 = vlib_buffer_get_current (b0);
 825
 826           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 827           rx_fib_index0 =
 828             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 829                                                  sw_if_index0);
 830
 831           if (PREDICT_FALSE (ip0->ttl == 1))
 832             {
 833               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 834               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 835                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 836                                            0);
 837               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
 838               goto trace00;
 839             }
 840
 841           udp0 = ip4_next_header (ip0);
 842           tcp0 = (tcp_header_t *) udp0;
 843           icmp0 = (icmp46_header_t *) udp0;
 844           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 845
 846           if (is_slow_path)
 847             {
 848               if (PREDICT_FALSE (proto0 == ~0))
 849                 {
 850                   s0 =
 851                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
 852                                                    thread_index, now, vm,
 853                                                    node);
 854                   other_packets++;
 855                   if (!sm->forwarding_enabled)
 856                     {
 857                       if (!s0)
 858                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 859                       goto trace00;
 860                     }
 861                 }
 862
 863               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 864                 {
 865                   next0 = icmp_out2in_ed_slow_path
 866                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 867                      next0, now, thread_index, &s0);
 868                   icmp_packets++;
 869                   goto trace00;
 870                 }
 871             }
 872           else
 873             {
 874               if (PREDICT_FALSE (proto0 == ~0))
 875                 {
 876                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 877                   goto trace00;
 878                 }
 879
 880               if (ip4_is_fragment (ip0))
 881                 {
 882                   next0 = NAT44_ED_OUT2IN_NEXT_REASS;
 883                   fragments++;
 884                   goto trace00;
 885                 }
 886
 887               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 888                 {
 889                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 890                   goto trace00;
 891                 }
 892             }
 893
 894           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
 895                       ip0->protocol, rx_fib_index0, udp0->dst_port,
 896                       udp0->src_port);
 897
 898           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
 899             {
 900               if (is_slow_path)
 901                 {
 902                   /* Try to match static mapping by external address and port,
 903                      destination address and port in packet */
 904                   e_key0.addr = ip0->dst_address;
 905                   e_key0.port = udp0->dst_port;
 906                   e_key0.protocol = proto0;
 907                   e_key0.fib_index = rx_fib_index0;
 908                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
 909                                                  &twice_nat0, &lb_nat0,
 910                                                  &ip0->src_address,
 911                                                  &identity_nat0))
 912                     {
 913                       /*
 914                        * Send DHCP packets to the ipv4 stack, or we won't
 915                        * be able to use dhcp client on the outside interface
 916                        */
 917                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 918                                          && (udp0->dst_port ==
 919                                              clib_host_to_net_u16
 920                                              (UDP_DST_PORT_dhcp_to_client))))
 921                         {
 922                           vnet_feature_next (&next0, b0);
 923                           goto trace00;
 924                         }
 925
 926                       if (!sm->forwarding_enabled)
 927                         {
 928                           b0->error =
 929                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 930                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 931                         }
 932                       else
 933                         {
 934                           if (next_src_nat (sm, ip0, ip0->protocol,
 935                                             udp0->src_port, udp0->dst_port,
 936                                             thread_index, rx_fib_index0))
 937                             {
 938                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
 939                               goto trace00;
 940                             }
 941                           if (sm->num_workers > 1)
 942                             create_bypass_for_fwd_worker (sm, ip0,
 943                                                           rx_fib_index0);
 944                           else
 945                             create_bypass_for_fwd (sm, ip0, rx_fib_index0,
 946                                                    thread_index);
 947                         }
 948                       goto trace00;
 949                     }
 950
 951                   if (PREDICT_FALSE (identity_nat0))
 952                     goto trace00;
 953
 954                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
 955                     {
 956                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
 957                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 958                       goto trace00;
 959                     }
 960
 961                   /* Create session initiated by host from external network */
 962                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
 963                                                              e_key0, node,
 964                                                              thread_index,
 965                                                              twice_nat0,
 966                                                              lb_nat0, now);
 967
 968                   if (!s0)
 969                     {
 970                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 971                       goto trace00;
 972                     }
 973                 }
 974               else
 975                 {
 976                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 977                   goto trace00;
 978                 }
 979             }
 980           else
 981             {
 982               s0 = pool_elt_at_index (tsm->sessions, value0.value);
 983             }
 984
 985           old_addr0 = ip0->dst_address.as_u32;
 986           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 987           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 988
 989           sum0 = ip0->checksum;
 990           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 991                                  dst_address);
 992           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 993             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 994                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
 995                                    src_address);
 996           ip0->checksum = ip_csum_fold (sum0);
 997
 998           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 999             {
1000               old_port0 = tcp0->dst_port;
1001               new_port0 = tcp0->dst_port = s0->in2out.port;
1002
1003               sum0 = tcp0->checksum;
1004               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1005                                      dst_address);
1006               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1007                                      length);
1008               if (is_twice_nat_session (s0))
1009                 {
1010                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1011                                          s0->ext_host_nat_addr.as_u32,
1012                                          ip4_header_t, dst_address);
1013                   sum0 = ip_csum_update (sum0, tcp0->src_port,
1014                                          s0->ext_host_nat_port, ip4_header_t,
1015                                          length);
1016                   tcp0->src_port = s0->ext_host_nat_port;
1017                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1018                 }
1019               tcp0->checksum = ip_csum_fold (sum0);
1020               tcp_packets++;
1021               if (nat44_set_tcp_session_state_o2i
1022                   (sm, s0, tcp0, thread_index))
1023                 goto trace00;
1024             }
1025           else
1026             {
1027               udp0->dst_port = s0->in2out.port;
1028               if (is_twice_nat_session (s0))
1029                 {
1030                   udp0->src_port = s0->ext_host_nat_port;
1031                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1032                 }
1033               udp0->checksum = 0;
1034               udp_packets++;
1035             }
1036
1037           /* Accounting */
1038           nat44_session_update_counters (s0, now,
1039                                          vlib_buffer_length_in_chain (vm, b0),
1040                                          thread_index);
1041           /* Per-user LRU list maintenance */
1042           nat44_session_update_lru (sm, s0, thread_index);
1043
1044         trace00:
1045           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1046                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1047             {
1048               nat44_ed_out2in_trace_t *t =
1049                 vlib_add_trace (vm, node, b0, sizeof (*t));
1050               t->is_slow_path = is_slow_path;
1051               t->sw_if_index = sw_if_index0;
1052               t->next_index = next0;
1053               t->session_index = ~0;
1054               if (s0)
1055                 t->session_index = s0 - tsm->sessions;
1056             }
1057
1058           pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1059
1060           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1061           vnet_buffer (b1)->snat.flags = 0;
1062           ip1 = vlib_buffer_get_current (b1);
1063
1064           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1065           rx_fib_index1 =
1066             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1067                                                  sw_if_index1);
1068
1069           if (PREDICT_FALSE (ip1->ttl == 1))
1070             {
1071               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1072               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1073                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1074                                            0);
1075               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1076               goto trace01;
1077             }
1078
1079           udp1 = ip4_next_header (ip1);
1080           tcp1 = (tcp_header_t *) udp1;
1081           icmp1 = (icmp46_header_t *) udp1;
1082           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1083
1084           if (is_slow_path)
1085             {
1086               if (PREDICT_FALSE (proto1 == ~0))
1087                 {
1088                   s1 =
1089                     nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1090                                                    thread_index, now, vm,
1091                                                    node);
1092                   other_packets++;
1093                   if (!sm->forwarding_enabled)
1094                     {
1095                       if (!s1)
1096                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1097                       goto trace01;
1098                     }
1099                 }
1100
1101               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1102                 {
1103                   next1 = icmp_out2in_ed_slow_path
1104                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1105                      next1, now, thread_index, &s1);
1106                   icmp_packets++;
1107                   goto trace01;
1108                 }
1109             }
1110           else
1111             {
1112               if (PREDICT_FALSE (proto1 == ~0))
1113                 {
1114                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1115                   goto trace01;
1116                 }
1117
1118               if (ip4_is_fragment (ip1))
1119                 {
1120                   next1 = NAT44_ED_OUT2IN_NEXT_REASS;
1121                   fragments++;
1122                   goto trace01;
1123                 }
1124
1125               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1126                 {
1127                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1128                   goto trace01;
1129                 }
1130             }
1131
1132           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1133                       ip1->protocol, rx_fib_index1, udp1->dst_port,
1134                       udp1->src_port);
1135
1136           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1137             {
1138               if (is_slow_path)
1139                 {
1140                   /* Try to match static mapping by external address and port,
1141                      destination address and port in packet */
1142                   e_key1.addr = ip1->dst_address;
1143                   e_key1.port = udp1->dst_port;
1144                   e_key1.protocol = proto1;
1145                   e_key1.fib_index = rx_fib_index1;
1146                   if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1147                                                  &twice_nat1, &lb_nat1,
1148                                                  &ip1->src_address,
1149                                                  &identity_nat1))
1150                     {
1151                       /*
1152                        * Send DHCP packets to the ipv4 stack, or we won't
1153                        * be able to use dhcp client on the outside interface
1154                        */
1155                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1156                                          && (udp1->dst_port ==
1157                                              clib_host_to_net_u16
1158                                              (UDP_DST_PORT_dhcp_to_client))))
1159                         {
1160                           vnet_feature_next (&next1, b1);
1161                           goto trace01;
1162                         }
1163
1164                       if (!sm->forwarding_enabled)
1165                         {
1166                           b1->error =
1167                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1168                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1169                         }
1170                       else
1171                         {
1172                           if (next_src_nat (sm, ip1, ip1->protocol,
1173                                             udp1->src_port, udp1->dst_port,
1174                                             thread_index, rx_fib_index1))
1175                             {
1176                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1177                               goto trace01;
1178                             }
1179                           if (sm->num_workers > 1)
1180                             create_bypass_for_fwd_worker (sm, ip1,
1181                                                           rx_fib_index1);
1182                           else
1183                             create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1184                                                    thread_index);
1185                         }
1186                       goto trace01;
1187                     }
1188
1189                   if (PREDICT_FALSE (identity_nat1))
1190                     goto trace01;
1191
1192                   if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1193                     {
1194                       b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1195                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1196                       goto trace01;
1197                     }
1198
1199                   /* Create session initiated by host from external network */
1200                   s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1201                                                              e_key1, node,
1202                                                              thread_index,
1203                                                              twice_nat1,
1204                                                              lb_nat1, now);
1205
1206                   if (!s1)
1207                     {
1208                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1209                       goto trace01;
1210                     }
1211                 }
1212               else
1213                 {
1214                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1215                   goto trace01;
1216                 }
1217             }
1218           else
1219             {
1220               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1221             }
1222
1223           old_addr1 = ip1->dst_address.as_u32;
1224           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1225           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1226
1227           sum1 = ip1->checksum;
1228           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1229                                  dst_address);
1230           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1231             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1232                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
1233                                    src_address);
1234           ip1->checksum = ip_csum_fold (sum1);
1235
1236           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1237             {
1238               old_port1 = tcp1->dst_port;
1239               new_port1 = tcp1->dst_port = s1->in2out.port;
1240
1241               sum1 = tcp1->checksum;
1242               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1243                                      dst_address);
1244               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1245                                      length);
1246               if (is_twice_nat_session (s1))
1247                 {
1248                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1249                                          s1->ext_host_nat_addr.as_u32,
1250                                          ip4_header_t, dst_address);
1251                   sum1 = ip_csum_update (sum1, tcp1->src_port,
1252                                          s1->ext_host_nat_port, ip4_header_t,
1253                                          length);
1254                   tcp1->src_port = s1->ext_host_nat_port;
1255                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1256                 }
1257               tcp1->checksum = ip_csum_fold (sum1);
1258               tcp_packets++;
1259               if (nat44_set_tcp_session_state_o2i
1260                   (sm, s1, tcp1, thread_index))
1261                 goto trace01;
1262             }
1263           else
1264             {
1265               udp1->dst_port = s1->in2out.port;
1266               if (is_twice_nat_session (s1))
1267                 {
1268                   udp1->src_port = s1->ext_host_nat_port;
1269                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1270                 }
1271               udp1->checksum = 0;
1272               udp_packets++;
1273             }
1274
1275           /* Accounting */
1276           nat44_session_update_counters (s1, now,
1277                                          vlib_buffer_length_in_chain (vm, b1),
1278                                          thread_index);
1279           /* Per-user LRU list maintenance */
1280           nat44_session_update_lru (sm, s1, thread_index);
1281
1282         trace01:
1283           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1284                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1285             {
1286               nat44_ed_out2in_trace_t *t =
1287                 vlib_add_trace (vm, node, b1, sizeof (*t));
1288               t->is_slow_path = is_slow_path;
1289               t->sw_if_index = sw_if_index1;
1290               t->next_index = next1;
1291               t->session_index = ~0;
1292               if (s1)
1293                 t->session_index = s1 - tsm->sessions;
1294             }
1295
1296           pkts_processed += next1 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1297
1298           /* verify speculative enqueues, maybe switch current next frame */
1299           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1300                                            to_next, n_left_to_next,
1301                                            bi0, bi1, next0, next1);
1302         }
1303
1304       while (n_left_from > 0 && n_left_to_next > 0)
1305         {
1306           u32 bi0;
1307           vlib_buffer_t *b0;
1308           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1309             new_addr0;
1310           u16 old_port0, new_port0;
1311           ip4_header_t *ip0;
1312           udp_header_t *udp0;
1313           tcp_header_t *tcp0;
1314           icmp46_header_t *icmp0;
1315           snat_session_t *s0 = 0;
1316           clib_bihash_kv_16_8_t kv0, value0;
1317           ip_csum_t sum0;
1318           snat_session_key_t e_key0, l_key0;
1319           lb_nat_type_t lb_nat0;
1320           twice_nat_type_t twice_nat0;
1321           u8 identity_nat0;
1322
1323           /* speculatively enqueue b0 to the current next frame */
1324           bi0 = from[0];
1325           to_next[0] = bi0;
1326           from += 1;
1327           to_next += 1;
1328           n_left_from -= 1;
1329           n_left_to_next -= 1;
1330
1331           b0 = vlib_get_buffer (vm, bi0);
1332           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1333           vnet_buffer (b0)->snat.flags = 0;
1334           ip0 = vlib_buffer_get_current (b0);
1335
1336           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1337           rx_fib_index0 =
1338             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1339                                                  sw_if_index0);
1340
1341           if (PREDICT_FALSE (ip0->ttl == 1))
1342             {
1343               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1344               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1345                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1346                                            0);
1347               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1348               goto trace0;
1349             }
1350
1351           udp0 = ip4_next_header (ip0);
1352           tcp0 = (tcp_header_t *) udp0;
1353           icmp0 = (icmp46_header_t *) udp0;
1354           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1355
1356           if (is_slow_path)
1357             {
1358               if (PREDICT_FALSE (proto0 == ~0))
1359                 {
1360                   s0 =
1361                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1362                                                    thread_index, now, vm,
1363                                                    node);
1364                   other_packets++;
1365                   if (!sm->forwarding_enabled)
1366                     {
1367                       if (!s0)
1368                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1369                       goto trace0;
1370                     }
1371                 }
1372
1373               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1374                 {
1375                   next0 = icmp_out2in_ed_slow_path
1376                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1377                      next0, now, thread_index, &s0);
1378                   icmp_packets++;
1379                   goto trace0;
1380                 }
1381             }
1382           else
1383             {
1384               if (PREDICT_FALSE (proto0 == ~0))
1385                 {
1386                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1387                   goto trace0;
1388                 }
1389
1390               if (ip4_is_fragment (ip0))
1391                 {
1392                   next0 = NAT44_ED_OUT2IN_NEXT_REASS;
1393                   fragments++;
1394                   goto trace0;
1395                 }
1396
1397               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1398                 {
1399                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1400                   goto trace0;
1401                 }
1402             }
1403
1404           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1405                       ip0->protocol, rx_fib_index0, udp0->dst_port,
1406                       udp0->src_port);
1407
1408           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1409             {
1410               if (is_slow_path)
1411                 {
1412                   /* Try to match static mapping by external address and port,
1413                      destination address and port in packet */
1414                   e_key0.addr = ip0->dst_address;
1415                   e_key0.port = udp0->dst_port;
1416                   e_key0.protocol = proto0;
1417                   e_key0.fib_index = rx_fib_index0;
1418                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1419                                                  &twice_nat0, &lb_nat0,
1420                                                  &ip0->src_address,
1421                                                  &identity_nat0))
1422                     {
1423                       /*
1424                        * Send DHCP packets to the ipv4 stack, or we won't
1425                        * be able to use dhcp client on the outside interface
1426                        */
1427                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1428                                          && (udp0->dst_port ==
1429                                              clib_host_to_net_u16
1430                                              (UDP_DST_PORT_dhcp_to_client))))
1431                         {
1432                           vnet_feature_next (&next0, b0);
1433                           goto trace0;
1434                         }
1435
1436                       if (!sm->forwarding_enabled)
1437                         {
1438                           b0->error =
1439                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1440                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1441                         }
1442                       else
1443                         {
1444                           if (next_src_nat (sm, ip0, ip0->protocol,
1445                                             udp0->src_port, udp0->dst_port,
1446                                             thread_index, rx_fib_index0))
1447                             {
1448                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1449                               goto trace0;
1450                             }
1451                           if (sm->num_workers > 1)
1452                             create_bypass_for_fwd_worker (sm, ip0,
1453                                                           rx_fib_index0);
1454                           else
1455                             create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1456                                                    thread_index);
1457                         }
1458                       goto trace0;
1459                     }
1460
1461                   if (PREDICT_FALSE (identity_nat0))
1462                     goto trace0;
1463
1464                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1465                     {
1466                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1467                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1468                       goto trace0;
1469                     }
1470
1471                   /* Create session initiated by host from external network */
1472                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1473                                                              e_key0, node,
1474                                                              thread_index,
1475                                                              twice_nat0,
1476                                                              lb_nat0, now);
1477
1478                   if (!s0)
1479                     {
1480                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1481                       goto trace0;
1482                     }
1483                 }
1484               else
1485                 {
1486                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1487                   goto trace0;
1488                 }
1489             }
1490           else
1491             {
1492               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1493             }
1494
1495           old_addr0 = ip0->dst_address.as_u32;
1496           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1497           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1498
1499           sum0 = ip0->checksum;
1500           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1501                                  dst_address);
1502           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1503             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1504                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1505                                    src_address);
1506           ip0->checksum = ip_csum_fold (sum0);
1507
1508           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1509             {
1510               old_port0 = tcp0->dst_port;
1511               new_port0 = tcp0->dst_port = s0->in2out.port;
1512
1513               sum0 = tcp0->checksum;
1514               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1515                                      dst_address);
1516               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1517                                      length);
1518               if (is_twice_nat_session (s0))
1519                 {
1520                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1521                                          s0->ext_host_nat_addr.as_u32,
1522                                          ip4_header_t, dst_address);
1523                   sum0 = ip_csum_update (sum0, tcp0->src_port,
1524                                          s0->ext_host_nat_port, ip4_header_t,
1525                                          length);
1526                   tcp0->src_port = s0->ext_host_nat_port;
1527                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1528                 }
1529               tcp0->checksum = ip_csum_fold (sum0);
1530               tcp_packets++;
1531               if (nat44_set_tcp_session_state_o2i
1532                   (sm, s0, tcp0, thread_index))
1533                 goto trace0;
1534             }
1535           else
1536             {
1537               udp0->dst_port = s0->in2out.port;
1538               if (is_twice_nat_session (s0))
1539                 {
1540                   udp0->src_port = s0->ext_host_nat_port;
1541                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1542                 }
1543               udp0->checksum = 0;
1544               udp_packets++;
1545             }
1546
1547           /* Accounting */
1548           nat44_session_update_counters (s0, now,
1549                                          vlib_buffer_length_in_chain (vm, b0),
1550                                          thread_index);
1551           /* Per-user LRU list maintenance */
1552           nat44_session_update_lru (sm, s0, thread_index);
1553
1554         trace0:
1555           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1556                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1557             {
1558               nat44_ed_out2in_trace_t *t =
1559                 vlib_add_trace (vm, node, b0, sizeof (*t));
1560               t->is_slow_path = is_slow_path;
1561               t->sw_if_index = sw_if_index0;
1562               t->next_index = next0;
1563               t->session_index = ~0;
1564               if (s0)
1565                 t->session_index = s0 - tsm->sessions;
1566             }
1567
1568           pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1569           /* verify speculative enqueue, maybe switch current next frame */
1570           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1571                                            to_next, n_left_to_next,
1572                                            bi0, next0);
1573         }
1574
1575       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1576     }
1577
1578   vlib_node_increment_counter (vm, stats_node_index,
1579                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1580                                pkts_processed);
1581   vlib_node_increment_counter (vm, stats_node_index,
1582                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1583   vlib_node_increment_counter (vm, stats_node_index,
1584                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1585   vlib_node_increment_counter (vm, stats_node_index,
1586                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1587                                icmp_packets);
1588   vlib_node_increment_counter (vm, stats_node_index,
1589                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1590                                other_packets);
1591   vlib_node_increment_counter (vm, stats_node_index,
1592                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1593   return frame->n_vectors;
1594 }
1595
1596 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1597                                      vlib_node_runtime_t * node,
1598                                      vlib_frame_t * frame)
1599 {
1600   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1601 }
1602
1603 /* *INDENT-OFF* */
1604 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1605   .name = "nat44-ed-out2in",
1606   .vector_size = sizeof (u32),
1607   .format_trace = format_nat44_ed_out2in_trace,
1608   .type = VLIB_NODE_TYPE_INTERNAL,
1609   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1610   .error_strings = nat_out2in_ed_error_strings,
1611   .runtime_data_bytes = sizeof (snat_runtime_t),
1612   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1613   .next_nodes = {
1614     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1615     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1616     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1617     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1618     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1619     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1620   },
1621 };
1622 /* *INDENT-ON* */
1623
1624 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1625                                               vlib_node_runtime_t * node,
1626                                               vlib_frame_t * frame)
1627 {
1628   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1629 }
1630
1631 /* *INDENT-OFF* */
1632 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1633   .name = "nat44-ed-out2in-slowpath",
1634   .vector_size = sizeof (u32),
1635   .format_trace = format_nat44_ed_out2in_trace,
1636   .type = VLIB_NODE_TYPE_INTERNAL,
1637   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1638   .error_strings = nat_out2in_ed_error_strings,
1639   .runtime_data_bytes = sizeof (snat_runtime_t),
1640   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1641   .next_nodes = {
1642     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1643     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1644     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1645     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1646     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1647     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1648   },
1649 };
1650 /* *INDENT-ON* */
1651
1652 VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm,
1653                                            vlib_node_runtime_t * node,
1654                                            vlib_frame_t * frame)
1655 {
1656   u32 n_left_from, *from, *to_next;
1657   nat44_ed_out2in_next_t next_index;
1658   u32 pkts_processed = 0;
1659   snat_main_t *sm = &snat_main;
1660   f64 now = vlib_time_now (vm);
1661   u32 thread_index = vm->thread_index;
1662   snat_main_per_thread_data_t *per_thread_data =
1663     &sm->per_thread_data[thread_index];
1664   u32 *fragments_to_drop = 0;
1665   u32 *fragments_to_loopback = 0;
1666
1667   from = vlib_frame_vector_args (frame);
1668   n_left_from = frame->n_vectors;
1669   next_index = node->cached_next_index;
1670
1671   while (n_left_from > 0)
1672     {
1673       u32 n_left_to_next;
1674
1675       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1676
1677       while (n_left_from > 0 && n_left_to_next > 0)
1678         {
1679           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1680           vlib_buffer_t *b0;
1681           u32 next0;
1682           u8 cached0 = 0;
1683           ip4_header_t *ip0;
1684           nat_reass_ip4_t *reass0;
1685           udp_header_t *udp0;
1686           tcp_header_t *tcp0;
1687           icmp46_header_t *icmp0;
1688           clib_bihash_kv_16_8_t kv0, value0;
1689           snat_session_t *s0 = 0;
1690           u16 old_port0, new_port0;
1691           ip_csum_t sum0;
1692           snat_session_key_t e_key0, l_key0;
1693           lb_nat_type_t lb0;
1694           twice_nat_type_t twice_nat0;
1695           u8 identity_nat0;
1696
1697           /* speculatively enqueue b0 to the current next frame */
1698           bi0 = from[0];
1699           to_next[0] = bi0;
1700           from += 1;
1701           to_next += 1;
1702           n_left_from -= 1;
1703           n_left_to_next -= 1;
1704
1705           b0 = vlib_get_buffer (vm, bi0);
1706           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1707
1708           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1709           rx_fib_index0 =
1710             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1711                                                  sw_if_index0);
1712
1713           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1714             {
1715               next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1716               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1717               goto trace0;
1718             }
1719
1720           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1721           udp0 = ip4_next_header (ip0);
1722           tcp0 = (tcp_header_t *) udp0;
1723           icmp0 = (icmp46_header_t *) udp0;
1724           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1725
1726           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1727                                                  ip0->dst_address,
1728                                                  ip0->fragment_id,
1729                                                  ip0->protocol,
1730                                                  1, &fragments_to_drop);
1731
1732           if (PREDICT_FALSE (!reass0))
1733             {
1734               next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1735               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1736               nat_elog_notice ("maximum reassemblies exceeded");
1737               goto trace0;
1738             }
1739
1740           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1741             {
1742               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1743                 {
1744                   next0 = icmp_out2in_ed_slow_path
1745                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1746                      next0, now, thread_index, &s0);
1747
1748                   if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP))
1749                     {
1750                       if (s0)
1751                         reass0->sess_index = s0 - per_thread_data->sessions;
1752                       else
1753                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1754                       reass0->thread_index = thread_index;
1755                       nat_ip4_reass_get_frags (reass0,
1756                                                &fragments_to_loopback);
1757                     }
1758
1759                   goto trace0;
1760                 }
1761
1762               make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1763                           ip0->protocol, rx_fib_index0, udp0->dst_port,
1764                           udp0->src_port);
1765
1766               if (clib_bihash_search_16_8
1767                   (&per_thread_data->out2in_ed, &kv0, &value0))
1768                 {
1769                   /* Try to match static mapping by external address and port,
1770                      destination address and port in packet */
1771                   e_key0.addr = ip0->dst_address;
1772                   e_key0.port = udp0->dst_port;
1773                   e_key0.protocol = proto0;
1774                   e_key0.fib_index = rx_fib_index0;
1775                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1776                                                  &twice_nat0, &lb0, 0,
1777                                                  &identity_nat0))
1778                     {
1779                       /*
1780                        * Send DHCP packets to the ipv4 stack, or we won't
1781                        * be able to use dhcp client on the outside interface
1782                        */
1783                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1784                                          && (udp0->dst_port
1785                                              ==
1786                                              clib_host_to_net_u16
1787                                              (UDP_DST_PORT_dhcp_to_client))))
1788                         {
1789                           vnet_feature_next (&next0, b0);
1790                           goto trace0;
1791                         }
1792
1793                       if (!sm->forwarding_enabled)
1794                         {
1795                           b0->error =
1796                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1797                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1798                         }
1799                       else
1800                         {
1801                           if (next_src_nat (sm, ip0, ip0->protocol,
1802                                             udp0->src_port, udp0->dst_port,
1803                                             thread_index, rx_fib_index0))
1804                             {
1805                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1806                               goto trace0;
1807                             }
1808                           if (sm->num_workers > 1)
1809                             create_bypass_for_fwd_worker (sm, ip0,
1810                                                           rx_fib_index0);
1811                           else
1812                             create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1813                                                    thread_index);
1814                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1815                           nat_ip4_reass_get_frags (reass0,
1816                                                    &fragments_to_loopback);
1817                         }
1818                       goto trace0;
1819                     }
1820
1821                   if (PREDICT_FALSE (identity_nat0))
1822                     {
1823                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1824                       goto trace0;
1825                     }
1826
1827                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1828                     {
1829                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1830                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1831                       goto trace0;
1832                     }
1833
1834                   /* Create session initiated by host from external network */
1835                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1836                                                              e_key0, node,
1837                                                              thread_index,
1838                                                              twice_nat0, lb0,
1839                                                              now);
1840                   if (!s0)
1841                     {
1842                       b0->error =
1843                         node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1844                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1845                       goto trace0;
1846                     }
1847                   reass0->sess_index = s0 - per_thread_data->sessions;
1848                   reass0->thread_index = thread_index;
1849                 }
1850               else
1851                 {
1852                   s0 = pool_elt_at_index (per_thread_data->sessions,
1853                                           value0.value);
1854                   reass0->sess_index = value0.value;
1855                 }
1856               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1857             }
1858           else
1859             {
1860               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1861                 goto trace0;
1862               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1863                 {
1864                   if (nat_ip4_reass_add_fragment
1865                       (thread_index, reass0, bi0, &fragments_to_drop))
1866                     {
1867                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1868                       nat_elog_notice
1869                         ("maximum fragments per reassembly exceeded");
1870                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1871                       goto trace0;
1872                     }
1873                   cached0 = 1;
1874                   goto trace0;
1875                 }
1876               s0 = pool_elt_at_index (per_thread_data->sessions,
1877                                       reass0->sess_index);
1878             }
1879
1880           old_addr0 = ip0->dst_address.as_u32;
1881           ip0->dst_address = s0->in2out.addr;
1882           new_addr0 = ip0->dst_address.as_u32;
1883           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1884
1885           sum0 = ip0->checksum;
1886           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1887                                  ip4_header_t,
1888                                  dst_address /* changed member */ );
1889           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1890             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1891                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1892                                    src_address);
1893           ip0->checksum = ip_csum_fold (sum0);
1894
1895           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1896             {
1897               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1898                 {
1899                   old_port0 = tcp0->dst_port;
1900                   tcp0->dst_port = s0->in2out.port;
1901                   new_port0 = tcp0->dst_port;
1902
1903                   sum0 = tcp0->checksum;
1904                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1905                                          ip4_header_t,
1906                                          dst_address /* changed member */ );
1907
1908                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1909                                          ip4_header_t /* cheat */ ,
1910                                          length /* changed member */ );
1911                   if (is_twice_nat_session (s0))
1912                     {
1913                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1914                                              s0->ext_host_nat_addr.as_u32,
1915                                              ip4_header_t, dst_address);
1916                       sum0 = ip_csum_update (sum0, tcp0->src_port,
1917                                              s0->ext_host_nat_port,
1918                                              ip4_header_t, length);
1919                       tcp0->src_port = s0->ext_host_nat_port;
1920                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1921                     }
1922                   tcp0->checksum = ip_csum_fold (sum0);
1923                 }
1924               else
1925                 {
1926                   old_port0 = udp0->dst_port;
1927                   udp0->dst_port = s0->in2out.port;
1928                   if (is_twice_nat_session (s0))
1929                     {
1930                       udp0->src_port = s0->ext_host_nat_port;
1931                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1932                     }
1933                   udp0->checksum = 0;
1934                 }
1935             }
1936
1937           /* Accounting */
1938           nat44_session_update_counters (s0, now,
1939                                          vlib_buffer_length_in_chain (vm, b0),
1940                                          thread_index);
1941           /* Per-user LRU list maintenance */
1942           nat44_session_update_lru (sm, s0, thread_index);
1943
1944         trace0:
1945           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1946                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1947             {
1948               nat44_reass_trace_t *t =
1949                 vlib_add_trace (vm, node, b0, sizeof (*t));
1950               t->cached = cached0;
1951               t->sw_if_index = sw_if_index0;
1952               t->next_index = next0;
1953             }
1954
1955           if (cached0)
1956             {
1957               n_left_to_next++;
1958               to_next--;
1959             }
1960           else
1961             {
1962               pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1963
1964               /* verify speculative enqueue, maybe switch current next frame */
1965               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1966                                                to_next, n_left_to_next,
1967                                                bi0, next0);
1968             }
1969
1970           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1971             {
1972               from = vlib_frame_vector_args (frame);
1973               u32 len = vec_len (fragments_to_loopback);
1974               if (len <= VLIB_FRAME_SIZE)
1975                 {
1976                   clib_memcpy_fast (from, fragments_to_loopback,
1977                                     sizeof (u32) * len);
1978                   n_left_from = len;
1979                   vec_reset_length (fragments_to_loopback);
1980                 }
1981               else
1982                 {
1983                   clib_memcpy_fast (from, fragments_to_loopback +
1984                                     (len - VLIB_FRAME_SIZE),
1985                                     sizeof (u32) * VLIB_FRAME_SIZE);
1986                   n_left_from = VLIB_FRAME_SIZE;
1987                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1988                 }
1989             }
1990         }
1991
1992       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1993     }
1994
1995   vlib_node_increment_counter (vm, sm->ed_out2in_reass_node_index,
1996                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1997                                pkts_processed);
1998
1999   nat_send_all_to_node (vm, fragments_to_drop, node,
2000                         &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
2001                         NAT44_ED_OUT2IN_NEXT_DROP);
2002
2003   vec_free (fragments_to_drop);
2004   vec_free (fragments_to_loopback);
2005   return frame->n_vectors;
2006 }
2007
2008 /* *INDENT-OFF* */
2009 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
2010   .name = "nat44-ed-out2in-reass",
2011   .vector_size = sizeof (u32),
2012   .format_trace = format_nat44_reass_trace,
2013   .type = VLIB_NODE_TYPE_INTERNAL,
2014   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
2015   .error_strings = nat_out2in_ed_error_strings,
2016   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2017   .next_nodes = {
2018     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2019     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2020     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2021     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2022     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2023     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
2024   },
2025 };
2026 /* *INDENT-ON* */
2027
2028 /*
2029  * fd.io coding-style-patch-verification: ON
2030  *
2031  * Local Variables:
2032  * eval: (c-set-style "gnu")
2033  * End:
2034  */