src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_reass.h>
  31 #include <nat/nat_inlines.h>
  32 #include <nat/nat44_inlines.h>
  33 #include <nat/nat_syslog.h>
  34 #include <nat/nat_ha.h>
  35
  36 static char *nat_out2in_ed_error_strings[] = {
  37 #define _(sym,string) string,
  38   foreach_nat_out2in_ed_error
  39 #undef _
  40 };
  41
  42 typedef struct
  43 {
  44   u32 sw_if_index;
  45   u32 next_index;
  46   u32 session_index;
  47   u32 is_slow_path;
  48 } nat44_ed_out2in_trace_t;
  49
  50 static u8 *
  51 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  56   char *tag;
  57
  58   tag =
  59     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  60     "NAT44_OUT2IN_ED_FAST_PATH";
  61
  62   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  63               t->sw_if_index, t->next_index, t->session_index);
  64
  65   return s;
  66 }
  67
  68 static inline u32
  69 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
  70                           ip4_header_t * ip0, icmp46_header_t * icmp0,
  71                           u32 sw_if_index0, u32 rx_fib_index0,
  72                           vlib_node_runtime_t * node, u32 next0, f64 now,
  73                           u32 thread_index, snat_session_t ** p_s0)
  74 {
  75   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
  76                        next0, thread_index, p_s0, 0);
  77   snat_session_t *s0 = *p_s0;
  78   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
  79     {
  80       /* Accounting */
  81       nat44_session_update_counters (s0, now,
  82                                      vlib_buffer_length_in_chain
  83                                      (sm->vlib_main, b0), thread_index);
  84       /* Per-user LRU list maintenance */
  85       nat44_session_update_lru (sm, s0, thread_index);
  86     }
  87   return next0;
  88 }
  89
  90 #ifndef CLIB_MARCH_VARIANT
  91 int
  92 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  93 {
  94   snat_main_t *sm = &snat_main;
  95   nat44_is_idle_session_ctx_t *ctx = arg;
  96   snat_session_t *s;
  97   u64 sess_timeout_time;
  98   nat_ed_ses_key_t ed_key;
  99   clib_bihash_kv_16_8_t ed_kv;
 100   int i;
 101   snat_address_t *a;
 102   snat_session_key_t key;
 103   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 104                                                        ctx->thread_index);
 105
 106   s = pool_elt_at_index (tsm->sessions, kv->value);
 107   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 108   if (ctx->now >= sess_timeout_time)
 109     {
 110       ed_key.l_addr = s->in2out.addr;
 111       ed_key.r_addr = s->ext_host_addr;
 112       ed_key.fib_index = s->in2out.fib_index;
 113       if (snat_is_unk_proto_session (s))
 114         {
 115           ed_key.proto = s->in2out.port;
 116           ed_key.r_port = 0;
 117           ed_key.l_port = 0;
 118         }
 119       else
 120         {
 121           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 122           ed_key.l_port = s->in2out.port;
 123           ed_key.r_port = s->ext_host_port;
 124         }
 125       if (is_twice_nat_session (s))
 126         {
 127           ed_key.r_addr = s->ext_host_nat_addr;
 128           ed_key.r_port = s->ext_host_nat_port;
 129         }
 130       ed_kv.key[0] = ed_key.as_u64[0];
 131       ed_kv.key[1] = ed_key.as_u64[1];
 132       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 133         nat_elog_warn ("in2out_ed key del failed");
 134
 135       if (snat_is_unk_proto_session (s))
 136         goto delete;
 137
 138       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 139                                            s->in2out.addr.as_u32,
 140                                            s->out2in.addr.as_u32,
 141                                            s->in2out.protocol,
 142                                            s->in2out.port,
 143                                            s->out2in.port,
 144                                            s->in2out.fib_index);
 145
 146       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 147                              &s->in2out.addr, s->in2out.port,
 148                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 149                              &s->out2in.addr, s->out2in.port,
 150                              &s->ext_host_addr, s->ext_host_port,
 151                              s->in2out.protocol, is_twice_nat_session (s));
 152
 153       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 154                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 155                    ctx->thread_index);
 156
 157       if (is_twice_nat_session (s))
 158         {
 159           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 160             {
 161               key.protocol = s->in2out.protocol;
 162               key.port = s->ext_host_nat_port;
 163               a = sm->twice_nat_addresses + i;
 164               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 165                 {
 166                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 167                                                       ctx->thread_index,
 168                                                       &key);
 169                   break;
 170                 }
 171             }
 172         }
 173
 174       if (snat_is_session_static (s))
 175         goto delete;
 176
 177       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 178                                           &s->out2in);
 179     delete:
 180       nat44_delete_session (sm, s, ctx->thread_index);
 181       return 1;
 182     }
 183
 184   return 0;
 185 }
 186 #endif
 187
 188 static snat_session_t *
 189 create_session_for_static_mapping_ed (snat_main_t * sm,
 190                                       vlib_buffer_t * b,
 191                                       snat_session_key_t l_key,
 192                                       snat_session_key_t e_key,
 193                                       vlib_node_runtime_t * node,
 194                                       u32 thread_index,
 195                                       twice_nat_type_t twice_nat,
 196                                       lb_nat_type_t lb_nat, f64 now)
 197 {
 198   snat_session_t *s;
 199   snat_user_t *u;
 200   ip4_header_t *ip;
 201   udp_header_t *udp;
 202   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 203   clib_bihash_kv_16_8_t kv;
 204   snat_session_key_t eh_key;
 205   nat44_is_idle_session_ctx_t ctx;
 206
 207   nat44_session_try_cleanup (&l_key.addr, l_key.fib_index, thread_index, now);
 208
 209   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 210     {
 211       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 212       nat_elog_notice ("maximum sessions exceeded");
 213       return 0;
 214     }
 215
 216   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
 217   if (!u)
 218     {
 219       nat_elog_warn ("create NAT user failed");
 220       return 0;
 221     }
 222
 223   s = nat_ed_session_alloc (sm, u, thread_index, now);
 224   if (!s)
 225     {
 226       nat44_delete_user_with_no_session (sm, u, thread_index);
 227       nat_elog_warn ("create NAT session failed");
 228       return 0;
 229     }
 230
 231   ip = vlib_buffer_get_current (b);
 232   udp = ip4_next_header (ip);
 233
 234   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 235   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 236   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 237   if (lb_nat)
 238     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 239   if (lb_nat == AFFINITY_LB_NAT)
 240     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 241   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 242   s->out2in = e_key;
 243   s->in2out = l_key;
 244   s->in2out.protocol = s->out2in.protocol;
 245   user_session_increment (sm, u, 1);
 246
 247   /* Add to lookup tables */
 248   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
 249               e_key.fib_index, e_key.port, s->ext_host_port);
 250   kv.value = s - tsm->sessions;
 251   ctx.now = now;
 252   ctx.thread_index = thread_index;
 253   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
 254                                                nat44_o2i_ed_is_idle_session_cb,
 255                                                &ctx))
 256     nat_elog_notice ("out2in-ed key add failed");
 257
 258   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 259                                  ip->src_address.as_u32 == l_key.addr.as_u32))
 260     {
 261       eh_key.protocol = e_key.protocol;
 262       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 263                                                thread_index, &eh_key,
 264                                                sm->port_per_thread,
 265                                                tsm->snat_thread_index))
 266         {
 267           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 268           nat44_delete_session (sm, s, thread_index);
 269           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
 270             nat_elog_notice ("out2in-ed key del failed");
 271           return 0;
 272         }
 273       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 274       s->ext_host_nat_port = eh_key.port;
 275       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 276       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
 277                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
 278     }
 279   else
 280     {
 281       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
 282                   l_key.fib_index, l_key.port, s->ext_host_port);
 283     }
 284   kv.value = s - tsm->sessions;
 285   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 286                                                nat44_i2o_ed_is_idle_session_cb,
 287                                                &ctx))
 288     nat_elog_notice ("in2out-ed key add failed");
 289
 290   snat_ipfix_logging_nat44_ses_create (thread_index,
 291                                        s->in2out.addr.as_u32,
 292                                        s->out2in.addr.as_u32,
 293                                        s->in2out.protocol,
 294                                        s->in2out.port,
 295                                        s->out2in.port, s->in2out.fib_index);
 296
 297   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 298                          &s->in2out.addr, s->in2out.port,
 299                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 300                          &s->out2in.addr, s->out2in.port,
 301                          &s->ext_host_addr, s->ext_host_port,
 302                          s->in2out.protocol, is_twice_nat_session (s));
 303
 304   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 305                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 306                &s->ext_host_nat_addr, s->ext_host_nat_port,
 307                s->in2out.protocol, s->in2out.fib_index, s->flags,
 308                thread_index, 0);
 309
 310   return s;
 311 }
 312
 313 static int
 314 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
 315               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 316 {
 317   clib_bihash_kv_16_8_t kv, value;
 318   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 319
 320   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
 321               rx_fib_index, src_port, dst_port);
 322   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 323     return 1;
 324
 325   return 0;
 326 }
 327
 328 static void
 329 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 330                        u32 thread_index)
 331 {
 332   nat_ed_ses_key_t key;
 333   clib_bihash_kv_16_8_t kv, value;
 334   udp_header_t *udp;
 335   snat_user_t *u;
 336   snat_session_t *s = 0;
 337   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 338   f64 now = vlib_time_now (sm->vlib_main);
 339
 340   if (ip->protocol == IP_PROTOCOL_ICMP)
 341     {
 342       if (get_icmp_o2i_ed_key (ip, &key))
 343         return;
 344     }
 345   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 346     {
 347       udp = ip4_next_header (ip);
 348       key.r_addr = ip->src_address;
 349       key.l_addr = ip->dst_address;
 350       key.proto = ip->protocol;
 351       key.l_port = udp->dst_port;
 352       key.r_port = udp->src_port;
 353     }
 354   else
 355     {
 356       key.r_addr = ip->src_address;
 357       key.l_addr = ip->dst_address;
 358       key.proto = ip->protocol;
 359       key.l_port = key.r_port = 0;
 360     }
 361   key.fib_index = 0;
 362   kv.key[0] = key.as_u64[0];
 363   kv.key[1] = key.as_u64[1];
 364
 365   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 366     {
 367       s = pool_elt_at_index (tsm->sessions, value.value);
 368     }
 369   else
 370     {
 371       u32 proto;
 372
 373       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 374         return;
 375
 376       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
 377                                   thread_index);
 378       if (!u)
 379         {
 380           nat_elog_warn ("create NAT user failed");
 381           return;
 382         }
 383
 384       s = nat_ed_session_alloc (sm, u, thread_index, now);
 385       if (!s)
 386         {
 387           nat44_delete_user_with_no_session (sm, u, thread_index);
 388           nat_elog_warn ("create NAT session failed");
 389           return;
 390         }
 391
 392       proto = ip_proto_to_snat_proto (key.proto);
 393
 394       s->ext_host_addr = key.r_addr;
 395       s->ext_host_port = key.r_port;
 396       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 397       s->out2in.addr = key.l_addr;
 398       s->out2in.port = key.l_port;
 399       s->out2in.protocol = proto;
 400       if (proto == ~0)
 401         {
 402           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 403           s->out2in.port = ip->protocol;
 404         }
 405       s->out2in.fib_index = 0;
 406       s->in2out = s->out2in;
 407       user_session_increment (sm, u, 0);
 408
 409       kv.value = s - tsm->sessions;
 410       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 411         nat_elog_notice ("in2out_ed key add failed");
 412     }
 413
 414   if (ip->protocol == IP_PROTOCOL_TCP)
 415     {
 416       tcp_header_t *tcp = ip4_next_header (ip);
 417       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
 418         return;
 419     }
 420
 421   /* Accounting */
 422   nat44_session_update_counters (s, now, 0, thread_index);
 423   /* Per-user LRU list maintenance */
 424   nat44_session_update_lru (sm, s, thread_index);
 425 }
 426
 427 static inline void
 428 create_bypass_for_fwd_worker (snat_main_t * sm, ip4_header_t * ip,
 429                               u32 rx_fib_index)
 430 {
 431   ip4_header_t ip_wkr = {
 432     .src_address = ip->dst_address,
 433   };
 434   u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
 435
 436   create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
 437 }
 438
 439 #ifndef CLIB_MARCH_VARIANT
 440 u32
 441 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 442                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 443                       u8 * p_proto, snat_session_key_t * p_value,
 444                       u8 * p_dont_translate, void *d, void *e)
 445 {
 446   u32 next = ~0, sw_if_index, rx_fib_index;
 447   icmp46_header_t *icmp;
 448   nat_ed_ses_key_t key;
 449   clib_bihash_kv_16_8_t kv, value;
 450   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 451   snat_session_t *s = 0;
 452   u8 dont_translate = 0, is_addr_only, identity_nat;
 453   snat_session_key_t e_key, l_key;
 454
 455   icmp = (icmp46_header_t *) ip4_next_header (ip);
 456   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 457   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 458
 459   if (get_icmp_o2i_ed_key (ip, &key))
 460     {
 461       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 462       next = NAT_NEXT_DROP;
 463       goto out;
 464     }
 465   key.fib_index = rx_fib_index;
 466   kv.key[0] = key.as_u64[0];
 467   kv.key[1] = key.as_u64[1];
 468
 469   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 470     {
 471       /* Try to match static mapping */
 472       e_key.addr = ip->dst_address;
 473       e_key.port = key.l_port;
 474       e_key.protocol = ip_proto_to_snat_proto (key.proto);
 475       e_key.fib_index = rx_fib_index;
 476       if (snat_static_mapping_match
 477           (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
 478         {
 479           if (!sm->forwarding_enabled)
 480             {
 481               /* Don't NAT packet aimed at the intfc address */
 482               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 483                                                     ip->dst_address.as_u32)))
 484                 {
 485                   dont_translate = 1;
 486                   goto out;
 487                 }
 488               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 489               next = NAT_NEXT_DROP;
 490               goto out;
 491             }
 492           else
 493             {
 494               dont_translate = 1;
 495               if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
 496                                 thread_index, rx_fib_index))
 497                 {
 498                   next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 499                   goto out;
 500                 }
 501               if (sm->num_workers > 1)
 502                 create_bypass_for_fwd_worker (sm, ip, rx_fib_index);
 503               else
 504                 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
 505               goto out;
 506             }
 507         }
 508
 509       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 510                          (icmp->type != ICMP4_echo_request || !is_addr_only)))
 511         {
 512           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 513           next = NAT_NEXT_DROP;
 514           goto out;
 515         }
 516
 517       if (PREDICT_FALSE (identity_nat))
 518         {
 519           dont_translate = 1;
 520           goto out;
 521         }
 522
 523       /* Create session initiated by host from external network */
 524       s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
 525                                                 thread_index, 0, 0,
 526                                                 vlib_time_now
 527                                                 (sm->vlib_main));
 528
 529       if (!s)
 530         {
 531           next = NAT_NEXT_DROP;
 532           goto out;
 533         }
 534     }
 535   else
 536     {
 537       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 538                          icmp->type != ICMP4_echo_request &&
 539                          !icmp_is_error_message (icmp)))
 540         {
 541           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 542           next = NAT_NEXT_DROP;
 543           goto out;
 544         }
 545
 546       s = pool_elt_at_index (tsm->sessions, value.value);
 547     }
 548
 549   *p_proto = ip_proto_to_snat_proto (key.proto);
 550 out:
 551   if (s)
 552     *p_value = s->in2out;
 553   *p_dont_translate = dont_translate;
 554   if (d)
 555     *(snat_session_t **) d = s;
 556   return next;
 557 }
 558 #endif
 559
 560 static snat_session_t *
 561 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 562                                vlib_buffer_t * b,
 563                                ip4_header_t * ip,
 564                                u32 rx_fib_index,
 565                                u32 thread_index,
 566                                f64 now,
 567                                vlib_main_t * vm, vlib_node_runtime_t * node)
 568 {
 569   clib_bihash_kv_8_8_t kv, value;
 570   clib_bihash_kv_16_8_t s_kv, s_value;
 571   snat_static_mapping_t *m;
 572   u32 old_addr, new_addr;
 573   ip_csum_t sum;
 574   snat_session_t *s;
 575   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 576   snat_user_t *u;
 577
 578   old_addr = ip->dst_address.as_u32;
 579
 580   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 581               rx_fib_index, 0, 0);
 582
 583   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 584     {
 585       s = pool_elt_at_index (tsm->sessions, s_value.value);
 586       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 587     }
 588   else
 589     {
 590       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 591         {
 592           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 593           nat_elog_notice ("maximum sessions exceeded");
 594           return 0;
 595         }
 596
 597       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 598       if (clib_bihash_search_8_8
 599           (&sm->static_mapping_by_external, &kv, &value))
 600         {
 601           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 602           return 0;
 603         }
 604
 605       m = pool_elt_at_index (sm->static_mappings, value.value);
 606
 607       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 608
 609       u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
 610                                   thread_index);
 611       if (!u)
 612         {
 613           nat_elog_warn ("create NAT user failed");
 614           return 0;
 615         }
 616
 617       /* Create a new session */
 618       s = nat_ed_session_alloc (sm, u, thread_index, now);
 619       if (!s)
 620         {
 621           nat44_delete_user_with_no_session (sm, u, thread_index);
 622           nat_elog_warn ("create NAT session failed");
 623           return 0;
 624         }
 625
 626       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 627       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 628       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 629       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 630       s->out2in.addr.as_u32 = old_addr;
 631       s->out2in.fib_index = rx_fib_index;
 632       s->in2out.addr.as_u32 = new_addr;
 633       s->in2out.fib_index = m->fib_index;
 634       s->in2out.port = s->out2in.port = ip->protocol;
 635       user_session_increment (sm, u, 1);
 636
 637       /* Add to lookup tables */
 638       s_kv.value = s - tsm->sessions;
 639       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 640         nat_elog_notice ("out2in key add failed");
 641
 642       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 643                   m->fib_index, 0, 0);
 644       s_kv.value = s - tsm->sessions;
 645       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 646         nat_elog_notice ("in2out key add failed");
 647     }
 648
 649   /* Update IP checksum */
 650   sum = ip->checksum;
 651   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 652   ip->checksum = ip_csum_fold (sum);
 653
 654   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 655
 656   /* Accounting */
 657   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 658                                  thread_index);
 659   /* Per-user LRU list maintenance */
 660   nat44_session_update_lru (sm, s, thread_index);
 661
 662   return s;
 663 }
 664
 665 static inline uword
 666 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
 667                                 vlib_node_runtime_t * node,
 668                                 vlib_frame_t * frame, int is_slow_path)
 669 {
 670   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 671   nat_next_t next_index;
 672   snat_main_t *sm = &snat_main;
 673   f64 now = vlib_time_now (vm);
 674   u32 thread_index = vm->thread_index;
 675   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 676   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 677     0, fragments = 0;
 678
 679   stats_node_index = is_slow_path ? sm->ed_out2in_slowpath_node_index :
 680     sm->ed_out2in_node_index;
 681
 682   from = vlib_frame_vector_args (frame);
 683   n_left_from = frame->n_vectors;
 684   next_index = node->cached_next_index;
 685
 686   while (n_left_from > 0)
 687     {
 688       u32 n_left_to_next;
 689
 690       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 691
 692       while (n_left_from >= 4 && n_left_to_next >= 2)
 693         {
 694           u32 bi0, bi1;
 695           vlib_buffer_t *b0, *b1;
 696           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 697             new_addr0;
 698           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
 699             new_addr1;
 700           u16 old_port0, new_port0, old_port1, new_port1;
 701           ip4_header_t *ip0, *ip1;
 702           udp_header_t *udp0, *udp1;
 703           tcp_header_t *tcp0, *tcp1;
 704           icmp46_header_t *icmp0, *icmp1;
 705           snat_session_t *s0 = 0, *s1 = 0;
 706           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 707           ip_csum_t sum0, sum1;
 708           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
 709           lb_nat_type_t lb_nat0, lb_nat1;
 710           twice_nat_type_t twice_nat0, twice_nat1;
 711           u8 identity_nat0, identity_nat1;
 712
 713           /* Prefetch next iteration. */
 714           {
 715             vlib_buffer_t *p2, *p3;
 716
 717             p2 = vlib_get_buffer (vm, from[2]);
 718             p3 = vlib_get_buffer (vm, from[3]);
 719
 720             vlib_prefetch_buffer_header (p2, LOAD);
 721             vlib_prefetch_buffer_header (p3, LOAD);
 722
 723             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 724             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 725           }
 726
 727           /* speculatively enqueue b0 and b1 to the current next frame */
 728           to_next[0] = bi0 = from[0];
 729           to_next[1] = bi1 = from[1];
 730           from += 2;
 731           to_next += 2;
 732           n_left_from -= 2;
 733           n_left_to_next -= 2;
 734
 735           b0 = vlib_get_buffer (vm, bi0);
 736           b1 = vlib_get_buffer (vm, bi1);
 737
 738           next0 = nat_buffer_opaque (b0)->arc_next;
 739           next1 = nat_buffer_opaque (b1)->arc_next;
 740
 741           vnet_buffer (b0)->snat.flags = 0;
 742           ip0 = vlib_buffer_get_current (b0);
 743
 744           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 745           rx_fib_index0 =
 746             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 747                                                  sw_if_index0);
 748
 749           if (PREDICT_FALSE (ip0->ttl == 1))
 750             {
 751               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 752               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 753                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 754                                            0);
 755               next0 = NAT_NEXT_ICMP_ERROR;
 756               goto trace00;
 757             }
 758
 759           udp0 = ip4_next_header (ip0);
 760           tcp0 = (tcp_header_t *) udp0;
 761           icmp0 = (icmp46_header_t *) udp0;
 762           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 763
 764           if (is_slow_path)
 765             {
 766               if (PREDICT_FALSE (proto0 == ~0))
 767                 {
 768                   s0 =
 769                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
 770                                                    thread_index, now, vm,
 771                                                    node);
 772                   other_packets++;
 773                   if (!sm->forwarding_enabled)
 774                     {
 775                       if (!s0)
 776                         next0 = NAT_NEXT_DROP;
 777                       goto trace00;
 778                     }
 779                 }
 780
 781               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 782                 {
 783                   next0 = icmp_out2in_ed_slow_path
 784                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 785                      next0, now, thread_index, &s0);
 786                   icmp_packets++;
 787                   goto trace00;
 788                 }
 789             }
 790           else
 791             {
 792               if (PREDICT_FALSE (proto0 == ~0))
 793                 {
 794                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 795                   goto trace00;
 796                 }
 797
 798               if (ip4_is_fragment (ip0))
 799                 {
 800                   next0 = NAT_NEXT_OUT2IN_ED_REASS;
 801                   fragments++;
 802                   goto trace00;
 803                 }
 804
 805               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 806                 {
 807                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 808                   goto trace00;
 809                 }
 810             }
 811
 812           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
 813                       ip0->protocol, rx_fib_index0, udp0->dst_port,
 814                       udp0->src_port);
 815
 816           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
 817             {
 818               if (is_slow_path)
 819                 {
 820                   /* Try to match static mapping by external address and port,
 821                      destination address and port in packet */
 822                   e_key0.addr = ip0->dst_address;
 823                   e_key0.port = udp0->dst_port;
 824                   e_key0.protocol = proto0;
 825                   e_key0.fib_index = rx_fib_index0;
 826                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
 827                                                  &twice_nat0, &lb_nat0,
 828                                                  &ip0->src_address,
 829                                                  &identity_nat0))
 830                     {
 831                       /*
 832                        * Send DHCP packets to the ipv4 stack, or we won't
 833                        * be able to use dhcp client on the outside interface
 834                        */
 835                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 836                                          && (udp0->dst_port ==
 837                                              clib_host_to_net_u16
 838                                              (UDP_DST_PORT_dhcp_to_client))))
 839                         {
 840                           goto trace00;
 841                         }
 842
 843                       if (!sm->forwarding_enabled)
 844                         {
 845                           b0->error =
 846                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 847                           next0 = NAT_NEXT_DROP;
 848                         }
 849                       else
 850                         {
 851                           if (next_src_nat (sm, ip0, ip0->protocol,
 852                                             udp0->src_port, udp0->dst_port,
 853                                             thread_index, rx_fib_index0))
 854                             {
 855                               next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 856                               goto trace00;
 857                             }
 858                           if (sm->num_workers > 1)
 859                             create_bypass_for_fwd_worker (sm, ip0,
 860                                                           rx_fib_index0);
 861                           else
 862                             create_bypass_for_fwd (sm, ip0, rx_fib_index0,
 863                                                    thread_index);
 864                         }
 865                       goto trace00;
 866                     }
 867
 868                   if (PREDICT_FALSE (identity_nat0))
 869                     goto trace00;
 870
 871                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
 872                     {
 873                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
 874                       next0 = NAT_NEXT_DROP;
 875                       goto trace00;
 876                     }
 877
 878                   /* Create session initiated by host from external network */
 879                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
 880                                                              e_key0, node,
 881                                                              thread_index,
 882                                                              twice_nat0,
 883                                                              lb_nat0, now);
 884
 885                   if (!s0)
 886                     {
 887                       next0 = NAT_NEXT_DROP;
 888                       goto trace00;
 889                     }
 890                 }
 891               else
 892                 {
 893                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 894                   goto trace00;
 895                 }
 896             }
 897           else
 898             {
 899               s0 = pool_elt_at_index (tsm->sessions, value0.value);
 900             }
 901
 902           old_addr0 = ip0->dst_address.as_u32;
 903           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 904           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 905
 906           sum0 = ip0->checksum;
 907           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 908                                  dst_address);
 909           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 910             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 911                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
 912                                    src_address);
 913           ip0->checksum = ip_csum_fold (sum0);
 914
 915           old_port0 = udp0->dst_port;
 916           new_port0 = udp0->dst_port = s0->in2out.port;
 917
 918           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 919             {
 920               sum0 = tcp0->checksum;
 921               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 922                                      dst_address);
 923               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 924                                      length);
 925               if (is_twice_nat_session (s0))
 926                 {
 927                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 928                                          s0->ext_host_nat_addr.as_u32,
 929                                          ip4_header_t, dst_address);
 930                   sum0 = ip_csum_update (sum0, tcp0->src_port,
 931                                          s0->ext_host_nat_port, ip4_header_t,
 932                                          length);
 933                   tcp0->src_port = s0->ext_host_nat_port;
 934                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 935                 }
 936               tcp0->checksum = ip_csum_fold (sum0);
 937               tcp_packets++;
 938               if (nat44_set_tcp_session_state_o2i
 939                   (sm, s0, tcp0, thread_index))
 940                 goto trace00;
 941             }
 942           else if (udp0->checksum)
 943             {
 944               sum0 = udp0->checksum;
 945               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 946                                      dst_address);
 947               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 948                                      length);
 949               if (PREDICT_FALSE (is_twice_nat_session (s0)))
 950                 {
 951                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 952                                          s0->ext_host_nat_addr.as_u32,
 953                                          ip4_header_t, dst_address);
 954                   sum0 = ip_csum_update (sum0, udp0->src_port,
 955                                          s0->ext_host_nat_port, ip4_header_t,
 956                                          length);
 957                   udp0->src_port = s0->ext_host_nat_port;
 958                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 959                 }
 960               udp0->checksum = ip_csum_fold (sum0);
 961               udp_packets++;
 962             }
 963           else
 964             {
 965               if (PREDICT_FALSE (is_twice_nat_session (s0)))
 966                 {
 967                   udp0->src_port = s0->ext_host_nat_port;
 968                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 969                 }
 970               udp_packets++;
 971             }
 972
 973           /* Accounting */
 974           nat44_session_update_counters (s0, now,
 975                                          vlib_buffer_length_in_chain (vm, b0),
 976                                          thread_index);
 977           /* Per-user LRU list maintenance */
 978           nat44_session_update_lru (sm, s0, thread_index);
 979
 980         trace00:
 981           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 982                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 983             {
 984               nat44_ed_out2in_trace_t *t =
 985                 vlib_add_trace (vm, node, b0, sizeof (*t));
 986               t->is_slow_path = is_slow_path;
 987               t->sw_if_index = sw_if_index0;
 988               t->next_index = next0;
 989               t->session_index = ~0;
 990               if (s0)
 991                 t->session_index = s0 - tsm->sessions;
 992             }
 993
 994           pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
 995
 996           vnet_buffer (b1)->snat.flags = 0;
 997           ip1 = vlib_buffer_get_current (b1);
 998
 999           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1000           rx_fib_index1 =
1001             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1002                                                  sw_if_index1);
1003
1004           if (PREDICT_FALSE (ip1->ttl == 1))
1005             {
1006               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1007               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1008                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1009                                            0);
1010               next1 = NAT_NEXT_ICMP_ERROR;
1011               goto trace01;
1012             }
1013
1014           udp1 = ip4_next_header (ip1);
1015           tcp1 = (tcp_header_t *) udp1;
1016           icmp1 = (icmp46_header_t *) udp1;
1017           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1018
1019           if (is_slow_path)
1020             {
1021               if (PREDICT_FALSE (proto1 == ~0))
1022                 {
1023                   s1 =
1024                     nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1025                                                    thread_index, now, vm,
1026                                                    node);
1027                   other_packets++;
1028                   if (!sm->forwarding_enabled)
1029                     {
1030                       if (!s1)
1031                         next1 = NAT_NEXT_DROP;
1032                       goto trace01;
1033                     }
1034                 }
1035
1036               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1037                 {
1038                   next1 = icmp_out2in_ed_slow_path
1039                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1040                      next1, now, thread_index, &s1);
1041                   icmp_packets++;
1042                   goto trace01;
1043                 }
1044             }
1045           else
1046             {
1047               if (PREDICT_FALSE (proto1 == ~0))
1048                 {
1049                   next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1050                   goto trace01;
1051                 }
1052
1053               if (ip4_is_fragment (ip1))
1054                 {
1055                   next1 = NAT_NEXT_OUT2IN_ED_REASS;
1056                   fragments++;
1057                   goto trace01;
1058                 }
1059
1060               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1061                 {
1062                   next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1063                   goto trace01;
1064                 }
1065             }
1066
1067           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1068                       ip1->protocol, rx_fib_index1, udp1->dst_port,
1069                       udp1->src_port);
1070
1071           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1072             {
1073               if (is_slow_path)
1074                 {
1075                   /* Try to match static mapping by external address and port,
1076                      destination address and port in packet */
1077                   e_key1.addr = ip1->dst_address;
1078                   e_key1.port = udp1->dst_port;
1079                   e_key1.protocol = proto1;
1080                   e_key1.fib_index = rx_fib_index1;
1081                   if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1082                                                  &twice_nat1, &lb_nat1,
1083                                                  &ip1->src_address,
1084                                                  &identity_nat1))
1085                     {
1086                       /*
1087                        * Send DHCP packets to the ipv4 stack, or we won't
1088                        * be able to use dhcp client on the outside interface
1089                        */
1090                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1091                                          && (udp1->dst_port ==
1092                                              clib_host_to_net_u16
1093                                              (UDP_DST_PORT_dhcp_to_client))))
1094                         {
1095                           goto trace01;
1096                         }
1097
1098                       if (!sm->forwarding_enabled)
1099                         {
1100                           b1->error =
1101                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1102                           next1 = NAT_NEXT_DROP;
1103                         }
1104                       else
1105                         {
1106                           if (next_src_nat (sm, ip1, ip1->protocol,
1107                                             udp1->src_port, udp1->dst_port,
1108                                             thread_index, rx_fib_index1))
1109                             {
1110                               next1 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1111                               goto trace01;
1112                             }
1113                           if (sm->num_workers > 1)
1114                             create_bypass_for_fwd_worker (sm, ip1,
1115                                                           rx_fib_index1);
1116                           else
1117                             create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1118                                                    thread_index);
1119                         }
1120                       goto trace01;
1121                     }
1122
1123                   if (PREDICT_FALSE (identity_nat1))
1124                     goto trace01;
1125
1126                   if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1127                     {
1128                       b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1129                       next1 = NAT_NEXT_DROP;
1130                       goto trace01;
1131                     }
1132
1133                   /* Create session initiated by host from external network */
1134                   s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1135                                                              e_key1, node,
1136                                                              thread_index,
1137                                                              twice_nat1,
1138                                                              lb_nat1, now);
1139
1140                   if (!s1)
1141                     {
1142                       next1 = NAT_NEXT_DROP;
1143                       goto trace01;
1144                     }
1145                 }
1146               else
1147                 {
1148                   next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1149                   goto trace01;
1150                 }
1151             }
1152           else
1153             {
1154               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1155             }
1156
1157           old_addr1 = ip1->dst_address.as_u32;
1158           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1159           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1160
1161           sum1 = ip1->checksum;
1162           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1163                                  dst_address);
1164           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1165             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1166                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
1167                                    src_address);
1168           ip1->checksum = ip_csum_fold (sum1);
1169
1170           old_port1 = udp1->dst_port;
1171           new_port1 = udp1->dst_port = s1->in2out.port;
1172
1173           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1174             {
1175               sum1 = tcp1->checksum;
1176               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1177                                      dst_address);
1178               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1179                                      length);
1180               if (is_twice_nat_session (s1))
1181                 {
1182                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1183                                          s1->ext_host_nat_addr.as_u32,
1184                                          ip4_header_t, dst_address);
1185                   sum1 = ip_csum_update (sum1, tcp1->src_port,
1186                                          s1->ext_host_nat_port, ip4_header_t,
1187                                          length);
1188                   tcp1->src_port = s1->ext_host_nat_port;
1189                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1190                 }
1191               tcp1->checksum = ip_csum_fold (sum1);
1192               tcp_packets++;
1193               if (nat44_set_tcp_session_state_o2i
1194                   (sm, s1, tcp1, thread_index))
1195                 goto trace01;
1196             }
1197           else if (udp1->checksum)
1198             {
1199               sum1 = udp1->checksum;
1200               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1201                                      dst_address);
1202               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1203                                      length);
1204               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1205                 {
1206                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1207                                          s1->ext_host_nat_addr.as_u32,
1208                                          ip4_header_t, dst_address);
1209                   sum1 = ip_csum_update (sum1, udp1->src_port,
1210                                          s1->ext_host_nat_port, ip4_header_t,
1211                                          length);
1212                   udp1->src_port = s1->ext_host_nat_port;
1213                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1214                 }
1215               udp1->checksum = ip_csum_fold (sum1);
1216               udp_packets++;
1217             }
1218           else
1219             {
1220               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1221                 {
1222                   udp1->src_port = s1->ext_host_nat_port;
1223                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1224                 }
1225               udp_packets++;
1226             }
1227
1228           /* Accounting */
1229           nat44_session_update_counters (s1, now,
1230                                          vlib_buffer_length_in_chain (vm, b1),
1231                                          thread_index);
1232           /* Per-user LRU list maintenance */
1233           nat44_session_update_lru (sm, s1, thread_index);
1234
1235         trace01:
1236           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1237                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1238             {
1239               nat44_ed_out2in_trace_t *t =
1240                 vlib_add_trace (vm, node, b1, sizeof (*t));
1241               t->is_slow_path = is_slow_path;
1242               t->sw_if_index = sw_if_index1;
1243               t->next_index = next1;
1244               t->session_index = ~0;
1245               if (s1)
1246                 t->session_index = s1 - tsm->sessions;
1247             }
1248
1249           pkts_processed += next1 == nat_buffer_opaque (b1)->arc_next;
1250
1251           /* verify speculative enqueues, maybe switch current next frame */
1252           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1253                                            to_next, n_left_to_next,
1254                                            bi0, bi1, next0, next1);
1255         }
1256
1257       while (n_left_from > 0 && n_left_to_next > 0)
1258         {
1259           u32 bi0;
1260           vlib_buffer_t *b0;
1261           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1262             new_addr0;
1263           u16 old_port0, new_port0;
1264           ip4_header_t *ip0;
1265           udp_header_t *udp0;
1266           tcp_header_t *tcp0;
1267           icmp46_header_t *icmp0;
1268           snat_session_t *s0 = 0;
1269           clib_bihash_kv_16_8_t kv0, value0;
1270           ip_csum_t sum0;
1271           snat_session_key_t e_key0, l_key0;
1272           lb_nat_type_t lb_nat0;
1273           twice_nat_type_t twice_nat0;
1274           u8 identity_nat0;
1275
1276           /* speculatively enqueue b0 to the current next frame */
1277           bi0 = from[0];
1278           to_next[0] = bi0;
1279           from += 1;
1280           to_next += 1;
1281           n_left_from -= 1;
1282           n_left_to_next -= 1;
1283
1284           b0 = vlib_get_buffer (vm, bi0);
1285           next0 = nat_buffer_opaque (b0)->arc_next;
1286
1287           vnet_buffer (b0)->snat.flags = 0;
1288           ip0 = vlib_buffer_get_current (b0);
1289
1290           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1291           rx_fib_index0 =
1292             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1293                                                  sw_if_index0);
1294
1295           if (PREDICT_FALSE (ip0->ttl == 1))
1296             {
1297               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1298               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1299                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1300                                            0);
1301               next0 = NAT_NEXT_ICMP_ERROR;
1302               goto trace0;
1303             }
1304
1305           udp0 = ip4_next_header (ip0);
1306           tcp0 = (tcp_header_t *) udp0;
1307           icmp0 = (icmp46_header_t *) udp0;
1308           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1309
1310           if (is_slow_path)
1311             {
1312               if (PREDICT_FALSE (proto0 == ~0))
1313                 {
1314                   s0 =
1315                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1316                                                    thread_index, now, vm,
1317                                                    node);
1318                   other_packets++;
1319                   if (!sm->forwarding_enabled)
1320                     {
1321                       if (!s0)
1322                         next0 = NAT_NEXT_DROP;
1323                       goto trace0;
1324                     }
1325                 }
1326
1327               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1328                 {
1329                   next0 = icmp_out2in_ed_slow_path
1330                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1331                      next0, now, thread_index, &s0);
1332                   icmp_packets++;
1333                   goto trace0;
1334                 }
1335             }
1336           else
1337             {
1338               if (PREDICT_FALSE (proto0 == ~0))
1339                 {
1340                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1341                   goto trace0;
1342                 }
1343
1344               if (ip4_is_fragment (ip0))
1345                 {
1346                   next0 = NAT_NEXT_OUT2IN_ED_REASS;
1347                   fragments++;
1348                   goto trace0;
1349                 }
1350
1351               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1352                 {
1353                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1354                   goto trace0;
1355                 }
1356             }
1357
1358           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1359                       ip0->protocol, rx_fib_index0, udp0->dst_port,
1360                       udp0->src_port);
1361
1362           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1363             {
1364               if (is_slow_path)
1365                 {
1366                   /* Try to match static mapping by external address and port,
1367                      destination address and port in packet */
1368                   e_key0.addr = ip0->dst_address;
1369                   e_key0.port = udp0->dst_port;
1370                   e_key0.protocol = proto0;
1371                   e_key0.fib_index = rx_fib_index0;
1372                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1373                                                  &twice_nat0, &lb_nat0,
1374                                                  &ip0->src_address,
1375                                                  &identity_nat0))
1376                     {
1377                       /*
1378                        * Send DHCP packets to the ipv4 stack, or we won't
1379                        * be able to use dhcp client on the outside interface
1380                        */
1381                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1382                                          && (udp0->dst_port ==
1383                                              clib_host_to_net_u16
1384                                              (UDP_DST_PORT_dhcp_to_client))))
1385                         {
1386                           goto trace0;
1387                         }
1388
1389                       if (!sm->forwarding_enabled)
1390                         {
1391                           b0->error =
1392                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1393                           next0 = NAT_NEXT_DROP;
1394                         }
1395                       else
1396                         {
1397                           if (next_src_nat (sm, ip0, ip0->protocol,
1398                                             udp0->src_port, udp0->dst_port,
1399                                             thread_index, rx_fib_index0))
1400                             {
1401                               next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1402                               goto trace0;
1403                             }
1404                           if (sm->num_workers > 1)
1405                             create_bypass_for_fwd_worker (sm, ip0,
1406                                                           rx_fib_index0);
1407                           else
1408                             create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1409                                                    thread_index);
1410                         }
1411                       goto trace0;
1412                     }
1413
1414                   if (PREDICT_FALSE (identity_nat0))
1415                     goto trace0;
1416
1417                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1418                     {
1419                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1420                       next0 = NAT_NEXT_DROP;
1421                       goto trace0;
1422                     }
1423
1424                   /* Create session initiated by host from external network */
1425                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1426                                                              e_key0, node,
1427                                                              thread_index,
1428                                                              twice_nat0,
1429                                                              lb_nat0, now);
1430
1431                   if (!s0)
1432                     {
1433                       next0 = NAT_NEXT_DROP;
1434                       goto trace0;
1435                     }
1436                 }
1437               else
1438                 {
1439                   next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1440                   goto trace0;
1441                 }
1442             }
1443           else
1444             {
1445               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1446             }
1447
1448           old_addr0 = ip0->dst_address.as_u32;
1449           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1450           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1451
1452           sum0 = ip0->checksum;
1453           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1454                                  dst_address);
1455           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1456             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1457                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1458                                    src_address);
1459           ip0->checksum = ip_csum_fold (sum0);
1460
1461           old_port0 = udp0->dst_port;
1462           new_port0 = udp0->dst_port = s0->in2out.port;
1463
1464           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1465             {
1466               sum0 = tcp0->checksum;
1467               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1468                                      dst_address);
1469               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1470                                      length);
1471               if (is_twice_nat_session (s0))
1472                 {
1473                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1474                                          s0->ext_host_nat_addr.as_u32,
1475                                          ip4_header_t, dst_address);
1476                   sum0 = ip_csum_update (sum0, tcp0->src_port,
1477                                          s0->ext_host_nat_port, ip4_header_t,
1478                                          length);
1479                   tcp0->src_port = s0->ext_host_nat_port;
1480                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1481                 }
1482               tcp0->checksum = ip_csum_fold (sum0);
1483               tcp_packets++;
1484               if (nat44_set_tcp_session_state_o2i
1485                   (sm, s0, tcp0, thread_index))
1486                 goto trace0;
1487             }
1488           else if (udp0->checksum)
1489             {
1490               sum0 = udp0->checksum;
1491               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1492                                      dst_address);
1493               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1494                                      length);
1495               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1496                 {
1497                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1498                                          s0->ext_host_nat_addr.as_u32,
1499                                          ip4_header_t, dst_address);
1500                   sum0 = ip_csum_update (sum0, udp0->src_port,
1501                                          s0->ext_host_nat_port, ip4_header_t,
1502                                          length);
1503                   udp0->src_port = s0->ext_host_nat_port;
1504                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1505                 }
1506               udp0->checksum = ip_csum_fold (sum0);
1507               udp_packets++;
1508             }
1509           else
1510             {
1511               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1512                 {
1513                   udp0->src_port = s0->ext_host_nat_port;
1514                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1515                 }
1516               udp_packets++;
1517             }
1518
1519           /* Accounting */
1520           nat44_session_update_counters (s0, now,
1521                                          vlib_buffer_length_in_chain (vm, b0),
1522                                          thread_index);
1523           /* Per-user LRU list maintenance */
1524           nat44_session_update_lru (sm, s0, thread_index);
1525
1526         trace0:
1527           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1528                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1529             {
1530               nat44_ed_out2in_trace_t *t =
1531                 vlib_add_trace (vm, node, b0, sizeof (*t));
1532               t->is_slow_path = is_slow_path;
1533               t->sw_if_index = sw_if_index0;
1534               t->next_index = next0;
1535               t->session_index = ~0;
1536               if (s0)
1537                 t->session_index = s0 - tsm->sessions;
1538             }
1539
1540           pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1541           /* verify speculative enqueue, maybe switch current next frame */
1542           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1543                                            to_next, n_left_to_next,
1544                                            bi0, next0);
1545         }
1546
1547       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1548     }
1549
1550   vlib_node_increment_counter (vm, stats_node_index,
1551                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1552                                pkts_processed);
1553   vlib_node_increment_counter (vm, stats_node_index,
1554                                NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1555   vlib_node_increment_counter (vm, stats_node_index,
1556                                NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1557   vlib_node_increment_counter (vm, stats_node_index,
1558                                NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1559                                icmp_packets);
1560   vlib_node_increment_counter (vm, stats_node_index,
1561                                NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1562                                other_packets);
1563   vlib_node_increment_counter (vm, stats_node_index,
1564                                NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1565   return frame->n_vectors;
1566 }
1567
1568 static inline uword
1569 nat44_ed_out2in_reass_node_fn_inline (vlib_main_t * vm,
1570                                       vlib_node_runtime_t * node,
1571                                       vlib_frame_t * frame)
1572 {
1573   u32 n_left_from, *from, *to_next;
1574   nat_next_t next_index;
1575   u32 pkts_processed = 0;
1576   snat_main_t *sm = &snat_main;
1577   f64 now = vlib_time_now (vm);
1578   u32 thread_index = vm->thread_index;
1579   snat_main_per_thread_data_t *per_thread_data =
1580     &sm->per_thread_data[thread_index];
1581   u32 *fragments_to_drop = 0;
1582   u32 *fragments_to_loopback = 0;
1583
1584   from = vlib_frame_vector_args (frame);
1585   n_left_from = frame->n_vectors;
1586   next_index = node->cached_next_index;
1587
1588   while (n_left_from > 0)
1589     {
1590       u32 n_left_to_next;
1591
1592       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1593
1594       while (n_left_from > 0 && n_left_to_next > 0)
1595         {
1596           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1597           vlib_buffer_t *b0;
1598           u32 next0;
1599           u8 cached0 = 0;
1600           ip4_header_t *ip0;
1601           nat_reass_ip4_t *reass0;
1602           udp_header_t *udp0;
1603           tcp_header_t *tcp0;
1604           icmp46_header_t *icmp0;
1605           clib_bihash_kv_16_8_t kv0, value0;
1606           snat_session_t *s0 = 0;
1607           u16 old_port0, new_port0;
1608           ip_csum_t sum0;
1609           snat_session_key_t e_key0, l_key0;
1610           lb_nat_type_t lb0;
1611           twice_nat_type_t twice_nat0;
1612           u8 identity_nat0;
1613
1614           /* speculatively enqueue b0 to the current next frame */
1615           bi0 = from[0];
1616           to_next[0] = bi0;
1617           from += 1;
1618           to_next += 1;
1619           n_left_from -= 1;
1620           n_left_to_next -= 1;
1621
1622           b0 = vlib_get_buffer (vm, bi0);
1623           next0 = nat_buffer_opaque (b0)->arc_next;
1624
1625           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1626           rx_fib_index0 =
1627             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1628                                                  sw_if_index0);
1629
1630           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1631             {
1632               next0 = NAT_NEXT_DROP;
1633               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1634               goto trace0;
1635             }
1636
1637           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1638           udp0 = ip4_next_header (ip0);
1639           tcp0 = (tcp_header_t *) udp0;
1640           icmp0 = (icmp46_header_t *) udp0;
1641           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1642
1643           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1644                                                  ip0->dst_address,
1645                                                  ip0->fragment_id,
1646                                                  ip0->protocol,
1647                                                  1, &fragments_to_drop);
1648
1649           if (PREDICT_FALSE (!reass0))
1650             {
1651               next0 = NAT_NEXT_DROP;
1652               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1653               nat_elog_notice ("maximum reassemblies exceeded");
1654               goto trace0;
1655             }
1656
1657           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1658             {
1659               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1660                 {
1661                   next0 = icmp_out2in_ed_slow_path
1662                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1663                      next0, now, thread_index, &s0);
1664
1665                   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP))
1666                     {
1667                       if (s0)
1668                         reass0->sess_index = s0 - per_thread_data->sessions;
1669                       else
1670                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1671                       reass0->thread_index = thread_index;
1672                       nat_ip4_reass_get_frags (reass0,
1673                                                &fragments_to_loopback);
1674                     }
1675
1676                   goto trace0;
1677                 }
1678
1679               make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1680                           ip0->protocol, rx_fib_index0, udp0->dst_port,
1681                           udp0->src_port);
1682
1683               if (clib_bihash_search_16_8
1684                   (&per_thread_data->out2in_ed, &kv0, &value0))
1685                 {
1686                   /* Try to match static mapping by external address and port,
1687                      destination address and port in packet */
1688                   e_key0.addr = ip0->dst_address;
1689                   e_key0.port = udp0->dst_port;
1690                   e_key0.protocol = proto0;
1691                   e_key0.fib_index = rx_fib_index0;
1692                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1693                                                  &twice_nat0, &lb0, 0,
1694                                                  &identity_nat0))
1695                     {
1696                       /*
1697                        * Send DHCP packets to the ipv4 stack, or we won't
1698                        * be able to use dhcp client on the outside interface
1699                        */
1700                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1701                                          && (udp0->dst_port
1702                                              ==
1703                                              clib_host_to_net_u16
1704                                              (UDP_DST_PORT_dhcp_to_client))))
1705                         {
1706                           goto trace0;
1707                         }
1708
1709                       if (!sm->forwarding_enabled)
1710                         {
1711                           b0->error =
1712                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1713                           next0 = NAT_NEXT_DROP;
1714                         }
1715                       else
1716                         {
1717                           if (next_src_nat (sm, ip0, ip0->protocol,
1718                                             udp0->src_port, udp0->dst_port,
1719                                             thread_index, rx_fib_index0))
1720                             {
1721                               next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1722                               goto trace0;
1723                             }
1724                           if (sm->num_workers > 1)
1725                             create_bypass_for_fwd_worker (sm, ip0,
1726                                                           rx_fib_index0);
1727                           else
1728                             create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1729                                                    thread_index);
1730                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1731                           nat_ip4_reass_get_frags (reass0,
1732                                                    &fragments_to_loopback);
1733                         }
1734                       goto trace0;
1735                     }
1736
1737                   if (PREDICT_FALSE (identity_nat0))
1738                     {
1739                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1740                       goto trace0;
1741                     }
1742
1743                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1744                     {
1745                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1746                       next0 = NAT_NEXT_DROP;
1747                       goto trace0;
1748                     }
1749
1750                   /* Create session initiated by host from external network */
1751                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1752                                                              e_key0, node,
1753                                                              thread_index,
1754                                                              twice_nat0, lb0,
1755                                                              now);
1756                   if (!s0)
1757                     {
1758                       b0->error =
1759                         node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1760                       next0 = NAT_NEXT_DROP;
1761                       goto trace0;
1762                     }
1763                   reass0->sess_index = s0 - per_thread_data->sessions;
1764                   reass0->thread_index = thread_index;
1765                 }
1766               else
1767                 {
1768                   s0 = pool_elt_at_index (per_thread_data->sessions,
1769                                           value0.value);
1770                   reass0->sess_index = value0.value;
1771                 }
1772               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1773             }
1774           else
1775             {
1776               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1777                 goto trace0;
1778               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1779                 {
1780                   if (nat_ip4_reass_add_fragment
1781                       (thread_index, reass0, bi0, &fragments_to_drop))
1782                     {
1783                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1784                       nat_elog_notice
1785                         ("maximum fragments per reassembly exceeded");
1786                       next0 = NAT_NEXT_DROP;
1787                       goto trace0;
1788                     }
1789                   cached0 = 1;
1790                   goto trace0;
1791                 }
1792               s0 = pool_elt_at_index (per_thread_data->sessions,
1793                                       reass0->sess_index);
1794             }
1795
1796           old_addr0 = ip0->dst_address.as_u32;
1797           ip0->dst_address = s0->in2out.addr;
1798           new_addr0 = ip0->dst_address.as_u32;
1799           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1800
1801           sum0 = ip0->checksum;
1802           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1803                                  ip4_header_t,
1804                                  dst_address /* changed member */ );
1805           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1806             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1807                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1808                                    src_address);
1809           ip0->checksum = ip_csum_fold (sum0);
1810
1811           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1812             {
1813               old_port0 = udp0->dst_port;
1814               new_port0 = udp0->dst_port = s0->in2out.port;
1815
1816               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1817                 {
1818                   sum0 = tcp0->checksum;
1819                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1820                                          ip4_header_t,
1821                                          dst_address /* changed member */ );
1822
1823                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1824                                          ip4_header_t /* cheat */ ,
1825                                          length /* changed member */ );
1826                   if (is_twice_nat_session (s0))
1827                     {
1828                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1829                                              s0->ext_host_nat_addr.as_u32,
1830                                              ip4_header_t, dst_address);
1831                       sum0 = ip_csum_update (sum0, tcp0->src_port,
1832                                              s0->ext_host_nat_port,
1833                                              ip4_header_t, length);
1834                       tcp0->src_port = s0->ext_host_nat_port;
1835                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1836                     }
1837                   tcp0->checksum = ip_csum_fold (sum0);
1838                 }
1839               else if (udp0->checksum)
1840                 {
1841                   sum0 = udp0->checksum;
1842                   sum0 =
1843                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1844                                     dst_address);
1845                   sum0 =
1846                     ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1847                                     length);
1848                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1849                     {
1850                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1851                                              s0->ext_host_nat_addr.as_u32,
1852                                              ip4_header_t, dst_address);
1853                       sum0 = ip_csum_update (sum0, udp0->src_port,
1854                                              s0->ext_host_nat_port,
1855                                              ip4_header_t, length);
1856                       udp0->src_port = s0->ext_host_nat_port;
1857                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1858                     }
1859                   udp0->checksum = ip_csum_fold (sum0);
1860                 }
1861               else
1862                 {
1863                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1864                     {
1865                       udp0->src_port = s0->ext_host_nat_port;
1866                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1867                     }
1868                 }
1869             }
1870
1871           /* Accounting */
1872           nat44_session_update_counters (s0, now,
1873                                          vlib_buffer_length_in_chain (vm, b0),
1874                                          thread_index);
1875           /* Per-user LRU list maintenance */
1876           nat44_session_update_lru (sm, s0, thread_index);
1877
1878         trace0:
1879           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1880                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1881             {
1882               nat44_reass_trace_t *t =
1883                 vlib_add_trace (vm, node, b0, sizeof (*t));
1884               t->cached = cached0;
1885               t->sw_if_index = sw_if_index0;
1886               t->next_index = next0;
1887             }
1888
1889           if (cached0)
1890             {
1891               n_left_to_next++;
1892               to_next--;
1893             }
1894           else
1895             {
1896               pkts_processed += next0 != NAT_NEXT_DROP;
1897
1898               /* verify speculative enqueue, maybe switch current next frame */
1899               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1900                                                to_next, n_left_to_next,
1901                                                bi0, next0);
1902             }
1903
1904           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1905             {
1906               from = vlib_frame_vector_args (frame);
1907               u32 len = vec_len (fragments_to_loopback);
1908               if (len <= VLIB_FRAME_SIZE)
1909                 {
1910                   clib_memcpy_fast (from, fragments_to_loopback,
1911                                     sizeof (u32) * len);
1912                   n_left_from = len;
1913                   vec_reset_length (fragments_to_loopback);
1914                 }
1915               else
1916                 {
1917                   clib_memcpy_fast (from, fragments_to_loopback +
1918                                     (len - VLIB_FRAME_SIZE),
1919                                     sizeof (u32) * VLIB_FRAME_SIZE);
1920                   n_left_from = VLIB_FRAME_SIZE;
1921                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1922                 }
1923             }
1924         }
1925
1926       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1927     }
1928
1929   vlib_node_increment_counter (vm, sm->ed_out2in_reass_node_index,
1930                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1931                                pkts_processed);
1932
1933   nat_send_all_to_node (vm, fragments_to_drop, node,
1934                         &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
1935                         NAT_NEXT_DROP);
1936
1937   vec_free (fragments_to_drop);
1938   vec_free (fragments_to_loopback);
1939   return frame->n_vectors;
1940 }
1941
1942 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1943                                      vlib_node_runtime_t * node,
1944                                      vlib_frame_t * frame)
1945 {
1946   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1947 }
1948
1949 /* *INDENT-OFF* */
1950 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1951   .name = "nat44-ed-out2in",
1952   .vector_size = sizeof (u32),
1953   .sibling_of = "nat-default",
1954   .format_trace = format_nat44_ed_out2in_trace,
1955   .type = VLIB_NODE_TYPE_INTERNAL,
1956   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1957   .error_strings = nat_out2in_ed_error_strings,
1958   .runtime_data_bytes = sizeof (snat_runtime_t),
1959 };
1960 /* *INDENT-ON* */
1961
1962 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1963                                               vlib_node_runtime_t * node,
1964                                               vlib_frame_t * frame)
1965 {
1966   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1967 }
1968
1969 /* *INDENT-OFF* */
1970 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1971   .name = "nat44-ed-out2in-slowpath",
1972   .vector_size = sizeof (u32),
1973   .sibling_of = "nat-default",
1974   .format_trace = format_nat44_ed_out2in_trace,
1975   .type = VLIB_NODE_TYPE_INTERNAL,
1976   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1977   .error_strings = nat_out2in_ed_error_strings,
1978   .runtime_data_bytes = sizeof (snat_runtime_t),
1979 };
1980 /* *INDENT-ON* */
1981
1982 VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm,
1983                                            vlib_node_runtime_t * node,
1984                                            vlib_frame_t * frame)
1985 {
1986   return nat44_ed_out2in_reass_node_fn_inline (vm, node, frame);
1987 }
1988
1989 /* *INDENT-OFF* */
1990 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
1991   .name = "nat44-ed-out2in-reass",
1992   .vector_size = sizeof (u32),
1993   .sibling_of = "nat-default",
1994   .format_trace = format_nat44_reass_trace,
1995   .type = VLIB_NODE_TYPE_INTERNAL,
1996   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1997   .error_strings = nat_out2in_ed_error_strings,
1998 };
1999 /* *INDENT-ON* */
2000
2001 static u8 *
2002 format_nat_pre_trace (u8 * s, va_list * args)
2003 {
2004   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2005   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2006   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
2007   return format (s, "out2in next_index %d", t->next_index);
2008 }
2009
2010 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
2011                                     vlib_node_runtime_t * node,
2012                                     vlib_frame_t * frame)
2013 {
2014   return nat_pre_node_fn_inline (vm, node, frame,
2015                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
2016 }
2017
2018 /* *INDENT-OFF* */
2019 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
2020   .name = "nat-pre-out2in",
2021   .vector_size = sizeof (u32),
2022   .sibling_of = "nat-default",
2023   .format_trace = format_nat_pre_trace,
2024   .type = VLIB_NODE_TYPE_INTERNAL,
2025   .n_errors = 0,
2026  };
2027 /* *INDENT-ON* */
2028
2029 /*
2030  * fd.io coding-style-patch-verification: ON
2031  *
2032  * Local Variables:
2033  * eval: (c-set-style "gnu")
2034  * End:
2035  */