src/plugins/nat/out2in_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_reass.h>
  31 #include <nat/nat_inlines.h>
  32
  33 #define foreach_nat_out2in_ed_error                     \
  34 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  35 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  36 _(OUT_OF_PORTS, "Out of ports")                         \
  37 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  38 _(NO_TRANSLATION, "No translation")                     \
  39 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  40 _(DROP_FRAGMENT, "Drop fragment")                       \
  41 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  42 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  43
  44 typedef enum
  45 {
  46 #define _(sym,str) NAT_OUT2IN_ED_ERROR_##sym,
  47   foreach_nat_out2in_ed_error
  48 #undef _
  49     NAT_OUT2IN_ED_N_ERROR,
  50 } nat_out2in_ed_error_t;
  51
  52 static char *nat_out2in_ed_error_strings[] = {
  53 #define _(sym,string) string,
  54   foreach_nat_out2in_ed_error
  55 #undef _
  56 };
  57
  58 typedef enum
  59 {
  60   NAT44_ED_OUT2IN_NEXT_DROP,
  61   NAT44_ED_OUT2IN_NEXT_LOOKUP,
  62   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
  63   NAT44_ED_OUT2IN_NEXT_IN2OUT,
  64   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
  65   NAT44_ED_OUT2IN_NEXT_REASS,
  66   NAT44_ED_OUT2IN_N_NEXT,
  67 } nat44_ed_out2in_next_t;
  68
  69 typedef struct
  70 {
  71   u32 sw_if_index;
  72   u32 next_index;
  73   u32 session_index;
  74   u32 is_slow_path;
  75 } nat44_ed_out2in_trace_t;
  76
  77 vlib_node_registration_t nat44_ed_out2in_node;
  78 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
  79 vlib_node_registration_t nat44_ed_out2in_reass_node;
  80
  81 static u8 *
  82 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  83 {
  84   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  85   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  86   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  87   char *tag;
  88
  89   tag =
  90     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  91     "NAT44_OUT2IN_ED_FAST_PATH";
  92
  93   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  94               t->sw_if_index, t->next_index, t->session_index);
  95
  96   return s;
  97 }
  98
  99 static inline u32
 100 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 101                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 102                           u32 sw_if_index0, u32 rx_fib_index0,
 103                           vlib_node_runtime_t * node, u32 next0, f64 now,
 104                           u32 thread_index, snat_session_t ** p_s0)
 105 {
 106   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 107                        next0, thread_index, p_s0, 0);
 108   snat_session_t *s0 = *p_s0;
 109   if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0))
 110     {
 111       /* Accounting */
 112       nat44_session_update_counters (s0, now,
 113                                      vlib_buffer_length_in_chain
 114                                      (sm->vlib_main, b0));
 115     }
 116   return next0;
 117 }
 118
 119 int
 120 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 121 {
 122   snat_main_t *sm = &snat_main;
 123   nat44_is_idle_session_ctx_t *ctx = arg;
 124   snat_session_t *s;
 125   u64 sess_timeout_time;
 126   nat_ed_ses_key_t ed_key;
 127   clib_bihash_kv_16_8_t ed_kv;
 128   int i;
 129   snat_address_t *a;
 130   snat_session_key_t key;
 131   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 132                                                        ctx->thread_index);
 133
 134   s = pool_elt_at_index (tsm->sessions, kv->value);
 135   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 136   if (ctx->now >= sess_timeout_time)
 137     {
 138       ed_key.l_addr = s->in2out.addr;
 139       ed_key.r_addr = s->ext_host_addr;
 140       ed_key.fib_index = s->in2out.fib_index;
 141       if (snat_is_unk_proto_session (s))
 142         {
 143           ed_key.proto = s->in2out.port;
 144           ed_key.r_port = 0;
 145           ed_key.l_port = 0;
 146         }
 147       else
 148         {
 149           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 150           ed_key.l_port = s->in2out.port;
 151           ed_key.r_port = s->ext_host_port;
 152         }
 153       if (is_twice_nat_session (s))
 154         {
 155           ed_key.r_addr = s->ext_host_nat_addr;
 156           ed_key.r_port = s->ext_host_nat_port;
 157         }
 158       ed_kv.key[0] = ed_key.as_u64[0];
 159       ed_kv.key[1] = ed_key.as_u64[1];
 160       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 161         nat_log_warn ("in2out_ed key del failed");
 162
 163       if (snat_is_unk_proto_session (s))
 164         goto delete;
 165
 166       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 167                                            s->out2in.addr.as_u32,
 168                                            s->in2out.protocol,
 169                                            s->in2out.port,
 170                                            s->out2in.port,
 171                                            s->in2out.fib_index);
 172
 173       if (is_twice_nat_session (s))
 174         {
 175           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 176             {
 177               key.protocol = s->in2out.protocol;
 178               key.port = s->ext_host_nat_port;
 179               a = sm->twice_nat_addresses + i;
 180               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 181                 {
 182                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 183                                                       ctx->thread_index,
 184                                                       &key);
 185                   break;
 186                 }
 187             }
 188         }
 189
 190       if (snat_is_session_static (s))
 191         goto delete;
 192
 193       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 194                                           &s->out2in);
 195     delete:
 196       nat44_delete_session (sm, s, ctx->thread_index);
 197       return 1;
 198     }
 199
 200   return 0;
 201 }
 202
 203 static snat_session_t *
 204 create_session_for_static_mapping_ed (snat_main_t * sm,
 205                                       vlib_buffer_t * b,
 206                                       snat_session_key_t l_key,
 207                                       snat_session_key_t e_key,
 208                                       vlib_node_runtime_t * node,
 209                                       u32 thread_index,
 210                                       twice_nat_type_t twice_nat,
 211                                       lb_nat_type_t lb_nat, f64 now)
 212 {
 213   snat_session_t *s;
 214   snat_user_t *u;
 215   ip4_header_t *ip;
 216   udp_header_t *udp;
 217   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 218   clib_bihash_kv_16_8_t kv;
 219   snat_session_key_t eh_key;
 220   nat44_is_idle_session_ctx_t ctx;
 221
 222   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 223     {
 224       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 225       nat_log_notice ("maximum sessions exceeded");
 226       return 0;
 227     }
 228
 229   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
 230   if (!u)
 231     {
 232       nat_log_warn ("create NAT user failed");
 233       return 0;
 234     }
 235
 236   s = nat_ed_session_alloc (sm, u, thread_index, now);
 237   if (!s)
 238     {
 239       nat44_delete_user_with_no_session (sm, u, thread_index);
 240       nat_log_warn ("create NAT session failed");
 241       return 0;
 242     }
 243
 244   ip = vlib_buffer_get_current (b);
 245   udp = ip4_next_header (ip);
 246
 247   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 248   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
 249   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 250   if (lb_nat)
 251     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 252   if (lb_nat == AFFINITY_LB_NAT)
 253     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 254   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 255   s->out2in = e_key;
 256   s->in2out = l_key;
 257   s->in2out.protocol = s->out2in.protocol;
 258   user_session_increment (sm, u, 1);
 259
 260   /* Add to lookup tables */
 261   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
 262               e_key.fib_index, e_key.port, s->ext_host_port);
 263   kv.value = s - tsm->sessions;
 264   ctx.now = now;
 265   ctx.thread_index = thread_index;
 266   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
 267                                                nat44_o2i_ed_is_idle_session_cb,
 268                                                &ctx))
 269     nat_log_notice ("out2in-ed key add failed");
 270
 271   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 272                                  ip->src_address.as_u32 == l_key.addr.as_u32))
 273     {
 274       eh_key.protocol = e_key.protocol;
 275       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 276                                                thread_index, &eh_key,
 277                                                sm->port_per_thread,
 278                                                tsm->snat_thread_index))
 279         {
 280           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 281           nat44_delete_session (sm, s, thread_index);
 282           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
 283             nat_log_notice ("out2in-ed key del failed");
 284           return 0;
 285         }
 286       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 287       s->ext_host_nat_port = eh_key.port;
 288       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 289       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
 290                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
 291     }
 292   else
 293     {
 294       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
 295                   l_key.fib_index, l_key.port, s->ext_host_port);
 296     }
 297   kv.value = s - tsm->sessions;
 298   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
 299                                                nat44_i2o_ed_is_idle_session_cb,
 300                                                &ctx))
 301     nat_log_notice ("in2out-ed key add failed");
 302
 303   return s;
 304 }
 305
 306 static_always_inline int
 307 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 308 {
 309   icmp46_header_t *icmp0;
 310   nat_ed_ses_key_t key0;
 311   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 312   ip4_header_t *inner_ip0;
 313   void *l4_header = 0;
 314   icmp46_header_t *inner_icmp0;
 315
 316   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 317   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 318
 319   if (!icmp_is_error_message (icmp0))
 320     {
 321       key0.proto = IP_PROTOCOL_ICMP;
 322       key0.l_addr = ip0->dst_address;
 323       key0.r_addr = ip0->src_address;
 324       key0.l_port = echo0->identifier;
 325       key0.r_port = 0;
 326     }
 327   else
 328     {
 329       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 330       l4_header = ip4_next_header (inner_ip0);
 331       key0.proto = inner_ip0->protocol;
 332       key0.l_addr = inner_ip0->src_address;
 333       key0.r_addr = inner_ip0->dst_address;
 334       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 335         {
 336         case SNAT_PROTOCOL_ICMP:
 337           inner_icmp0 = (icmp46_header_t *) l4_header;
 338           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 339           key0.l_port = inner_echo0->identifier;
 340           key0.r_port = 0;
 341           break;
 342         case SNAT_PROTOCOL_UDP:
 343         case SNAT_PROTOCOL_TCP:
 344           key0.l_port = ((tcp_udp_header_t *) l4_header)->src_port;
 345           key0.r_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 346           break;
 347         default:
 348           return -1;
 349         }
 350     }
 351   *p_key0 = key0;
 352   return 0;
 353 }
 354
 355 static int
 356 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
 357               u16 dst_port, u32 thread_index, u32 rx_fib_index)
 358 {
 359   clib_bihash_kv_16_8_t kv, value;
 360   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 361
 362   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
 363               rx_fib_index, src_port, dst_port);
 364   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 365     return 1;
 366
 367   return 0;
 368 }
 369
 370 static void
 371 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
 372                        u32 thread_index)
 373 {
 374   nat_ed_ses_key_t key;
 375   clib_bihash_kv_16_8_t kv, value;
 376   udp_header_t *udp;
 377   snat_user_t *u;
 378   snat_session_t *s = 0;
 379   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 380   f64 now = vlib_time_now (sm->vlib_main);
 381
 382   if (ip->protocol == IP_PROTOCOL_ICMP)
 383     {
 384       if (icmp_get_ed_key (ip, &key))
 385         return;
 386     }
 387   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 388     {
 389       udp = ip4_next_header (ip);
 390       key.r_addr = ip->src_address;
 391       key.l_addr = ip->dst_address;
 392       key.proto = ip->protocol;
 393       key.l_port = udp->dst_port;
 394       key.r_port = udp->src_port;
 395     }
 396   else
 397     {
 398       key.r_addr = ip->src_address;
 399       key.l_addr = ip->dst_address;
 400       key.proto = ip->protocol;
 401       key.l_port = key.r_port = 0;
 402     }
 403   key.fib_index = 0;
 404   kv.key[0] = key.as_u64[0];
 405   kv.key[1] = key.as_u64[1];
 406
 407   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 408     {
 409       s = pool_elt_at_index (tsm->sessions, value.value);
 410     }
 411   else
 412     {
 413       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 414         return;
 415
 416       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
 417                                   thread_index);
 418       if (!u)
 419         {
 420           nat_log_warn ("create NAT user failed");
 421           return;
 422         }
 423
 424       s = nat_ed_session_alloc (sm, u, thread_index, now);
 425       if (!s)
 426         {
 427           nat44_delete_user_with_no_session (sm, u, thread_index);
 428           nat_log_warn ("create NAT session failed");
 429           return;
 430         }
 431
 432       s->ext_host_addr = key.r_addr;
 433       s->ext_host_port = key.r_port;
 434       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 435       s->out2in.addr = key.l_addr;
 436       s->out2in.port = key.l_port;
 437       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
 438       s->out2in.fib_index = 0;
 439       s->in2out = s->out2in;
 440       user_session_increment (sm, u, 0);
 441
 442       kv.value = s - tsm->sessions;
 443       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
 444         nat_log_notice ("in2out_ed key add failed");
 445     }
 446
 447   if (ip->protocol == IP_PROTOCOL_TCP)
 448     {
 449       tcp_header_t *tcp = ip4_next_header (ip);
 450       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
 451         return;
 452     }
 453
 454   /* Accounting */
 455   nat44_session_update_counters (s, now, 0);
 456 }
 457
 458 u32
 459 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 460                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 461                       u8 * p_proto, snat_session_key_t * p_value,
 462                       u8 * p_dont_translate, void *d, void *e)
 463 {
 464   u32 next = ~0, sw_if_index, rx_fib_index;
 465   icmp46_header_t *icmp;
 466   nat_ed_ses_key_t key;
 467   clib_bihash_kv_16_8_t kv, value;
 468   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 469   snat_session_t *s = 0;
 470   u8 dont_translate = 0, is_addr_only;
 471   snat_session_key_t e_key, l_key;
 472
 473   icmp = (icmp46_header_t *) ip4_next_header (ip);
 474   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 475   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 476
 477   if (icmp_get_ed_key (ip, &key))
 478     {
 479       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 480       next = NAT44_ED_OUT2IN_NEXT_DROP;
 481       goto out;
 482     }
 483   key.fib_index = rx_fib_index;
 484   kv.key[0] = key.as_u64[0];
 485   kv.key[1] = key.as_u64[1];
 486
 487   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 488     {
 489       /* Try to match static mapping */
 490       e_key.addr = ip->dst_address;
 491       e_key.port = key.l_port;
 492       e_key.protocol = ip_proto_to_snat_proto (key.proto);
 493       e_key.fib_index = rx_fib_index;
 494       if (snat_static_mapping_match
 495           (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0))
 496         {
 497           if (!sm->forwarding_enabled)
 498             {
 499               /* Don't NAT packet aimed at the intfc address */
 500               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
 501                                                     ip->dst_address.as_u32)))
 502                 {
 503                   dont_translate = 1;
 504                   goto out;
 505                 }
 506               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 507               next = NAT44_ED_OUT2IN_NEXT_DROP;
 508               goto out;
 509             }
 510           else
 511             {
 512               dont_translate = 1;
 513               if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
 514                                 thread_index, rx_fib_index))
 515                 {
 516                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
 517                   goto out;
 518                 }
 519               create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
 520               goto out;
 521             }
 522         }
 523
 524       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 525                          (icmp->type != ICMP4_echo_request || !is_addr_only)))
 526         {
 527           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 528           next = NAT44_ED_OUT2IN_NEXT_DROP;
 529           goto out;
 530         }
 531
 532       /* Create session initiated by host from external network */
 533       s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
 534                                                 thread_index, 0, 0,
 535                                                 vlib_time_now
 536                                                 (sm->vlib_main));
 537
 538       if (!s)
 539         {
 540           next = NAT44_ED_OUT2IN_NEXT_DROP;
 541           goto out;
 542         }
 543     }
 544   else
 545     {
 546       if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
 547                          icmp->type != ICMP4_echo_request &&
 548                          !icmp_is_error_message (icmp)))
 549         {
 550           b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 551           next = NAT44_ED_OUT2IN_NEXT_DROP;
 552           goto out;
 553         }
 554
 555       s = pool_elt_at_index (tsm->sessions, value.value);
 556     }
 557
 558   *p_proto = ip_proto_to_snat_proto (key.proto);
 559 out:
 560   if (s)
 561     *p_value = s->in2out;
 562   *p_dont_translate = dont_translate;
 563   if (d)
 564     *(snat_session_t **) d = s;
 565   return next;
 566 }
 567
 568 static snat_session_t *
 569 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
 570                                vlib_buffer_t * b,
 571                                ip4_header_t * ip,
 572                                u32 rx_fib_index,
 573                                u32 thread_index,
 574                                f64 now,
 575                                vlib_main_t * vm, vlib_node_runtime_t * node)
 576 {
 577   clib_bihash_kv_8_8_t kv, value;
 578   clib_bihash_kv_16_8_t s_kv, s_value;
 579   snat_static_mapping_t *m;
 580   u32 old_addr, new_addr;
 581   ip_csum_t sum;
 582   snat_session_t *s;
 583   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 584   snat_user_t *u;
 585
 586   old_addr = ip->dst_address.as_u32;
 587
 588   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 589               rx_fib_index, 0, 0);
 590
 591   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 592     {
 593       s = pool_elt_at_index (tsm->sessions, s_value.value);
 594       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 595     }
 596   else
 597     {
 598       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 599         {
 600           b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 601           nat_log_notice ("maximum sessions exceeded");
 602           return 0;
 603         }
 604
 605       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 606       if (clib_bihash_search_8_8
 607           (&sm->static_mapping_by_external, &kv, &value))
 608         {
 609           b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 610           return 0;
 611         }
 612
 613       m = pool_elt_at_index (sm->static_mappings, value.value);
 614
 615       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 616
 617       u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
 618                                   thread_index);
 619       if (!u)
 620         {
 621           nat_log_warn ("create NAT user failed");
 622           return 0;
 623         }
 624
 625       /* Create a new session */
 626       s = nat_ed_session_alloc (sm, u, thread_index, now);
 627       if (!s)
 628         {
 629           nat44_delete_user_with_no_session (sm, u, thread_index);
 630           nat_log_warn ("create NAT session failed");
 631           return 0;
 632         }
 633
 634       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 635       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 636       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 637       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 638       s->out2in.addr.as_u32 = old_addr;
 639       s->out2in.fib_index = rx_fib_index;
 640       s->in2out.addr.as_u32 = new_addr;
 641       s->in2out.fib_index = m->fib_index;
 642       s->in2out.port = s->out2in.port = ip->protocol;
 643       user_session_increment (sm, u, 1);
 644
 645       /* Add to lookup tables */
 646       s_kv.value = s - tsm->sessions;
 647       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 648         nat_log_notice ("out2in key add failed");
 649
 650       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 651                   m->fib_index, 0, 0);
 652       s_kv.value = s - tsm->sessions;
 653       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 654         nat_log_notice ("in2out key add failed");
 655     }
 656
 657   /* Update IP checksum */
 658   sum = ip->checksum;
 659   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 660   ip->checksum = ip_csum_fold (sum);
 661
 662   vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 663
 664   /* Accounting */
 665   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 666
 667   return s;
 668 }
 669
 670 static inline uword
 671 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
 672                                 vlib_node_runtime_t * node,
 673                                 vlib_frame_t * frame, int is_slow_path)
 674 {
 675   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 676   nat44_ed_out2in_next_t next_index;
 677   snat_main_t *sm = &snat_main;
 678   f64 now = vlib_time_now (vm);
 679   u32 thread_index = vm->thread_index;
 680   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 681
 682   stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
 683     nat44_ed_out2in_node.index;
 684
 685   from = vlib_frame_vector_args (frame);
 686   n_left_from = frame->n_vectors;
 687   next_index = node->cached_next_index;
 688
 689   while (n_left_from > 0)
 690     {
 691       u32 n_left_to_next;
 692
 693       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 694
 695       while (n_left_from >= 4 && n_left_to_next >= 2)
 696         {
 697           u32 bi0, bi1;
 698           vlib_buffer_t *b0, *b1;
 699           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
 700             new_addr0;
 701           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
 702             new_addr1;
 703           u16 old_port0, new_port0, old_port1, new_port1;
 704           ip4_header_t *ip0, *ip1;
 705           udp_header_t *udp0, *udp1;
 706           tcp_header_t *tcp0, *tcp1;
 707           icmp46_header_t *icmp0, *icmp1;
 708           snat_session_t *s0 = 0, *s1 = 0;
 709           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 710           ip_csum_t sum0, sum1;
 711           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
 712           lb_nat_type_t lb_nat0, lb_nat1;
 713           twice_nat_type_t twice_nat0, twice_nat1;
 714
 715           /* Prefetch next iteration. */
 716           {
 717             vlib_buffer_t *p2, *p3;
 718
 719             p2 = vlib_get_buffer (vm, from[2]);
 720             p3 = vlib_get_buffer (vm, from[3]);
 721
 722             vlib_prefetch_buffer_header (p2, LOAD);
 723             vlib_prefetch_buffer_header (p3, LOAD);
 724
 725             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 726             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 727           }
 728
 729           /* speculatively enqueue b0 and b1 to the current next frame */
 730           to_next[0] = bi0 = from[0];
 731           to_next[1] = bi1 = from[1];
 732           from += 2;
 733           to_next += 2;
 734           n_left_from -= 2;
 735           n_left_to_next -= 2;
 736
 737           b0 = vlib_get_buffer (vm, bi0);
 738           b1 = vlib_get_buffer (vm, bi1);
 739
 740           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
 741           vnet_buffer (b0)->snat.flags = 0;
 742           ip0 = vlib_buffer_get_current (b0);
 743
 744           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 745           rx_fib_index0 =
 746             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 747                                                  sw_if_index0);
 748
 749           if (PREDICT_FALSE (ip0->ttl == 1))
 750             {
 751               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 752               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 753                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 754                                            0);
 755               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
 756               goto trace00;
 757             }
 758
 759           udp0 = ip4_next_header (ip0);
 760           tcp0 = (tcp_header_t *) udp0;
 761           icmp0 = (icmp46_header_t *) udp0;
 762           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 763
 764           if (is_slow_path)
 765             {
 766               if (PREDICT_FALSE (proto0 == ~0))
 767                 {
 768                   s0 =
 769                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
 770                                                    thread_index, now, vm,
 771                                                    node);
 772                   if (!sm->forwarding_enabled)
 773                     {
 774                       if (!s0)
 775                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 776                       goto trace00;
 777                     }
 778                 }
 779
 780               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 781                 {
 782                   next0 = icmp_out2in_ed_slow_path
 783                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 784                      next0, now, thread_index, &s0);
 785                   goto trace00;
 786                 }
 787             }
 788           else
 789             {
 790               if (PREDICT_FALSE (proto0 == ~0))
 791                 {
 792                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 793                   goto trace00;
 794                 }
 795
 796               if (ip4_is_fragment (ip0))
 797                 {
 798                   next0 = NAT44_ED_OUT2IN_NEXT_REASS;
 799                   goto trace00;
 800                 }
 801
 802               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 803                 {
 804                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 805                   goto trace00;
 806                 }
 807             }
 808
 809           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
 810                       ip0->protocol, rx_fib_index0, udp0->dst_port,
 811                       udp0->src_port);
 812
 813           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
 814             {
 815               if (is_slow_path)
 816                 {
 817                   /* Try to match static mapping by external address and port,
 818                      destination address and port in packet */
 819                   e_key0.addr = ip0->dst_address;
 820                   e_key0.port = udp0->dst_port;
 821                   e_key0.protocol = proto0;
 822                   e_key0.fib_index = rx_fib_index0;
 823                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
 824                                                  &twice_nat0, &lb_nat0,
 825                                                  &ip0->src_address))
 826                     {
 827                       /*
 828                        * Send DHCP packets to the ipv4 stack, or we won't
 829                        * be able to use dhcp client on the outside interface
 830                        */
 831                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 832                                          && (udp0->dst_port ==
 833                                              clib_host_to_net_u16
 834                                              (UDP_DST_PORT_dhcp_to_client))))
 835                         {
 836                           vnet_feature_next (&next0, b0);
 837                           goto trace00;
 838                         }
 839
 840                       if (!sm->forwarding_enabled)
 841                         {
 842                           b0->error =
 843                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 844                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 845                         }
 846                       else
 847                         {
 848                           if (next_src_nat (sm, ip0, ip0->protocol,
 849                                             udp0->src_port, udp0->dst_port,
 850                                             thread_index, rx_fib_index0))
 851                             {
 852                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
 853                               goto trace00;
 854                             }
 855                           create_bypass_for_fwd (sm, ip0, rx_fib_index0,
 856                                                  thread_index);
 857                         }
 858                       goto trace00;
 859                     }
 860
 861                   /* Create session initiated by host from external network */
 862                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
 863                                                              e_key0, node,
 864                                                              thread_index,
 865                                                              twice_nat0,
 866                                                              lb_nat0, now);
 867
 868                   if (!s0)
 869                     {
 870                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
 871                       goto trace00;
 872                     }
 873                 }
 874               else
 875                 {
 876                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
 877                   goto trace00;
 878                 }
 879             }
 880           else
 881             {
 882               s0 = pool_elt_at_index (tsm->sessions, value0.value);
 883             }
 884
 885           old_addr0 = ip0->dst_address.as_u32;
 886           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
 887           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 888
 889           sum0 = ip0->checksum;
 890           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 891                                  dst_address);
 892           if (PREDICT_FALSE (is_twice_nat_session (s0)))
 893             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 894                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
 895                                    src_address);
 896           ip0->checksum = ip_csum_fold (sum0);
 897
 898           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 899             {
 900               old_port0 = tcp0->dst_port;
 901               new_port0 = tcp0->dst_port = s0->in2out.port;
 902
 903               sum0 = tcp0->checksum;
 904               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 905                                      dst_address);
 906               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
 907                                      length);
 908               if (is_twice_nat_session (s0))
 909                 {
 910                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
 911                                          s0->ext_host_nat_addr.as_u32,
 912                                          ip4_header_t, dst_address);
 913                   sum0 = ip_csum_update (sum0, tcp0->src_port,
 914                                          s0->ext_host_nat_port, ip4_header_t,
 915                                          length);
 916                   tcp0->src_port = s0->ext_host_nat_port;
 917                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 918                 }
 919               tcp0->checksum = ip_csum_fold (sum0);
 920               if (nat44_set_tcp_session_state_o2i
 921                   (sm, s0, tcp0, thread_index))
 922                 goto trace00;
 923             }
 924           else
 925             {
 926               udp0->dst_port = s0->in2out.port;
 927               if (is_twice_nat_session (s0))
 928                 {
 929                   udp0->src_port = s0->ext_host_nat_port;
 930                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
 931                 }
 932               udp0->checksum = 0;
 933             }
 934
 935           /* Accounting */
 936           nat44_session_update_counters (s0, now,
 937                                          vlib_buffer_length_in_chain (vm,
 938                                                                       b0));
 939
 940         trace00:
 941           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 942                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 943             {
 944               nat44_ed_out2in_trace_t *t =
 945                 vlib_add_trace (vm, node, b0, sizeof (*t));
 946               t->is_slow_path = is_slow_path;
 947               t->sw_if_index = sw_if_index0;
 948               t->next_index = next0;
 949               t->session_index = ~0;
 950               if (s0)
 951                 t->session_index = s0 - tsm->sessions;
 952             }
 953
 954           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
 955
 956           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
 957           vnet_buffer (b1)->snat.flags = 0;
 958           ip1 = vlib_buffer_get_current (b1);
 959
 960           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 961           rx_fib_index1 =
 962             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 963                                                  sw_if_index1);
 964
 965           if (PREDICT_FALSE (ip1->ttl == 1))
 966             {
 967               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 968               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 969                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 970                                            0);
 971               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
 972               goto trace01;
 973             }
 974
 975           udp1 = ip4_next_header (ip1);
 976           tcp1 = (tcp_header_t *) udp1;
 977           icmp1 = (icmp46_header_t *) udp1;
 978           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 979
 980           if (is_slow_path)
 981             {
 982               if (PREDICT_FALSE (proto1 == ~0))
 983                 {
 984                   s1 =
 985                     nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
 986                                                    thread_index, now, vm,
 987                                                    node);
 988                   if (!sm->forwarding_enabled)
 989                     {
 990                       if (!s1)
 991                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
 992                       goto trace01;
 993                     }
 994                 }
 995
 996               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 997                 {
 998                   next1 = icmp_out2in_ed_slow_path
 999                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1000                      next1, now, thread_index, &s1);
1001                   goto trace01;
1002                 }
1003             }
1004           else
1005             {
1006               if (PREDICT_FALSE (proto1 == ~0))
1007                 {
1008                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1009                   goto trace01;
1010                 }
1011
1012               if (ip4_is_fragment (ip1))
1013                 {
1014                   next1 = NAT44_ED_OUT2IN_NEXT_REASS;
1015                   goto trace01;
1016                 }
1017
1018               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1019                 {
1020                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1021                   goto trace01;
1022                 }
1023             }
1024
1025           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1026                       ip1->protocol, rx_fib_index1, udp1->dst_port,
1027                       udp1->src_port);
1028
1029           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1030             {
1031               if (is_slow_path)
1032                 {
1033                   /* Try to match static mapping by external address and port,
1034                      destination address and port in packet */
1035                   e_key1.addr = ip1->dst_address;
1036                   e_key1.port = udp1->dst_port;
1037                   e_key1.protocol = proto1;
1038                   e_key1.fib_index = rx_fib_index1;
1039                   if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1040                                                  &twice_nat1, &lb_nat1,
1041                                                  &ip1->src_address))
1042                     {
1043                       /*
1044                        * Send DHCP packets to the ipv4 stack, or we won't
1045                        * be able to use dhcp client on the outside interface
1046                        */
1047                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1048                                          && (udp1->dst_port ==
1049                                              clib_host_to_net_u16
1050                                              (UDP_DST_PORT_dhcp_to_client))))
1051                         {
1052                           vnet_feature_next (&next1, b1);
1053                           goto trace01;
1054                         }
1055
1056                       if (!sm->forwarding_enabled)
1057                         {
1058                           b1->error =
1059                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1060                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1061                         }
1062                       else
1063                         {
1064                           if (next_src_nat (sm, ip1, ip1->protocol,
1065                                             udp1->src_port, udp1->dst_port,
1066                                             thread_index, rx_fib_index1))
1067                             {
1068                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1069                               goto trace01;
1070                             }
1071                           create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1072                                                  thread_index);
1073                         }
1074                       goto trace01;
1075                     }
1076
1077                   /* Create session initiated by host from external network */
1078                   s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1079                                                              e_key1, node,
1080                                                              thread_index,
1081                                                              twice_nat1,
1082                                                              lb_nat1, now);
1083
1084                   if (!s1)
1085                     {
1086                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1087                       goto trace01;
1088                     }
1089                 }
1090               else
1091                 {
1092                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1093                   goto trace01;
1094                 }
1095             }
1096           else
1097             {
1098               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1099             }
1100
1101           old_addr1 = ip1->dst_address.as_u32;
1102           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1103           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1104
1105           sum1 = ip1->checksum;
1106           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1107                                  dst_address);
1108           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1109             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1110                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
1111                                    src_address);
1112           ip1->checksum = ip_csum_fold (sum1);
1113
1114           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1115             {
1116               old_port1 = tcp1->dst_port;
1117               new_port1 = tcp1->dst_port = s1->in2out.port;
1118
1119               sum1 = tcp1->checksum;
1120               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1121                                      dst_address);
1122               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1123                                      length);
1124               if (is_twice_nat_session (s1))
1125                 {
1126                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1127                                          s1->ext_host_nat_addr.as_u32,
1128                                          ip4_header_t, dst_address);
1129                   sum1 = ip_csum_update (sum1, tcp1->src_port,
1130                                          s1->ext_host_nat_port, ip4_header_t,
1131                                          length);
1132                   tcp1->src_port = s1->ext_host_nat_port;
1133                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1134                 }
1135               tcp1->checksum = ip_csum_fold (sum1);
1136               if (nat44_set_tcp_session_state_o2i
1137                   (sm, s1, tcp1, thread_index))
1138                 goto trace01;
1139             }
1140           else
1141             {
1142               udp1->dst_port = s1->in2out.port;
1143               if (is_twice_nat_session (s1))
1144                 {
1145                   udp1->src_port = s1->ext_host_nat_port;
1146                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1147                 }
1148               udp1->checksum = 0;
1149             }
1150
1151           /* Accounting */
1152           nat44_session_update_counters (s1, now,
1153                                          vlib_buffer_length_in_chain (vm,
1154                                                                       b1));
1155
1156         trace01:
1157           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1158                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1159             {
1160               nat44_ed_out2in_trace_t *t =
1161                 vlib_add_trace (vm, node, b1, sizeof (*t));
1162               t->is_slow_path = is_slow_path;
1163               t->sw_if_index = sw_if_index1;
1164               t->next_index = next1;
1165               t->session_index = ~0;
1166               if (s1)
1167                 t->session_index = s1 - tsm->sessions;
1168             }
1169
1170           pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
1171
1172           /* verify speculative enqueues, maybe switch current next frame */
1173           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1174                                            to_next, n_left_to_next,
1175                                            bi0, bi1, next0, next1);
1176         }
1177
1178       while (n_left_from > 0 && n_left_to_next > 0)
1179         {
1180           u32 bi0;
1181           vlib_buffer_t *b0;
1182           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1183             new_addr0;
1184           u16 old_port0, new_port0;
1185           ip4_header_t *ip0;
1186           udp_header_t *udp0;
1187           tcp_header_t *tcp0;
1188           icmp46_header_t *icmp0;
1189           snat_session_t *s0 = 0;
1190           clib_bihash_kv_16_8_t kv0, value0;
1191           ip_csum_t sum0;
1192           snat_session_key_t e_key0, l_key0;
1193           lb_nat_type_t lb_nat0;
1194           twice_nat_type_t twice_nat0;
1195
1196           /* speculatively enqueue b0 to the current next frame */
1197           bi0 = from[0];
1198           to_next[0] = bi0;
1199           from += 1;
1200           to_next += 1;
1201           n_left_from -= 1;
1202           n_left_to_next -= 1;
1203
1204           b0 = vlib_get_buffer (vm, bi0);
1205           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1206           vnet_buffer (b0)->snat.flags = 0;
1207           ip0 = vlib_buffer_get_current (b0);
1208
1209           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1210           rx_fib_index0 =
1211             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1212                                                  sw_if_index0);
1213
1214           if (PREDICT_FALSE (ip0->ttl == 1))
1215             {
1216               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1217               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1218                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1219                                            0);
1220               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1221               goto trace0;
1222             }
1223
1224           udp0 = ip4_next_header (ip0);
1225           tcp0 = (tcp_header_t *) udp0;
1226           icmp0 = (icmp46_header_t *) udp0;
1227           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1228
1229           if (is_slow_path)
1230             {
1231               if (PREDICT_FALSE (proto0 == ~0))
1232                 {
1233                   s0 =
1234                     nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1235                                                    thread_index, now, vm,
1236                                                    node);
1237                   if (!sm->forwarding_enabled)
1238                     {
1239                       if (!s0)
1240                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1241                       goto trace0;
1242                     }
1243                 }
1244
1245               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1246                 {
1247                   next0 = icmp_out2in_ed_slow_path
1248                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1249                      next0, now, thread_index, &s0);
1250                   goto trace0;
1251                 }
1252             }
1253           else
1254             {
1255               if (PREDICT_FALSE (proto0 == ~0))
1256                 {
1257                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1258                   goto trace0;
1259                 }
1260
1261               if (ip4_is_fragment (ip0))
1262                 {
1263                   next0 = NAT44_ED_OUT2IN_NEXT_REASS;
1264                   goto trace0;
1265                 }
1266
1267               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1268                 {
1269                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1270                   goto trace0;
1271                 }
1272             }
1273
1274           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1275                       ip0->protocol, rx_fib_index0, udp0->dst_port,
1276                       udp0->src_port);
1277
1278           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1279             {
1280               if (is_slow_path)
1281                 {
1282                   /* Try to match static mapping by external address and port,
1283                      destination address and port in packet */
1284                   e_key0.addr = ip0->dst_address;
1285                   e_key0.port = udp0->dst_port;
1286                   e_key0.protocol = proto0;
1287                   e_key0.fib_index = rx_fib_index0;
1288                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1289                                                  &twice_nat0, &lb_nat0,
1290                                                  &ip0->src_address))
1291                     {
1292                       /*
1293                        * Send DHCP packets to the ipv4 stack, or we won't
1294                        * be able to use dhcp client on the outside interface
1295                        */
1296                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1297                                          && (udp0->dst_port ==
1298                                              clib_host_to_net_u16
1299                                              (UDP_DST_PORT_dhcp_to_client))))
1300                         {
1301                           vnet_feature_next (&next0, b0);
1302                           goto trace0;
1303                         }
1304
1305                       if (!sm->forwarding_enabled)
1306                         {
1307                           b0->error =
1308                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1309                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1310                         }
1311                       else
1312                         {
1313                           if (next_src_nat (sm, ip0, ip0->protocol,
1314                                             udp0->src_port, udp0->dst_port,
1315                                             thread_index, rx_fib_index0))
1316                             {
1317                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1318                               goto trace0;
1319                             }
1320                           create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1321                                                  thread_index);
1322                         }
1323                       goto trace0;
1324                     }
1325
1326                   /* Create session initiated by host from external network */
1327                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1328                                                              e_key0, node,
1329                                                              thread_index,
1330                                                              twice_nat0,
1331                                                              lb_nat0, now);
1332
1333                   if (!s0)
1334                     {
1335                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1336                       goto trace0;
1337                     }
1338                 }
1339               else
1340                 {
1341                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1342                   goto trace0;
1343                 }
1344             }
1345           else
1346             {
1347               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1348             }
1349
1350           old_addr0 = ip0->dst_address.as_u32;
1351           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1352           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1353
1354           sum0 = ip0->checksum;
1355           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1356                                  dst_address);
1357           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1358             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1359                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1360                                    src_address);
1361           ip0->checksum = ip_csum_fold (sum0);
1362
1363           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1364             {
1365               old_port0 = tcp0->dst_port;
1366               new_port0 = tcp0->dst_port = s0->in2out.port;
1367
1368               sum0 = tcp0->checksum;
1369               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1370                                      dst_address);
1371               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1372                                      length);
1373               if (is_twice_nat_session (s0))
1374                 {
1375                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1376                                          s0->ext_host_nat_addr.as_u32,
1377                                          ip4_header_t, dst_address);
1378                   sum0 = ip_csum_update (sum0, tcp0->src_port,
1379                                          s0->ext_host_nat_port, ip4_header_t,
1380                                          length);
1381                   tcp0->src_port = s0->ext_host_nat_port;
1382                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1383                 }
1384               tcp0->checksum = ip_csum_fold (sum0);
1385               if (nat44_set_tcp_session_state_o2i
1386                   (sm, s0, tcp0, thread_index))
1387                 goto trace0;
1388             }
1389           else
1390             {
1391               udp0->dst_port = s0->in2out.port;
1392               if (is_twice_nat_session (s0))
1393                 {
1394                   udp0->src_port = s0->ext_host_nat_port;
1395                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1396                 }
1397               udp0->checksum = 0;
1398             }
1399
1400           /* Accounting */
1401           nat44_session_update_counters (s0, now,
1402                                          vlib_buffer_length_in_chain (vm,
1403                                                                       b0));
1404
1405         trace0:
1406           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1407                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1408             {
1409               nat44_ed_out2in_trace_t *t =
1410                 vlib_add_trace (vm, node, b0, sizeof (*t));
1411               t->is_slow_path = is_slow_path;
1412               t->sw_if_index = sw_if_index0;
1413               t->next_index = next0;
1414               t->session_index = ~0;
1415               if (s0)
1416                 t->session_index = s0 - tsm->sessions;
1417             }
1418
1419           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1420           /* verify speculative enqueue, maybe switch current next frame */
1421           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1422                                            to_next, n_left_to_next,
1423                                            bi0, next0);
1424         }
1425
1426       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1427     }
1428
1429   vlib_node_increment_counter (vm, stats_node_index,
1430                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1431                                pkts_processed);
1432   return frame->n_vectors;
1433 }
1434
1435 static uword
1436 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
1437                               vlib_node_runtime_t * node,
1438                               vlib_frame_t * frame)
1439 {
1440   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1441 }
1442
1443 /* *INDENT-OFF* */
1444 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1445   .function = nat44_ed_out2in_fast_path_fn,
1446   .name = "nat44-ed-out2in",
1447   .vector_size = sizeof (u32),
1448   .format_trace = format_nat44_ed_out2in_trace,
1449   .type = VLIB_NODE_TYPE_INTERNAL,
1450   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1451   .error_strings = nat_out2in_ed_error_strings,
1452   .runtime_data_bytes = sizeof (snat_runtime_t),
1453   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1454   .next_nodes = {
1455     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1456     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1457     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1458     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1459     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1460     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1461   },
1462 };
1463 /* *INDENT-ON* */
1464
1465 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node,
1466                               nat44_ed_out2in_fast_path_fn);
1467
1468 static uword
1469 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
1470                               vlib_node_runtime_t * node,
1471                               vlib_frame_t * frame)
1472 {
1473   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1474 }
1475
1476 /* *INDENT-OFF* */
1477 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1478   .function = nat44_ed_out2in_slow_path_fn,
1479   .name = "nat44-ed-out2in-slowpath",
1480   .vector_size = sizeof (u32),
1481   .format_trace = format_nat44_ed_out2in_trace,
1482   .type = VLIB_NODE_TYPE_INTERNAL,
1483   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1484   .error_strings = nat_out2in_ed_error_strings,
1485   .runtime_data_bytes = sizeof (snat_runtime_t),
1486   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1487   .next_nodes = {
1488     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1489     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1490     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1491     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1492     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1493     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1494   },
1495 };
1496 /* *INDENT-ON* */
1497
1498 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
1499                               nat44_ed_out2in_slow_path_fn);
1500
1501 static uword
1502 nat44_ed_out2in_reass_node_fn (vlib_main_t * vm,
1503                                vlib_node_runtime_t * node,
1504                                vlib_frame_t * frame)
1505 {
1506   u32 n_left_from, *from, *to_next;
1507   nat44_ed_out2in_next_t next_index;
1508   u32 pkts_processed = 0;
1509   snat_main_t *sm = &snat_main;
1510   f64 now = vlib_time_now (vm);
1511   u32 thread_index = vm->thread_index;
1512   snat_main_per_thread_data_t *per_thread_data =
1513     &sm->per_thread_data[thread_index];
1514   u32 *fragments_to_drop = 0;
1515   u32 *fragments_to_loopback = 0;
1516
1517   from = vlib_frame_vector_args (frame);
1518   n_left_from = frame->n_vectors;
1519   next_index = node->cached_next_index;
1520
1521   while (n_left_from > 0)
1522     {
1523       u32 n_left_to_next;
1524
1525       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1526
1527       while (n_left_from > 0 && n_left_to_next > 0)
1528         {
1529           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1530           vlib_buffer_t *b0;
1531           u32 next0;
1532           u8 cached0 = 0;
1533           ip4_header_t *ip0;
1534           nat_reass_ip4_t *reass0;
1535           udp_header_t *udp0;
1536           tcp_header_t *tcp0;
1537           icmp46_header_t *icmp0;
1538           clib_bihash_kv_16_8_t kv0, value0;
1539           snat_session_t *s0 = 0;
1540           u16 old_port0, new_port0;
1541           ip_csum_t sum0;
1542           snat_session_key_t e_key0, l_key0;
1543           lb_nat_type_t lb0;
1544           twice_nat_type_t twice_nat0;
1545
1546           /* speculatively enqueue b0 to the current next frame */
1547           bi0 = from[0];
1548           to_next[0] = bi0;
1549           from += 1;
1550           to_next += 1;
1551           n_left_from -= 1;
1552           n_left_to_next -= 1;
1553
1554           b0 = vlib_get_buffer (vm, bi0);
1555           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1556
1557           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1558           rx_fib_index0 =
1559             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1560                                                  sw_if_index0);
1561
1562           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1563             {
1564               next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1565               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1566               goto trace0;
1567             }
1568
1569           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1570           udp0 = ip4_next_header (ip0);
1571           tcp0 = (tcp_header_t *) udp0;
1572           icmp0 = (icmp46_header_t *) udp0;
1573           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1574
1575           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1576                                                  ip0->dst_address,
1577                                                  ip0->fragment_id,
1578                                                  ip0->protocol,
1579                                                  1, &fragments_to_drop);
1580
1581           if (PREDICT_FALSE (!reass0))
1582             {
1583               next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1584               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1585               nat_log_notice ("maximum reassemblies exceeded");
1586               goto trace0;
1587             }
1588
1589           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1590             {
1591               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1592                 {
1593                   next0 = icmp_out2in_ed_slow_path
1594                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1595                      next0, now, thread_index, &s0);
1596
1597                   if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP))
1598                     {
1599                       if (s0)
1600                         reass0->sess_index = s0 - per_thread_data->sessions;
1601                       else
1602                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1603                       reass0->thread_index = thread_index;
1604                       nat_ip4_reass_get_frags (reass0,
1605                                                &fragments_to_loopback);
1606                     }
1607
1608                   goto trace0;
1609                 }
1610
1611               make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1612                           ip0->protocol, rx_fib_index0, udp0->dst_port,
1613                           udp0->src_port);
1614
1615               if (clib_bihash_search_16_8
1616                   (&per_thread_data->out2in_ed, &kv0, &value0))
1617                 {
1618                   /* Try to match static mapping by external address and port,
1619                      destination address and port in packet */
1620                   e_key0.addr = ip0->dst_address;
1621                   e_key0.port = udp0->dst_port;
1622                   e_key0.protocol = proto0;
1623                   e_key0.fib_index = rx_fib_index0;
1624                   if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1625                                                  &twice_nat0, &lb0, 0))
1626                     {
1627                       /*
1628                        * Send DHCP packets to the ipv4 stack, or we won't
1629                        * be able to use dhcp client on the outside interface
1630                        */
1631                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1632                                          && (udp0->dst_port
1633                                              ==
1634                                              clib_host_to_net_u16
1635                                              (UDP_DST_PORT_dhcp_to_client))))
1636                         {
1637                           vnet_feature_next (&next0, b0);
1638                           goto trace0;
1639                         }
1640
1641                       if (!sm->forwarding_enabled)
1642                         {
1643                           b0->error =
1644                             node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1645                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1646                         }
1647                       else
1648                         {
1649                           if (next_src_nat (sm, ip0, ip0->protocol,
1650                                             udp0->src_port, udp0->dst_port,
1651                                             thread_index, rx_fib_index0))
1652                             {
1653                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1654                               goto trace0;
1655                             }
1656                           create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1657                                                  thread_index);
1658                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1659                           nat_ip4_reass_get_frags (reass0,
1660                                                    &fragments_to_loopback);
1661                         }
1662                       goto trace0;
1663                     }
1664
1665                   /* Create session initiated by host from external network */
1666                   s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1667                                                              e_key0, node,
1668                                                              thread_index,
1669                                                              twice_nat0, lb0,
1670                                                              now);
1671                   if (!s0)
1672                     {
1673                       b0->error =
1674                         node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1675                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1676                       goto trace0;
1677                     }
1678                   reass0->sess_index = s0 - per_thread_data->sessions;
1679                   reass0->thread_index = thread_index;
1680                 }
1681               else
1682                 {
1683                   s0 = pool_elt_at_index (per_thread_data->sessions,
1684                                           value0.value);
1685                   reass0->sess_index = value0.value;
1686                 }
1687               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1688             }
1689           else
1690             {
1691               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1692                 goto trace0;
1693               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1694                 {
1695                   if (nat_ip4_reass_add_fragment
1696                       (reass0, bi0, &fragments_to_drop))
1697                     {
1698                       b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1699                       nat_log_notice
1700                         ("maximum fragments per reassembly exceeded");
1701                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1702                       goto trace0;
1703                     }
1704                   cached0 = 1;
1705                   goto trace0;
1706                 }
1707               s0 = pool_elt_at_index (per_thread_data->sessions,
1708                                       reass0->sess_index);
1709             }
1710
1711           old_addr0 = ip0->dst_address.as_u32;
1712           ip0->dst_address = s0->in2out.addr;
1713           new_addr0 = ip0->dst_address.as_u32;
1714           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1715
1716           sum0 = ip0->checksum;
1717           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1718                                  ip4_header_t,
1719                                  dst_address /* changed member */ );
1720           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1721             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1722                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
1723                                    src_address);
1724           ip0->checksum = ip_csum_fold (sum0);
1725
1726           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1727             {
1728               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1729                 {
1730                   old_port0 = tcp0->dst_port;
1731                   tcp0->dst_port = s0->in2out.port;
1732                   new_port0 = tcp0->dst_port;
1733
1734                   sum0 = tcp0->checksum;
1735                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1736                                          ip4_header_t,
1737                                          dst_address /* changed member */ );
1738
1739                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1740                                          ip4_header_t /* cheat */ ,
1741                                          length /* changed member */ );
1742                   if (is_twice_nat_session (s0))
1743                     {
1744                       sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1745                                              s0->ext_host_nat_addr.as_u32,
1746                                              ip4_header_t, dst_address);
1747                       sum0 = ip_csum_update (sum0, tcp0->src_port,
1748                                              s0->ext_host_nat_port,
1749                                              ip4_header_t, length);
1750                       tcp0->src_port = s0->ext_host_nat_port;
1751                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1752                     }
1753                   tcp0->checksum = ip_csum_fold (sum0);
1754                 }
1755               else
1756                 {
1757                   old_port0 = udp0->dst_port;
1758                   udp0->dst_port = s0->in2out.port;
1759                   if (is_twice_nat_session (s0))
1760                     {
1761                       udp0->src_port = s0->ext_host_nat_port;
1762                       ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1763                     }
1764                   udp0->checksum = 0;
1765                 }
1766             }
1767
1768           /* Accounting */
1769           nat44_session_update_counters (s0, now,
1770                                          vlib_buffer_length_in_chain (vm,
1771                                                                       b0));
1772
1773         trace0:
1774           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1775                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1776             {
1777               nat44_reass_trace_t *t =
1778                 vlib_add_trace (vm, node, b0, sizeof (*t));
1779               t->cached = cached0;
1780               t->sw_if_index = sw_if_index0;
1781               t->next_index = next0;
1782             }
1783
1784           if (cached0)
1785             {
1786               n_left_to_next++;
1787               to_next--;
1788             }
1789           else
1790             {
1791               pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1792
1793               /* verify speculative enqueue, maybe switch current next frame */
1794               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1795                                                to_next, n_left_to_next,
1796                                                bi0, next0);
1797             }
1798
1799           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1800             {
1801               from = vlib_frame_vector_args (frame);
1802               u32 len = vec_len (fragments_to_loopback);
1803               if (len <= VLIB_FRAME_SIZE)
1804                 {
1805                   clib_memcpy (from, fragments_to_loopback,
1806                                sizeof (u32) * len);
1807                   n_left_from = len;
1808                   vec_reset_length (fragments_to_loopback);
1809                 }
1810               else
1811                 {
1812                   clib_memcpy (from,
1813                                fragments_to_loopback + (len -
1814                                                         VLIB_FRAME_SIZE),
1815                                sizeof (u32) * VLIB_FRAME_SIZE);
1816                   n_left_from = VLIB_FRAME_SIZE;
1817                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1818                 }
1819             }
1820         }
1821
1822       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1823     }
1824
1825   vlib_node_increment_counter (vm, nat44_ed_out2in_reass_node.index,
1826                                NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1827                                pkts_processed);
1828
1829   nat_send_all_to_node (vm, fragments_to_drop, node,
1830                         &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
1831                         NAT44_ED_OUT2IN_NEXT_DROP);
1832
1833   vec_free (fragments_to_drop);
1834   vec_free (fragments_to_loopback);
1835   return frame->n_vectors;
1836 }
1837
1838 /* *INDENT-OFF* */
1839 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
1840   .function = nat44_ed_out2in_reass_node_fn,
1841   .name = "nat44-ed-out2in-reass",
1842   .vector_size = sizeof (u32),
1843   .format_trace = format_nat44_reass_trace,
1844   .type = VLIB_NODE_TYPE_INTERNAL,
1845   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1846   .error_strings = nat_out2in_ed_error_strings,
1847   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1848   .next_nodes = {
1849     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1850     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1851     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1852     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1853     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1854     [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1855   },
1856 };
1857 /* *INDENT-ON* */
1858
1859 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_reass_node,
1860                               nat44_ed_out2in_reass_node_fn);
1861
1862 /*
1863  * fd.io coding-style-patch-verification: ON
1864  *
1865  * Local Variables:
1866  * eval: (c-set-style "gnu")
1867  * End:
1868  */