src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33
  34 static char *nat_in2out_ed_error_strings[] = {
  35 #define _(sym,string) string,
  36   foreach_nat_in2out_ed_error
  37 #undef _
  38 };
  39
  40 typedef struct
  41 {
  42   u32 sw_if_index;
  43   u32 next_index;
  44   u32 session_index;
  45   u32 is_slow_path;
  46 } nat_in2out_ed_trace_t;
  47
  48 static u8 *
  49 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  50 {
  51   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  52   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  53   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  54   char *tag;
  55
  56   tag =
  57     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  58     "NAT44_IN2OUT_ED_FAST_PATH";
  59
  60   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  61               t->sw_if_index, t->next_index, t->session_index);
  62
  63   return s;
  64 }
  65
  66 #ifndef CLIB_MARCH_VARIANT
  67 int
  68 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  69 {
  70   snat_main_t *sm = &snat_main;
  71   nat44_is_idle_session_ctx_t *ctx = arg;
  72   snat_session_t *s;
  73   u64 sess_timeout_time;
  74   nat_ed_ses_key_t ed_key;
  75   clib_bihash_kv_16_8_t ed_kv;
  76   int i;
  77   snat_address_t *a;
  78   snat_session_key_t key;
  79   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  80                                                        ctx->thread_index);
  81
  82   s = pool_elt_at_index (tsm->sessions, kv->value);
  83   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  84   if (ctx->now >= sess_timeout_time)
  85     {
  86       if (is_fwd_bypass_session (s))
  87         goto delete;
  88
  89       ed_key.l_addr = s->out2in.addr;
  90       ed_key.r_addr = s->ext_host_addr;
  91       ed_key.fib_index = s->out2in.fib_index;
  92       if (snat_is_unk_proto_session (s))
  93         {
  94           ed_key.proto = s->in2out.port;
  95           ed_key.r_port = 0;
  96           ed_key.l_port = 0;
  97         }
  98       else
  99         {
 100           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 101           ed_key.l_port = s->out2in.port;
 102           ed_key.r_port = s->ext_host_port;
 103         }
 104       ed_kv.key[0] = ed_key.as_u64[0];
 105       ed_kv.key[1] = ed_key.as_u64[1];
 106       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 107         nat_elog_warn ("out2in_ed key del failed");
 108
 109       if (snat_is_unk_proto_session (s))
 110         goto delete;
 111
 112       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 113                                            s->in2out.addr.as_u32,
 114                                            s->out2in.addr.as_u32,
 115                                            s->in2out.protocol,
 116                                            s->in2out.port,
 117                                            s->out2in.port,
 118                                            s->in2out.fib_index);
 119
 120       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 121                              &s->in2out.addr, s->in2out.port,
 122                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 123                              &s->out2in.addr, s->out2in.port,
 124                              &s->ext_host_addr, s->ext_host_port,
 125                              s->in2out.protocol, is_twice_nat_session (s));
 126
 127       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 128                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 129                    ctx->thread_index);
 130
 131       if (is_twice_nat_session (s))
 132         {
 133           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 134             {
 135               key.protocol = s->in2out.protocol;
 136               key.port = s->ext_host_nat_port;
 137               a = sm->twice_nat_addresses + i;
 138               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 139                 {
 140                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 141                                                       ctx->thread_index,
 142                                                       &key);
 143                   break;
 144                 }
 145             }
 146         }
 147
 148       if (snat_is_session_static (s))
 149         goto delete;
 150
 151       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 152                                           &s->out2in);
 153     delete:
 154       nat44_delete_session (sm, s, ctx->thread_index);
 155       return 1;
 156     }
 157
 158   return 0;
 159 }
 160 #endif
 161
 162 static inline u32
 163 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 164                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 165                           u32 sw_if_index0, u32 rx_fib_index0,
 166                           vlib_node_runtime_t * node, u32 next0, f64 now,
 167                           u32 thread_index, snat_session_t ** p_s0)
 168 {
 169   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 170                        next0, thread_index, p_s0, 0);
 171   snat_session_t *s0 = *p_s0;
 172   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 173     {
 174       /* Accounting */
 175       nat44_session_update_counters (s0, now,
 176                                      vlib_buffer_length_in_chain
 177                                      (sm->vlib_main, b0), thread_index);
 178       /* Per-user LRU list maintenance */
 179       nat44_session_update_lru (sm, s0, thread_index);
 180     }
 181   return next0;
 182 }
 183
 184 static u32
 185 slow_path_ed (snat_main_t * sm,
 186               vlib_buffer_t * b,
 187               u32 rx_fib_index,
 188               clib_bihash_kv_16_8_t * kv,
 189               snat_session_t ** sessionp,
 190               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now,
 191               tcp_header_t * tcp)
 192 {
 193   snat_session_t *s = 0;
 194   snat_user_t *u;
 195   snat_session_key_t key0, key1;
 196   lb_nat_type_t lb = 0, is_sm = 0;
 197   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 198   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 199   u32 proto = ip_proto_to_snat_proto (key->proto);
 200   nat_outside_fib_t *outside_fib;
 201   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 202   u8 identity_nat;
 203   fib_prefix_t pfx = {
 204     .fp_proto = FIB_PROTOCOL_IP4,
 205     .fp_len = 32,
 206     .fp_addr = {
 207                 .ip4.as_u32 = key->r_addr.as_u32,
 208                 },
 209   };
 210   nat44_is_idle_session_ctx_t ctx;
 211
 212   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 213     {
 214       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 215       nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 216       nat_elog_notice ("maximum sessions exceeded");
 217       return NAT_NEXT_DROP;
 218     }
 219
 220   key0.addr = key->l_addr;
 221   key0.port = key->l_port;
 222   key1.protocol = key0.protocol = proto;
 223   key0.fib_index = rx_fib_index;
 224   key1.fib_index = sm->outside_fib_index;
 225   /* First try to match static mapping by local address and port */
 226   if (snat_static_mapping_match
 227       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 228     {
 229       /* Try to create dynamic translation */
 230       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 231                                                thread_index, &key1,
 232                                                sm->port_per_thread,
 233                                                tsm->snat_thread_index))
 234         {
 235           nat_elog_notice ("addresses exhausted");
 236           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 237           return NAT_NEXT_DROP;
 238         }
 239     }
 240   else
 241     {
 242       if (PREDICT_FALSE (identity_nat))
 243         {
 244           *sessionp = s;
 245           return next;
 246         }
 247
 248       is_sm = 1;
 249     }
 250
 251   if (proto == SNAT_PROTOCOL_TCP)
 252     {
 253       if (!tcp_is_init (tcp))
 254         {
 255           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 256           return NAT_NEXT_DROP;
 257         }
 258     }
 259
 260   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 261   if (!u)
 262     {
 263       nat_elog_warn ("create NAT user failed");
 264       if (!is_sm)
 265         snat_free_outside_address_and_port (sm->addresses,
 266                                             thread_index, &key1);
 267       return NAT_NEXT_DROP;
 268     }
 269
 270   s = nat_ed_session_alloc (sm, u, thread_index, now);
 271   if (!s)
 272     {
 273       nat44_delete_user_with_no_session (sm, u, thread_index);
 274       nat_elog_warn ("create NAT session failed");
 275       if (!is_sm)
 276         snat_free_outside_address_and_port (sm->addresses,
 277                                             thread_index, &key1);
 278       return NAT_NEXT_DROP;
 279     }
 280
 281   user_session_increment (sm, u, is_sm);
 282   if (is_sm)
 283     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 284   if (lb)
 285     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 286   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 287   s->ext_host_addr = key->r_addr;
 288   s->ext_host_port = key->r_port;
 289   s->in2out = key0;
 290   s->out2in = key1;
 291   s->out2in.protocol = key0.protocol;
 292
 293   switch (vec_len (sm->outside_fibs))
 294     {
 295     case 0:
 296       s->out2in.fib_index = sm->outside_fib_index;
 297       break;
 298     case 1:
 299       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 300       break;
 301     default:
 302       /* *INDENT-OFF* */
 303       vec_foreach (outside_fib, sm->outside_fibs)
 304        {
 305           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 306           if (FIB_NODE_INDEX_INVALID != fei)
 307             {
 308               if (fib_entry_get_resolving_interface (fei) != ~0)
 309                 {
 310                   s->out2in.fib_index = outside_fib->fib_index;
 311                   break;
 312                 }
 313             }
 314         }
 315       /* *INDENT-ON* */
 316       break;
 317     }
 318
 319   /* Add to lookup tables */
 320   kv->value = s - tsm->sessions;
 321   ctx.now = now;
 322   ctx.thread_index = thread_index;
 323   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 324                                                nat44_i2o_ed_is_idle_session_cb,
 325                                                &ctx))
 326     nat_elog_notice ("in2out-ed key add failed");
 327
 328   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 329               key1.port, key->r_port);
 330   kv->value = s - tsm->sessions;
 331   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 332                                                nat44_o2i_ed_is_idle_session_cb,
 333                                                &ctx))
 334     nat_elog_notice ("out2in-ed key add failed");
 335
 336   *sessionp = s;
 337
 338   /* log NAT event */
 339   snat_ipfix_logging_nat44_ses_create (thread_index,
 340                                        s->in2out.addr.as_u32,
 341                                        s->out2in.addr.as_u32,
 342                                        s->in2out.protocol,
 343                                        s->in2out.port,
 344                                        s->out2in.port, s->in2out.fib_index);
 345
 346   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 347                          &s->in2out.addr, s->in2out.port,
 348                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 349                          &s->out2in.addr, s->out2in.port,
 350                          &s->ext_host_addr, s->ext_host_port,
 351                          s->in2out.protocol, 0);
 352
 353   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 354                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 355                &s->ext_host_nat_addr, s->ext_host_nat_port,
 356                s->in2out.protocol, s->in2out.fib_index, s->flags,
 357                thread_index, 0);
 358
 359   return next;
 360 }
 361
 362 static_always_inline int
 363 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 364                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 365                         u32 rx_fib_index, u32 thread_index)
 366 {
 367   udp_header_t *udp = ip4_next_header (ip);
 368   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 369   clib_bihash_kv_16_8_t kv, value;
 370   snat_session_key_t key0, key1;
 371
 372   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 373               sm->outside_fib_index, udp->dst_port, udp->src_port);
 374
 375   /* NAT packet aimed at external address if */
 376   /* has active sessions */
 377   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 378     {
 379       key0.addr = ip->dst_address;
 380       key0.port = udp->dst_port;
 381       key0.protocol = proto;
 382       key0.fib_index = sm->outside_fib_index;
 383       /* or is static mappings */
 384       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 385         return 0;
 386     }
 387   else
 388     return 0;
 389
 390   if (sm->forwarding_enabled)
 391     return 1;
 392
 393   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 394                                   rx_fib_index);
 395 }
 396
 397 static_always_inline int
 398 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 399                                       u32 thread_index, f64 now,
 400                                       vlib_main_t * vm, vlib_buffer_t * b)
 401 {
 402   nat_ed_ses_key_t key;
 403   clib_bihash_kv_16_8_t kv, value;
 404   udp_header_t *udp;
 405   snat_session_t *s = 0;
 406   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 407
 408   if (!sm->forwarding_enabled)
 409     return 0;
 410
 411   if (ip->protocol == IP_PROTOCOL_ICMP)
 412     {
 413       key.as_u64[0] = key.as_u64[1] = 0;
 414       if (get_icmp_i2o_ed_key (ip, &key))
 415         return 0;
 416       key.fib_index = 0;
 417       kv.key[0] = key.as_u64[0];
 418       kv.key[1] = key.as_u64[1];
 419     }
 420   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 421     {
 422       udp = ip4_next_header (ip);
 423       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 424                   udp->src_port, udp->dst_port);
 425     }
 426   else
 427     {
 428       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 429                   0);
 430     }
 431
 432   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 433     {
 434       s = pool_elt_at_index (tsm->sessions, value.value);
 435       if (is_fwd_bypass_session (s))
 436         {
 437           if (ip->protocol == IP_PROTOCOL_TCP)
 438             {
 439               tcp_header_t *tcp = ip4_next_header (ip);
 440               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 441                 return 1;
 442             }
 443           /* Accounting */
 444           nat44_session_update_counters (s, now,
 445                                          vlib_buffer_length_in_chain (vm, b),
 446                                          thread_index);
 447           /* Per-user LRU list maintenance */
 448           nat44_session_update_lru (sm, s, thread_index);
 449           return 1;
 450         }
 451       else
 452         return 0;
 453     }
 454
 455   return 0;
 456 }
 457
 458 static_always_inline int
 459 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 460                                        u8 proto, u16 src_port, u16 dst_port,
 461                                        u32 thread_index, u32 rx_sw_if_index,
 462                                        u32 tx_sw_if_index)
 463 {
 464   clib_bihash_kv_16_8_t kv, value;
 465   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 466   snat_interface_t *i;
 467   snat_session_t *s;
 468   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 469   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 470
 471   /* src NAT check */
 472   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 473               src_port, dst_port);
 474   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 475     {
 476       s = pool_elt_at_index (tsm->sessions, value.value);
 477       if (nat44_is_ses_closed (s))
 478         {
 479           nat_free_session_data (sm, s, thread_index, 0);
 480           nat44_delete_session (sm, s, thread_index);
 481         }
 482       else
 483         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 484       return 1;
 485     }
 486
 487   /* dst NAT check */
 488   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 489               dst_port, src_port);
 490   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 491     {
 492       s = pool_elt_at_index (tsm->sessions, value.value);
 493       if (is_fwd_bypass_session (s))
 494         return 0;
 495
 496       /* hairpinning */
 497       /* *INDENT-OFF* */
 498       pool_foreach (i, sm->output_feature_interfaces,
 499       ({
 500         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 501            return 0;
 502       }));
 503       /* *INDENT-ON* */
 504       return 1;
 505     }
 506
 507   return 0;
 508 }
 509
 510 #ifndef CLIB_MARCH_VARIANT
 511 u32
 512 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 513                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 514                       u8 * p_proto, snat_session_key_t * p_value,
 515                       u8 * p_dont_translate, void *d, void *e)
 516 {
 517   icmp46_header_t *icmp;
 518   u32 sw_if_index;
 519   u32 rx_fib_index;
 520   nat_ed_ses_key_t key;
 521   snat_session_t *s = 0;
 522   u8 dont_translate = 0;
 523   clib_bihash_kv_16_8_t kv, value;
 524   u32 next = ~0;
 525   int err;
 526   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 527
 528   icmp = (icmp46_header_t *) ip4_next_header (ip);
 529   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 530   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 531
 532   key.as_u64[0] = key.as_u64[1] = 0;
 533   err = get_icmp_i2o_ed_key (ip, &key);
 534   if (err != 0)
 535     {
 536       b->error = node->errors[err];
 537       next = NAT_NEXT_DROP;
 538       goto out;
 539     }
 540   key.fib_index = rx_fib_index;
 541
 542   kv.key[0] = key.as_u64[0];
 543   kv.key[1] = key.as_u64[1];
 544
 545   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 546     {
 547       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 548         {
 549           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 550                                                                     key.proto,
 551                                                                     key.
 552                                                                     l_port,
 553                                                                     key.
 554                                                                     r_port,
 555                                                                     thread_index,
 556                                                                     sw_if_index,
 557                                                                     vnet_buffer
 558                                                                     (b)->
 559                                                                     sw_if_index
 560                                                                     [VLIB_TX])))
 561             {
 562               dont_translate = 1;
 563               goto out;
 564             }
 565         }
 566       else
 567         {
 568           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 569                                                      ip, SNAT_PROTOCOL_ICMP,
 570                                                      rx_fib_index,
 571                                                      thread_index)))
 572             {
 573               dont_translate = 1;
 574               goto out;
 575             }
 576         }
 577
 578       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 579         {
 580           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 581           next = NAT_NEXT_DROP;
 582           goto out;
 583         }
 584
 585       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 586                            thread_index, vlib_time_now (sm->vlib_main), 0);
 587
 588       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 589         goto out;
 590
 591       if (!s)
 592         {
 593           dont_translate = 1;
 594           goto out;
 595         }
 596     }
 597   else
 598     {
 599       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 600                          icmp->type != ICMP4_echo_reply &&
 601                          !icmp_is_error_message (icmp)))
 602         {
 603           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 604           next = NAT_NEXT_DROP;
 605           goto out;
 606         }
 607
 608       s = pool_elt_at_index (tsm->sessions, value.value);
 609     }
 610
 611   *p_proto = ip_proto_to_snat_proto (key.proto);
 612 out:
 613   if (s)
 614     *p_value = s->out2in;
 615   *p_dont_translate = dont_translate;
 616   if (d)
 617     *(snat_session_t **) d = s;
 618   return next;
 619 }
 620 #endif
 621
 622 static snat_session_t *
 623 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 624                                vlib_buffer_t * b,
 625                                ip4_header_t * ip,
 626                                u32 rx_fib_index,
 627                                u32 thread_index,
 628                                f64 now,
 629                                vlib_main_t * vm, vlib_node_runtime_t * node)
 630 {
 631   clib_bihash_kv_8_8_t kv, value;
 632   clib_bihash_kv_16_8_t s_kv, s_value;
 633   snat_static_mapping_t *m;
 634   u32 old_addr, new_addr = 0;
 635   ip_csum_t sum;
 636   snat_user_t *u;
 637   dlist_elt_t *head, *elt;
 638   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 639   u32 elt_index, head_index, ses_index;
 640   snat_session_t *s;
 641   u32 outside_fib_index = sm->outside_fib_index;
 642   int i;
 643   u8 is_sm = 0;
 644   nat_outside_fib_t *outside_fib;
 645   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 646   fib_prefix_t pfx = {
 647     .fp_proto = FIB_PROTOCOL_IP4,
 648     .fp_len = 32,
 649     .fp_addr = {
 650                 .ip4.as_u32 = ip->dst_address.as_u32,
 651                 },
 652   };
 653
 654   switch (vec_len (sm->outside_fibs))
 655     {
 656     case 0:
 657       outside_fib_index = sm->outside_fib_index;
 658       break;
 659     case 1:
 660       outside_fib_index = sm->outside_fibs[0].fib_index;
 661       break;
 662     default:
 663       /* *INDENT-OFF* */
 664       vec_foreach (outside_fib, sm->outside_fibs)
 665         {
 666           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 667           if (FIB_NODE_INDEX_INVALID != fei)
 668             {
 669               if (fib_entry_get_resolving_interface (fei) != ~0)
 670                 {
 671                   outside_fib_index = outside_fib->fib_index;
 672                   break;
 673                 }
 674             }
 675         }
 676       /* *INDENT-ON* */
 677       break;
 678     }
 679   old_addr = ip->src_address.as_u32;
 680
 681   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 682               rx_fib_index, 0, 0);
 683
 684   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 685     {
 686       s = pool_elt_at_index (tsm->sessions, s_value.value);
 687       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 688     }
 689   else
 690     {
 691       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 692         {
 693           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 694           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 695           nat_elog_notice ("maximum sessions exceeded");
 696           return 0;
 697         }
 698
 699       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 700                                   thread_index);
 701       if (!u)
 702         {
 703           nat_elog_warn ("create NAT user failed");
 704           return 0;
 705         }
 706
 707       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 708
 709       /* Try to find static mapping first */
 710       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 711         {
 712           m = pool_elt_at_index (sm->static_mappings, value.value);
 713           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 714           is_sm = 1;
 715           goto create_ses;
 716         }
 717       /* Fallback to 3-tuple key */
 718       else
 719         {
 720           /* Choose same out address as for TCP/UDP session to same destination */
 721           head_index = u->sessions_per_user_list_head_index;
 722           head = pool_elt_at_index (tsm->list_pool, head_index);
 723           elt_index = head->next;
 724           if (PREDICT_FALSE (elt_index == ~0))
 725             ses_index = ~0;
 726           else
 727             {
 728               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 729               ses_index = elt->value;
 730             }
 731
 732           while (ses_index != ~0)
 733             {
 734               s = pool_elt_at_index (tsm->sessions, ses_index);
 735               elt_index = elt->next;
 736               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 737               ses_index = elt->value;
 738
 739               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 740                 {
 741                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 742
 743                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 744                               ip->protocol, outside_fib_index, 0, 0);
 745                   if (clib_bihash_search_16_8
 746                       (&tsm->out2in_ed, &s_kv, &s_value))
 747                     goto create_ses;
 748
 749                   break;
 750                 }
 751             }
 752
 753           for (i = 0; i < vec_len (sm->addresses); i++)
 754             {
 755               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 756                           ip->protocol, outside_fib_index, 0, 0);
 757               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 758                 {
 759                   new_addr = ip->src_address.as_u32 =
 760                     sm->addresses[i].addr.as_u32;
 761                   goto create_ses;
 762                 }
 763             }
 764           return 0;
 765         }
 766
 767     create_ses:
 768       s = nat_ed_session_alloc (sm, u, thread_index, now);
 769       if (!s)
 770         {
 771           nat44_delete_user_with_no_session (sm, u, thread_index);
 772           nat_elog_warn ("create NAT session failed");
 773           return 0;
 774         }
 775
 776       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 777       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 778       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 779       s->out2in.addr.as_u32 = new_addr;
 780       s->out2in.fib_index = outside_fib_index;
 781       s->in2out.addr.as_u32 = old_addr;
 782       s->in2out.fib_index = rx_fib_index;
 783       s->in2out.port = s->out2in.port = ip->protocol;
 784       if (is_sm)
 785         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 786       user_session_increment (sm, u, is_sm);
 787
 788       /* Add to lookup tables */
 789       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 790                   rx_fib_index, 0, 0);
 791       s_kv.value = s - tsm->sessions;
 792       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 793         nat_elog_notice ("in2out key add failed");
 794
 795       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 796                   outside_fib_index, 0, 0);
 797       s_kv.value = s - tsm->sessions;
 798       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 799         nat_elog_notice ("out2in key add failed");
 800     }
 801
 802   /* Update IP checksum */
 803   sum = ip->checksum;
 804   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 805   ip->checksum = ip_csum_fold (sum);
 806
 807   /* Accounting */
 808   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 809                                  thread_index);
 810   /* Per-user LRU list maintenance */
 811   nat44_session_update_lru (sm, s, thread_index);
 812
 813   /* Hairpinning */
 814   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 815     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 816
 817   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 818     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 819
 820   return s;
 821 }
 822
 823 static inline uword
 824 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 825                                 vlib_node_runtime_t * node,
 826                                 vlib_frame_t * frame, int is_slow_path,
 827                                 int is_output_feature)
 828 {
 829   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 830   nat_next_t next_index;
 831   snat_main_t *sm = &snat_main;
 832   f64 now = vlib_time_now (vm);
 833   u32 thread_index = vm->thread_index;
 834   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 835   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 836     0, fragments = 0, def_slow, def_reass;
 837
 838   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 839     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 840
 841   def_reass = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_REASS :
 842     NAT_NEXT_IN2OUT_ED_REASS;
 843
 844   stats_node_index = is_slow_path ? sm->ed_in2out_slowpath_node_index :
 845     sm->ed_in2out_node_index;
 846
 847   from = vlib_frame_vector_args (frame);
 848   n_left_from = frame->n_vectors;
 849   next_index = node->cached_next_index;
 850
 851   while (n_left_from > 0)
 852     {
 853       u32 n_left_to_next;
 854
 855       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 856
 857       while (n_left_from >= 4 && n_left_to_next >= 2)
 858         {
 859           u32 bi0, bi1;
 860           vlib_buffer_t *b0, *b1;
 861           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 862             new_addr0, old_addr0;
 863           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 864             new_addr1, old_addr1;
 865           u16 old_port0, new_port0, old_port1, new_port1;
 866           ip4_header_t *ip0, *ip1;
 867           udp_header_t *udp0, *udp1;
 868           tcp_header_t *tcp0, *tcp1;
 869           icmp46_header_t *icmp0, *icmp1;
 870           snat_session_t *s0 = 0, *s1 = 0;
 871           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 872           ip_csum_t sum0, sum1;
 873
 874           /* Prefetch next iteration. */
 875           {
 876             vlib_buffer_t *p2, *p3;
 877
 878             p2 = vlib_get_buffer (vm, from[2]);
 879             p3 = vlib_get_buffer (vm, from[3]);
 880
 881             vlib_prefetch_buffer_header (p2, LOAD);
 882             vlib_prefetch_buffer_header (p3, LOAD);
 883
 884             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 885             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 886           }
 887
 888           /* speculatively enqueue b0 and b1 to the current next frame */
 889           to_next[0] = bi0 = from[0];
 890           to_next[1] = bi1 = from[1];
 891           from += 2;
 892           to_next += 2;
 893           n_left_from -= 2;
 894           n_left_to_next -= 2;
 895
 896           b0 = vlib_get_buffer (vm, bi0);
 897           b1 = vlib_get_buffer (vm, bi1);
 898
 899           if (is_output_feature)
 900             {
 901               // output feature fast path is enabled on the arc
 902               // we need new arc_next feature
 903               if (PREDICT_TRUE (!is_slow_path))
 904                 {
 905                   vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0);
 906                   vnet_feature_next (&nat_buffer_opaque (b1)->arc_next, b1);
 907                 }
 908
 909               iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 910               iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
 911             }
 912
 913           next0 = nat_buffer_opaque (b0)->arc_next;
 914           next1 = nat_buffer_opaque (b1)->arc_next;
 915
 916           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 917                                   iph_offset0);
 918
 919           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 920           rx_fib_index0 =
 921             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 922                                                  sw_if_index0);
 923
 924           if (PREDICT_FALSE (ip0->ttl == 1))
 925             {
 926               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 927               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 928                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 929                                            0);
 930               next0 = NAT_NEXT_ICMP_ERROR;
 931               goto trace00;
 932             }
 933
 934           udp0 = ip4_next_header (ip0);
 935           tcp0 = (tcp_header_t *) udp0;
 936           icmp0 = (icmp46_header_t *) udp0;
 937           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 938
 939           if (is_slow_path)
 940             {
 941               if (PREDICT_FALSE (proto0 == ~0))
 942                 {
 943                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
 944                                                       rx_fib_index0,
 945                                                       thread_index, now, vm,
 946                                                       node);
 947                   if (!s0)
 948                     next0 = NAT_NEXT_DROP;
 949                   other_packets++;
 950                   goto trace00;
 951                 }
 952
 953               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 954                 {
 955                   next0 = icmp_in2out_ed_slow_path
 956                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 957                      next0, now, thread_index, &s0);
 958                   icmp_packets++;
 959                   goto trace00;
 960                 }
 961             }
 962           else
 963             {
 964               if (PREDICT_FALSE (proto0 == ~0))
 965                 {
 966                   next0 = def_slow;
 967                   goto trace00;
 968                 }
 969
 970               if (ip4_is_fragment (ip0))
 971                 {
 972                   next0 = def_reass;
 973                   fragments++;
 974                   goto trace00;
 975                 }
 976
 977               if (is_output_feature)
 978                 {
 979                   if (PREDICT_FALSE
 980                       (nat_not_translate_output_feature_fwd
 981                        (sm, ip0, thread_index, now, vm, b0)))
 982                     goto trace00;
 983                 }
 984
 985               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 986                 {
 987                   next0 = def_slow;
 988                   goto trace00;
 989                 }
 990             }
 991
 992           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
 993                       ip0->protocol, rx_fib_index0, udp0->src_port,
 994                       udp0->dst_port);
 995
 996           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
 997             {
 998               if (is_slow_path)
 999                 {
1000                   if (is_output_feature)
1001                     {
1002                       if (PREDICT_FALSE
1003                           (nat44_ed_not_translate_output_feature
1004                            (sm, ip0, ip0->protocol, udp0->src_port,
1005                             udp0->dst_port, thread_index, sw_if_index0,
1006                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1007                         goto trace00;
1008                     }
1009                   else
1010                     {
1011                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1012                                                                  sw_if_index0,
1013                                                                  ip0, proto0,
1014                                                                  rx_fib_index0,
1015                                                                  thread_index)))
1016                         goto trace00;
1017                     }
1018
1019                   next0 =
1020                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1021                                   next0, thread_index, now, tcp0);
1022
1023                   if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1024                     goto trace00;
1025
1026                   if (PREDICT_FALSE (!s0))
1027                     goto trace00;
1028                 }
1029               else
1030                 {
1031                   next0 = def_slow;
1032                   goto trace00;
1033                 }
1034             }
1035           else
1036             {
1037               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1038             }
1039
1040           b0->flags |= VNET_BUFFER_F_IS_NATED;
1041
1042           if (!is_output_feature)
1043             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1044
1045           old_addr0 = ip0->src_address.as_u32;
1046           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1047           sum0 = ip0->checksum;
1048           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1049                                  src_address);
1050           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1051             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1052                                    s0->ext_host_addr.as_u32, ip4_header_t,
1053                                    dst_address);
1054           ip0->checksum = ip_csum_fold (sum0);
1055
1056           old_port0 = udp0->src_port;
1057           new_port0 = udp0->src_port = s0->out2in.port;
1058
1059           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1060             {
1061               sum0 = tcp0->checksum;
1062               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1063                                      dst_address);
1064               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1065                                      length);
1066               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1067                 {
1068                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1069                                          s0->ext_host_addr.as_u32,
1070                                          ip4_header_t, dst_address);
1071                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1072                                          s0->ext_host_port, ip4_header_t,
1073                                          length);
1074                   tcp0->dst_port = s0->ext_host_port;
1075                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1076                 }
1077               mss_clamping (sm, tcp0, &sum0);
1078               tcp0->checksum = ip_csum_fold (sum0);
1079               tcp_packets++;
1080               if (nat44_set_tcp_session_state_i2o
1081                   (sm, s0, tcp0, thread_index))
1082                 goto trace00;
1083             }
1084           else if (udp0->checksum)
1085             {
1086               sum0 = udp0->checksum;
1087               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1088                                      dst_address);
1089               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1090                                      length);
1091               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1092                 {
1093                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1094                                          s0->ext_host_addr.as_u32,
1095                                          ip4_header_t, dst_address);
1096                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1097                                          s0->ext_host_port, ip4_header_t,
1098                                          length);
1099                   udp0->dst_port = s0->ext_host_port;
1100                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1101                 }
1102               udp0->checksum = ip_csum_fold (sum0);
1103               udp_packets++;
1104             }
1105           else
1106             {
1107               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1108                 {
1109                   udp0->dst_port = s0->ext_host_port;
1110                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1111                 }
1112               udp_packets++;
1113             }
1114
1115           /* Accounting */
1116           nat44_session_update_counters (s0, now,
1117                                          vlib_buffer_length_in_chain (vm,
1118                                                                       b0),
1119                                          thread_index);
1120           /* Per-user LRU list maintenance */
1121           nat44_session_update_lru (sm, s0, thread_index);
1122
1123         trace00:
1124           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1125                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1126             {
1127               nat_in2out_ed_trace_t *t =
1128                 vlib_add_trace (vm, node, b0, sizeof (*t));
1129               t->is_slow_path = is_slow_path;
1130               t->sw_if_index = sw_if_index0;
1131               t->next_index = next0;
1132               t->session_index = ~0;
1133               if (s0)
1134                 t->session_index = s0 - tsm->sessions;
1135             }
1136
1137           pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1138
1139           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1140                                   iph_offset1);
1141
1142           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1143           rx_fib_index1 =
1144             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1145                                                  sw_if_index1);
1146
1147           if (PREDICT_FALSE (ip1->ttl == 1))
1148             {
1149               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1150               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1151                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1152                                            0);
1153               next1 = NAT_NEXT_ICMP_ERROR;
1154               goto trace01;
1155             }
1156
1157           udp1 = ip4_next_header (ip1);
1158           tcp1 = (tcp_header_t *) udp1;
1159           icmp1 = (icmp46_header_t *) udp1;
1160           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1161
1162           if (is_slow_path)
1163             {
1164               if (PREDICT_FALSE (proto1 == ~0))
1165                 {
1166                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1167                                                       rx_fib_index1,
1168                                                       thread_index, now, vm,
1169                                                       node);
1170                   if (!s1)
1171                     next1 = NAT_NEXT_DROP;
1172                   other_packets++;
1173                   goto trace01;
1174                 }
1175
1176               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1177                 {
1178                   next1 = icmp_in2out_ed_slow_path
1179                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1180                      next1, now, thread_index, &s1);
1181                   icmp_packets++;
1182                   goto trace01;
1183                 }
1184             }
1185           else
1186             {
1187               if (PREDICT_FALSE (proto1 == ~0))
1188                 {
1189                   next1 = def_slow;
1190                   goto trace01;
1191                 }
1192
1193               if (ip4_is_fragment (ip1))
1194                 {
1195                   next1 = def_reass;
1196                   fragments++;
1197                   goto trace01;
1198                 }
1199
1200               if (is_output_feature)
1201                 {
1202                   if (PREDICT_FALSE
1203                       (nat_not_translate_output_feature_fwd
1204                        (sm, ip1, thread_index, now, vm, b1)))
1205                     goto trace01;
1206                 }
1207
1208               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1209                 {
1210                   next1 = def_slow;
1211                   goto trace01;
1212                 }
1213             }
1214
1215           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1216                       ip1->protocol, rx_fib_index1, udp1->src_port,
1217                       udp1->dst_port);
1218
1219           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1220             {
1221               if (is_slow_path)
1222                 {
1223                   if (is_output_feature)
1224                     {
1225                       if (PREDICT_FALSE
1226                           (nat44_ed_not_translate_output_feature
1227                            (sm, ip1, ip1->protocol, udp1->src_port,
1228                             udp1->dst_port, thread_index, sw_if_index1,
1229                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1230                         goto trace01;
1231                     }
1232                   else
1233                     {
1234                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1235                                                                  sw_if_index1,
1236                                                                  ip1, proto1,
1237                                                                  rx_fib_index1,
1238                                                                  thread_index)))
1239                         goto trace01;
1240                     }
1241
1242                   next1 =
1243                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1244                                   next1, thread_index, now, tcp1);
1245
1246                   if (PREDICT_FALSE (next1 == NAT_NEXT_DROP))
1247                     goto trace01;
1248
1249                   if (PREDICT_FALSE (!s1))
1250                     goto trace01;
1251                 }
1252               else
1253                 {
1254                   next1 = def_slow;
1255                   goto trace01;
1256                 }
1257             }
1258           else
1259             {
1260               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1261             }
1262
1263           b1->flags |= VNET_BUFFER_F_IS_NATED;
1264
1265           if (!is_output_feature)
1266             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1267
1268           old_addr1 = ip1->src_address.as_u32;
1269           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1270           sum1 = ip1->checksum;
1271           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1272                                  src_address);
1273           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1274             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1275                                    s1->ext_host_addr.as_u32, ip4_header_t,
1276                                    dst_address);
1277           ip1->checksum = ip_csum_fold (sum1);
1278
1279           old_port1 = udp1->src_port;
1280           new_port1 = udp1->src_port = s1->out2in.port;
1281
1282           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1283             {
1284               sum1 = tcp1->checksum;
1285               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1286                                      dst_address);
1287               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1288                                      length);
1289               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1290                 {
1291                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1292                                          s1->ext_host_addr.as_u32,
1293                                          ip4_header_t, dst_address);
1294                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1295                                          s1->ext_host_port, ip4_header_t,
1296                                          length);
1297                   tcp1->dst_port = s1->ext_host_port;
1298                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1299                 }
1300               tcp1->checksum = ip_csum_fold (sum1);
1301               mss_clamping (sm, tcp1, &sum1);
1302               tcp_packets++;
1303               if (nat44_set_tcp_session_state_i2o
1304                   (sm, s1, tcp1, thread_index))
1305                 goto trace01;
1306             }
1307           else if (udp1->checksum)
1308             {
1309               sum1 = udp1->checksum;
1310               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1311                                      dst_address);
1312               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1313                                      length);
1314
1315               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1316                 {
1317                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1318                                          s1->ext_host_addr.as_u32,
1319                                          ip4_header_t, dst_address);
1320                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1321                                          s1->ext_host_port, ip4_header_t,
1322                                          length);
1323                   udp1->dst_port = s1->ext_host_port;
1324                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1325                 }
1326               udp1->checksum = ip_csum_fold (sum1);
1327               udp_packets++;
1328             }
1329           else
1330             {
1331               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1332                 {
1333                   udp1->dst_port = s1->ext_host_port;
1334                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1335                 }
1336               udp_packets++;
1337             }
1338
1339           /* Accounting */
1340           nat44_session_update_counters (s1, now,
1341                                          vlib_buffer_length_in_chain (vm, b1),
1342                                          thread_index);
1343           /* Per-user LRU list maintenance */
1344           nat44_session_update_lru (sm, s1, thread_index);
1345
1346         trace01:
1347           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1348                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1349             {
1350               nat_in2out_ed_trace_t *t =
1351                 vlib_add_trace (vm, node, b1, sizeof (*t));
1352               t->is_slow_path = is_slow_path;
1353               t->sw_if_index = sw_if_index1;
1354               t->next_index = next1;
1355               t->session_index = ~0;
1356               if (s1)
1357                 t->session_index = s1 - tsm->sessions;
1358             }
1359
1360           pkts_processed += next1 == nat_buffer_opaque (b1)->arc_next;
1361
1362
1363           /* verify speculative enqueues, maybe switch current next frame */
1364           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1365                                            to_next, n_left_to_next,
1366                                            bi0, bi1, next0, next1);
1367         }
1368
1369       while (n_left_from > 0 && n_left_to_next > 0)
1370         {
1371           u32 bi0;
1372           vlib_buffer_t *b0;
1373           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1374             new_addr0, old_addr0;
1375           u16 old_port0, new_port0;
1376           ip4_header_t *ip0;
1377           udp_header_t *udp0;
1378           tcp_header_t *tcp0;
1379           icmp46_header_t *icmp0;
1380           snat_session_t *s0 = 0;
1381           clib_bihash_kv_16_8_t kv0, value0;
1382           ip_csum_t sum0;
1383
1384           /* speculatively enqueue b0 to the current next frame */
1385           bi0 = from[0];
1386           to_next[0] = bi0;
1387           from += 1;
1388           to_next += 1;
1389           n_left_from -= 1;
1390           n_left_to_next -= 1;
1391
1392           b0 = vlib_get_buffer (vm, bi0);
1393
1394           if (is_output_feature)
1395             {
1396               // output feature fast path is enabled on the arc
1397               // we need new arc_next feature
1398               if (PREDICT_TRUE (!is_slow_path))
1399                 vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0);
1400
1401               iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1402             }
1403
1404           next0 = nat_buffer_opaque (b0)->arc_next;
1405
1406           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1407                                   iph_offset0);
1408
1409           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1410           rx_fib_index0 =
1411             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1412                                                  sw_if_index0);
1413
1414           if (PREDICT_FALSE (ip0->ttl == 1))
1415             {
1416               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1417               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1418                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1419                                            0);
1420               next0 = NAT_NEXT_ICMP_ERROR;
1421               goto trace0;
1422             }
1423
1424           udp0 = ip4_next_header (ip0);
1425           tcp0 = (tcp_header_t *) udp0;
1426           icmp0 = (icmp46_header_t *) udp0;
1427           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1428
1429           if (is_slow_path)
1430             {
1431               if (PREDICT_FALSE (proto0 == ~0))
1432                 {
1433                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1434                                                       rx_fib_index0,
1435                                                       thread_index, now, vm,
1436                                                       node);
1437                   if (!s0)
1438                     next0 = NAT_NEXT_DROP;
1439                   other_packets++;
1440                   goto trace0;
1441                 }
1442
1443               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1444                 {
1445                   next0 = icmp_in2out_ed_slow_path
1446                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1447                      next0, now, thread_index, &s0);
1448                   icmp_packets++;
1449                   goto trace0;
1450                 }
1451             }
1452           else
1453             {
1454               if (PREDICT_FALSE (proto0 == ~0))
1455                 {
1456                   next0 = def_slow;
1457                   goto trace0;
1458                 }
1459
1460               if (ip4_is_fragment (ip0))
1461                 {
1462                   next0 = def_reass;
1463                   fragments++;
1464                   goto trace0;
1465                 }
1466
1467               if (is_output_feature)
1468                 {
1469                   if (PREDICT_FALSE
1470                       (nat_not_translate_output_feature_fwd
1471                        (sm, ip0, thread_index, now, vm, b0)))
1472                     goto trace0;
1473                 }
1474
1475               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1476                 {
1477                   next0 = def_slow;
1478                   goto trace0;
1479                 }
1480             }
1481
1482           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1483                       ip0->protocol, rx_fib_index0, udp0->src_port,
1484                       udp0->dst_port);
1485
1486           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1487             {
1488               if (is_slow_path)
1489                 {
1490                   if (is_output_feature)
1491                     {
1492                       if (PREDICT_FALSE
1493                           (nat44_ed_not_translate_output_feature
1494                            (sm, ip0, ip0->protocol, udp0->src_port,
1495                             udp0->dst_port, thread_index, sw_if_index0,
1496                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1497                         goto trace0;
1498                     }
1499                   else
1500                     {
1501                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1502                                                                  sw_if_index0,
1503                                                                  ip0, proto0,
1504                                                                  rx_fib_index0,
1505                                                                  thread_index)))
1506                         goto trace0;
1507                     }
1508
1509                   next0 =
1510                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1511                                   next0, thread_index, now, tcp0);
1512
1513                   if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1514                     goto trace0;
1515
1516                   if (PREDICT_FALSE (!s0))
1517                     goto trace0;
1518                 }
1519               else
1520                 {
1521                   next0 = def_slow;
1522                   goto trace0;
1523                 }
1524             }
1525           else
1526             {
1527               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1528             }
1529
1530           b0->flags |= VNET_BUFFER_F_IS_NATED;
1531
1532           if (!is_output_feature)
1533             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1534
1535           old_addr0 = ip0->src_address.as_u32;
1536           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1537           sum0 = ip0->checksum;
1538           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1539                                  src_address);
1540           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1541             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1542                                    s0->ext_host_addr.as_u32, ip4_header_t,
1543                                    dst_address);
1544           ip0->checksum = ip_csum_fold (sum0);
1545
1546           old_port0 = udp0->src_port;
1547           new_port0 = udp0->src_port = s0->out2in.port;
1548
1549           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1550             {
1551               sum0 = tcp0->checksum;
1552               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1553                                      dst_address);
1554               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1555                                      length);
1556               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1557                 {
1558                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1559                                          s0->ext_host_addr.as_u32,
1560                                          ip4_header_t, dst_address);
1561                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1562                                          s0->ext_host_port, ip4_header_t,
1563                                          length);
1564                   tcp0->dst_port = s0->ext_host_port;
1565                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1566                 }
1567               mss_clamping (sm, tcp0, &sum0);
1568               tcp0->checksum = ip_csum_fold (sum0);
1569               tcp_packets++;
1570               if (nat44_set_tcp_session_state_i2o
1571                   (sm, s0, tcp0, thread_index))
1572                 goto trace0;
1573             }
1574           else if (udp0->checksum)
1575             {
1576               sum0 = udp0->checksum;
1577               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1578                                      dst_address);
1579               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1580                                      length);
1581               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1582                 {
1583                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1584                                          s0->ext_host_addr.as_u32,
1585                                          ip4_header_t, dst_address);
1586                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1587                                          s0->ext_host_port, ip4_header_t,
1588                                          length);
1589                   udp0->dst_port = s0->ext_host_port;
1590                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1591                 }
1592               udp0->checksum = ip_csum_fold (sum0);
1593               udp_packets++;
1594             }
1595           else
1596             {
1597               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1598                 {
1599                   udp0->dst_port = s0->ext_host_port;
1600                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1601                 }
1602               udp_packets++;
1603             }
1604
1605           /* Accounting */
1606           nat44_session_update_counters (s0, now,
1607                                          vlib_buffer_length_in_chain (vm, b0),
1608                                          thread_index);
1609           /* Per-user LRU list maintenance */
1610           nat44_session_update_lru (sm, s0, thread_index);
1611
1612         trace0:
1613           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1614                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1615             {
1616               nat_in2out_ed_trace_t *t =
1617                 vlib_add_trace (vm, node, b0, sizeof (*t));
1618               t->is_slow_path = is_slow_path;
1619               t->sw_if_index = sw_if_index0;
1620               t->next_index = next0;
1621               t->session_index = ~0;
1622               if (s0)
1623                 t->session_index = s0 - tsm->sessions;
1624             }
1625
1626           pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1627
1628           /* verify speculative enqueue, maybe switch current next frame */
1629           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1630                                            to_next, n_left_to_next,
1631                                            bi0, next0);
1632         }
1633
1634       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1635     }
1636
1637   vlib_node_increment_counter (vm, stats_node_index,
1638                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1639                                pkts_processed);
1640   vlib_node_increment_counter (vm, stats_node_index,
1641                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1642   vlib_node_increment_counter (vm, stats_node_index,
1643                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1644   vlib_node_increment_counter (vm, stats_node_index,
1645                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1646                                icmp_packets);
1647   vlib_node_increment_counter (vm, stats_node_index,
1648                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1649                                other_packets);
1650   vlib_node_increment_counter (vm, stats_node_index,
1651                                NAT_IN2OUT_ED_ERROR_FRAGMENTS, fragments);
1652
1653   return frame->n_vectors;
1654 }
1655
1656 static inline uword
1657 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1658                                       vlib_node_runtime_t * node,
1659                                       vlib_frame_t * frame,
1660                                       int is_output_feature)
1661 {
1662   u32 n_left_from, *from, *to_next;
1663   nat_next_t next_index;
1664   u32 pkts_processed = 0, cached_fragments = 0;
1665   snat_main_t *sm = &snat_main;
1666   f64 now = vlib_time_now (vm);
1667   u32 thread_index = vm->thread_index;
1668   snat_main_per_thread_data_t *per_thread_data =
1669     &sm->per_thread_data[thread_index];
1670   u32 *fragments_to_drop = 0;
1671   u32 *fragments_to_loopback = 0;
1672
1673   from = vlib_frame_vector_args (frame);
1674   n_left_from = frame->n_vectors;
1675   next_index = node->cached_next_index;
1676
1677   while (n_left_from > 0)
1678     {
1679       u32 n_left_to_next;
1680
1681       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1682
1683       while (n_left_from > 0 && n_left_to_next > 0)
1684         {
1685           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1686           u32 iph_offset0 = 0;
1687           vlib_buffer_t *b0;
1688           u32 next0;
1689           u8 cached0 = 0;
1690           ip4_header_t *ip0 = 0;
1691           nat_reass_ip4_t *reass0;
1692           udp_header_t *udp0;
1693           tcp_header_t *tcp0;
1694           icmp46_header_t *icmp0;
1695           clib_bihash_kv_16_8_t kv0, value0;
1696           snat_session_t *s0 = 0;
1697           u16 old_port0, new_port0;
1698           ip_csum_t sum0;
1699
1700           /* speculatively enqueue b0 to the current next frame */
1701           bi0 = from[0];
1702           to_next[0] = bi0;
1703           from += 1;
1704           to_next += 1;
1705           n_left_from -= 1;
1706           n_left_to_next -= 1;
1707
1708           b0 = vlib_get_buffer (vm, bi0);
1709
1710           next0 = nat_buffer_opaque (b0)->arc_next;
1711
1712           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1713           rx_fib_index0 =
1714             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1715                                                  sw_if_index0);
1716
1717           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1718             {
1719               next0 = NAT_NEXT_DROP;
1720               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1721               goto trace0;
1722             }
1723
1724           if (is_output_feature)
1725             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1726
1727           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1728                                   iph_offset0);
1729
1730           udp0 = ip4_next_header (ip0);
1731           tcp0 = (tcp_header_t *) udp0;
1732           icmp0 = (icmp46_header_t *) udp0;
1733           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1734
1735           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1736                                                  ip0->dst_address,
1737                                                  ip0->fragment_id,
1738                                                  ip0->protocol,
1739                                                  1, &fragments_to_drop);
1740
1741           if (PREDICT_FALSE (!reass0))
1742             {
1743               next0 = NAT_NEXT_DROP;
1744               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1745               nat_elog_notice ("maximum reassemblies exceeded");
1746               goto trace0;
1747             }
1748
1749           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1750             {
1751               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1752                 {
1753                   if (is_output_feature)
1754                     {
1755                       if (PREDICT_FALSE
1756                           (nat_not_translate_output_feature_fwd
1757                            (sm, ip0, thread_index, now, vm, b0)))
1758                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1759                       goto trace0;
1760                     }
1761
1762                   next0 = icmp_in2out_ed_slow_path
1763                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1764                      next0, now, thread_index, &s0);
1765
1766                   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP))
1767                     {
1768                       if (s0)
1769                         reass0->sess_index = s0 - per_thread_data->sessions;
1770                       else
1771                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1772                       nat_ip4_reass_get_frags (reass0,
1773                                                &fragments_to_loopback);
1774                     }
1775
1776                   goto trace0;
1777                 }
1778
1779               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1780                           ip0->protocol, rx_fib_index0, udp0->src_port,
1781                           udp0->dst_port);
1782
1783               if (clib_bihash_search_16_8
1784                   (&per_thread_data->in2out_ed, &kv0, &value0))
1785                 {
1786                   if (is_output_feature)
1787                     {
1788                       if (PREDICT_FALSE
1789                           (nat44_ed_not_translate_output_feature
1790                            (sm, ip0, ip0->protocol, udp0->src_port,
1791                             udp0->dst_port, thread_index, sw_if_index0,
1792                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1793                         {
1794                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1795                           nat_ip4_reass_get_frags (reass0,
1796                                                    &fragments_to_loopback);
1797                           goto trace0;
1798                         }
1799                     }
1800                   else
1801                     {
1802                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1803                                                                  sw_if_index0,
1804                                                                  ip0, proto0,
1805                                                                  rx_fib_index0,
1806                                                                  thread_index)))
1807                         {
1808                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1809                           nat_ip4_reass_get_frags (reass0,
1810                                                    &fragments_to_loopback);
1811                           goto trace0;
1812                         }
1813                     }
1814
1815                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1816                                         &s0, node, next0, thread_index, now,
1817                                         tcp0);
1818
1819                   if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1820                     goto trace0;
1821
1822                   if (PREDICT_FALSE (!s0))
1823                     {
1824                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1825                       goto trace0;
1826                     }
1827
1828                   reass0->sess_index = s0 - per_thread_data->sessions;
1829                 }
1830               else
1831                 {
1832                   s0 = pool_elt_at_index (per_thread_data->sessions,
1833                                           value0.value);
1834                   reass0->sess_index = value0.value;
1835                 }
1836               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1837             }
1838           else
1839             {
1840               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1841                 goto trace0;
1842               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1843                 {
1844                   if (nat_ip4_reass_add_fragment
1845                       (thread_index, reass0, bi0, &fragments_to_drop))
1846                     {
1847                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1848                       nat_elog_notice
1849                         ("maximum fragments per reassembly exceeded");
1850                       next0 = NAT_NEXT_DROP;
1851                       goto trace0;
1852                     }
1853                   cached0 = 1;
1854                   goto trace0;
1855                 }
1856               s0 = pool_elt_at_index (per_thread_data->sessions,
1857                                       reass0->sess_index);
1858             }
1859
1860           old_addr0 = ip0->src_address.as_u32;
1861           ip0->src_address = s0->out2in.addr;
1862           new_addr0 = ip0->src_address.as_u32;
1863           if (!is_output_feature)
1864             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1865
1866           sum0 = ip0->checksum;
1867           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1868                                  ip4_header_t,
1869                                  src_address /* changed member */ );
1870           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1871             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1872                                    s0->ext_host_addr.as_u32, ip4_header_t,
1873                                    dst_address);
1874           ip0->checksum = ip_csum_fold (sum0);
1875
1876           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1877             {
1878               old_port0 = udp0->src_port;
1879               new_port0 = udp0->src_port = s0->out2in.port;
1880
1881               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1882                 {
1883                   sum0 = tcp0->checksum;
1884                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1885                                          ip4_header_t,
1886                                          dst_address /* changed member */ );
1887                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1888                                          ip4_header_t /* cheat */ ,
1889                                          length /* changed member */ );
1890                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1891                     {
1892                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1893                                              s0->ext_host_addr.as_u32,
1894                                              ip4_header_t, dst_address);
1895                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1896                                              s0->ext_host_port, ip4_header_t,
1897                                              length);
1898                       tcp0->dst_port = s0->ext_host_port;
1899                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1900                     }
1901                   tcp0->checksum = ip_csum_fold (sum0);
1902                 }
1903               else if (udp0->checksum)
1904                 {
1905                   sum0 = udp0->checksum;
1906                   sum0 =
1907                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1908                                     dst_address);
1909                   sum0 =
1910                     ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1911                                     length);
1912                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1913                     {
1914                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1915                                              s0->ext_host_addr.as_u32,
1916                                              ip4_header_t, dst_address);
1917                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1918                                              s0->ext_host_port, ip4_header_t,
1919                                              length);
1920                       udp0->dst_port = s0->ext_host_port;
1921                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1922                     }
1923                   udp0->checksum = ip_csum_fold (sum0);
1924                 }
1925               else
1926                 {
1927                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1928                     {
1929                       udp0->dst_port = s0->ext_host_port;
1930                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1931                     }
1932                 }
1933             }
1934
1935           /* Hairpinning */
1936           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1937                                    s0->ext_host_port, proto0, 1);
1938
1939           /* Accounting */
1940           nat44_session_update_counters (s0, now,
1941                                          vlib_buffer_length_in_chain (vm, b0),
1942                                          thread_index);
1943           /* Per-user LRU list maintenance */
1944           nat44_session_update_lru (sm, s0, thread_index);
1945
1946         trace0:
1947           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1948                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1949             {
1950               nat44_reass_trace_t *t =
1951                 vlib_add_trace (vm, node, b0, sizeof (*t));
1952               t->cached = cached0;
1953               t->sw_if_index = sw_if_index0;
1954               t->next_index = next0;
1955             }
1956
1957           if (cached0)
1958             {
1959               n_left_to_next++;
1960               to_next--;
1961               cached_fragments++;
1962             }
1963           else
1964             {
1965               pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1966
1967               /* verify speculative enqueue, maybe switch current next frame */
1968               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1969                                                to_next, n_left_to_next,
1970                                                bi0, next0);
1971             }
1972
1973           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1974             {
1975               from = vlib_frame_vector_args (frame);
1976               u32 len = vec_len (fragments_to_loopback);
1977               if (len <= VLIB_FRAME_SIZE)
1978                 {
1979                   clib_memcpy_fast (from, fragments_to_loopback,
1980                                     sizeof (u32) * len);
1981                   n_left_from = len;
1982                   vec_reset_length (fragments_to_loopback);
1983                 }
1984               else
1985                 {
1986                   clib_memcpy_fast (from, fragments_to_loopback +
1987                                     (len - VLIB_FRAME_SIZE),
1988                                     sizeof (u32) * VLIB_FRAME_SIZE);
1989                   n_left_from = VLIB_FRAME_SIZE;
1990                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1991                 }
1992             }
1993         }
1994
1995       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1996     }
1997
1998   vlib_node_increment_counter (vm, sm->ed_in2out_reass_node_index,
1999                                NAT_IN2OUT_ED_ERROR_PROCESSED_FRAGMENTS,
2000                                pkts_processed);
2001   vlib_node_increment_counter (vm, sm->ed_in2out_reass_node_index,
2002                                NAT_IN2OUT_ED_ERROR_CACHED_FRAGMENTS,
2003                                cached_fragments);
2004
2005   nat_send_all_to_node (vm, fragments_to_drop, node,
2006                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
2007                         NAT_NEXT_DROP);
2008
2009   vec_free (fragments_to_drop);
2010   vec_free (fragments_to_loopback);
2011   return frame->n_vectors;
2012 }
2013
2014 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
2015                                      vlib_node_runtime_t * node,
2016                                      vlib_frame_t * frame)
2017 {
2018   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
2019 }
2020
2021 /* *INDENT-OFF* */
2022 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
2023   .name = "nat44-ed-in2out",
2024   .vector_size = sizeof (u32),
2025   .sibling_of = "nat-default",
2026   .format_trace = format_nat_in2out_ed_trace,
2027   .type = VLIB_NODE_TYPE_INTERNAL,
2028   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2029   .error_strings = nat_in2out_ed_error_strings,
2030   .runtime_data_bytes = sizeof (snat_runtime_t),
2031 };
2032 /* *INDENT-ON* */
2033
2034 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
2035                                             vlib_node_runtime_t * node,
2036                                             vlib_frame_t * frame)
2037 {
2038   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
2039 }
2040
2041 /* *INDENT-OFF* */
2042 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
2043   .name = "nat44-ed-in2out-output",
2044   .vector_size = sizeof (u32),
2045   .sibling_of = "nat-default",
2046   .format_trace = format_nat_in2out_ed_trace,
2047   .type = VLIB_NODE_TYPE_INTERNAL,
2048   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2049   .error_strings = nat_in2out_ed_error_strings,
2050   .runtime_data_bytes = sizeof (snat_runtime_t),
2051 };
2052 /* *INDENT-ON* */
2053
2054 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
2055                                               vlib_node_runtime_t * node,
2056                                               vlib_frame_t * frame)
2057 {
2058   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
2059 }
2060
2061 /* *INDENT-OFF* */
2062 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
2063   .name = "nat44-ed-in2out-slowpath",
2064   .vector_size = sizeof (u32),
2065   .sibling_of = "nat-default",
2066   .format_trace = format_nat_in2out_ed_trace,
2067   .type = VLIB_NODE_TYPE_INTERNAL,
2068   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2069   .error_strings = nat_in2out_ed_error_strings,
2070   .runtime_data_bytes = sizeof (snat_runtime_t),
2071 };
2072 /* *INDENT-ON* */
2073
2074 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
2075                                                      vlib_node_runtime_t *
2076                                                      node,
2077                                                      vlib_frame_t * frame)
2078 {
2079   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
2080 }
2081
2082 /* *INDENT-OFF* */
2083 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
2084   .name = "nat44-ed-in2out-output-slowpath",
2085   .vector_size = sizeof (u32),
2086   .sibling_of = "nat-default",
2087   .format_trace = format_nat_in2out_ed_trace,
2088   .type = VLIB_NODE_TYPE_INTERNAL,
2089   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2090   .error_strings = nat_in2out_ed_error_strings,
2091   .runtime_data_bytes = sizeof (snat_runtime_t),
2092 };
2093 /* *INDENT-ON* */
2094
2095
2096 VLIB_NODE_FN (nat44_ed_in2out_reass_node) (vlib_main_t * vm,
2097                                            vlib_node_runtime_t * node,
2098                                            vlib_frame_t * frame)
2099 {
2100   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2101 }
2102
2103 /* *INDENT-OFF* */
2104 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2105   .name = "nat44-ed-in2out-reass",
2106   .vector_size = sizeof (u32),
2107   .sibling_of = "nat-default",
2108   .format_trace = format_nat44_reass_trace,
2109   .type = VLIB_NODE_TYPE_INTERNAL,
2110   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2111   .error_strings = nat_in2out_ed_error_strings,
2112 };
2113 /* *INDENT-ON* */
2114
2115 VLIB_NODE_FN (nat44_ed_in2out_reass_output_node) (vlib_main_t * vm,
2116                                                   vlib_node_runtime_t * node,
2117                                                   vlib_frame_t * frame)
2118 {
2119   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2120 }
2121
2122 /* *INDENT-OFF* */
2123 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2124   .name = "nat44-ed-in2out-reass-output",
2125   .vector_size = sizeof (u32),
2126   .sibling_of = "nat-default",
2127   .format_trace = format_nat44_reass_trace,
2128   .type = VLIB_NODE_TYPE_INTERNAL,
2129   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2130   .error_strings = nat_in2out_ed_error_strings,
2131 };
2132 /* *INDENT-ON* */
2133
2134 static u8 *
2135 format_nat_pre_trace (u8 * s, va_list * args)
2136 {
2137   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2138   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2139   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
2140   return format (s, "in2out next_index %d", t->next_index);
2141 }
2142
2143 VLIB_NODE_FN (nat_pre_in2out_node) (vlib_main_t * vm,
2144                                     vlib_node_runtime_t * node,
2145                                     vlib_frame_t * frame)
2146 {
2147   return nat_pre_node_fn_inline (vm, node, frame,
2148                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
2149 }
2150
2151 /* *INDENT-OFF* */
2152 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
2153   .name = "nat-pre-in2out",
2154   .vector_size = sizeof (u32),
2155   .sibling_of = "nat-default",
2156   .format_trace = format_nat_pre_trace,
2157   .type = VLIB_NODE_TYPE_INTERNAL,
2158   .n_errors = 0,
2159 };
2160 /* *INDENT-ON* */
2161
2162 /*
2163  * fd.io coding-style-patch-verification: ON
2164  *
2165  * Local Variables:
2166  * eval: (c-set-style "gnu")
2167  * End:
2168  */