src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #define foreach_nat_in2out_ed_error                       \
  33 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  34 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  35 _(OUT_OF_PORTS, "Out of ports")                         \
  36 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  37 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  38 _(DROP_FRAGMENT, "Drop fragment")                       \
  39 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  40 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  41
  42 typedef enum
  43 {
  44 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
  45   foreach_nat_in2out_ed_error
  46 #undef _
  47     NAT_IN2OUT_ED_N_ERROR,
  48 } nat_in2out_ed_error_t;
  49
  50 static char *nat_in2out_ed_error_strings[] = {
  51 #define _(sym,string) string,
  52   foreach_nat_in2out_ed_error
  53 #undef _
  54 };
  55
  56 typedef enum
  57 {
  58   NAT_IN2OUT_ED_NEXT_LOOKUP,
  59   NAT_IN2OUT_ED_NEXT_DROP,
  60   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
  61   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
  62   NAT_IN2OUT_ED_NEXT_REASS,
  63   NAT_IN2OUT_ED_N_NEXT,
  64 } nat_in2out_ed_next_t;
  65
  66 typedef struct
  67 {
  68   u32 sw_if_index;
  69   u32 next_index;
  70   u32 session_index;
  71   u32 is_slow_path;
  72 } nat_in2out_ed_trace_t;
  73
  74 vlib_node_registration_t nat44_ed_in2out_node;
  75 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
  76 vlib_node_registration_t nat44_ed_in2out_output_node;
  77 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
  78 vlib_node_registration_t nat44_ed_in2out_reass_node;
  79
  80 static u8 *
  81 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  82 {
  83   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  84   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  85   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  86   char *tag;
  87
  88   tag =
  89     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  90     "NAT44_IN2OUT_ED_FAST_PATH";
  91
  92   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  93               t->sw_if_index, t->next_index, t->session_index);
  94
  95   return s;
  96 }
  97
  98 static_always_inline int
  99 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 100 {
 101   icmp46_header_t *icmp0;
 102   nat_ed_ses_key_t key0;
 103   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 104   ip4_header_t *inner_ip0 = 0;
 105   void *l4_header = 0;
 106   icmp46_header_t *inner_icmp0;
 107
 108   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 109   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 110
 111   if (!icmp_is_error_message (icmp0))
 112     {
 113       key0.proto = IP_PROTOCOL_ICMP;
 114       key0.l_addr = ip0->src_address;
 115       key0.r_addr = ip0->dst_address;
 116       key0.l_port = echo0->identifier;
 117       key0.r_port = 0;
 118     }
 119   else
 120     {
 121       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 122       l4_header = ip4_next_header (inner_ip0);
 123       key0.proto = inner_ip0->protocol;
 124       key0.r_addr = inner_ip0->src_address;
 125       key0.l_addr = inner_ip0->dst_address;
 126       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 127         {
 128         case SNAT_PROTOCOL_ICMP:
 129           inner_icmp0 = (icmp46_header_t *) l4_header;
 130           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 131           key0.r_port = 0;
 132           key0.l_port = inner_echo0->identifier;
 133           break;
 134         case SNAT_PROTOCOL_UDP:
 135         case SNAT_PROTOCOL_TCP:
 136           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 137           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
 138           break;
 139         default:
 140           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
 141         }
 142     }
 143   *p_key0 = key0;
 144   return 0;
 145 }
 146
 147 int
 148 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 149 {
 150   snat_main_t *sm = &snat_main;
 151   nat44_is_idle_session_ctx_t *ctx = arg;
 152   snat_session_t *s;
 153   u64 sess_timeout_time;
 154   nat_ed_ses_key_t ed_key;
 155   clib_bihash_kv_16_8_t ed_kv;
 156   int i;
 157   snat_address_t *a;
 158   snat_session_key_t key;
 159   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 160                                                        ctx->thread_index);
 161
 162   s = pool_elt_at_index (tsm->sessions, kv->value);
 163   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 164   if (ctx->now >= sess_timeout_time)
 165     {
 166       if (is_fwd_bypass_session (s))
 167         goto delete;
 168
 169       ed_key.l_addr = s->out2in.addr;
 170       ed_key.r_addr = s->ext_host_addr;
 171       ed_key.fib_index = s->out2in.fib_index;
 172       if (snat_is_unk_proto_session (s))
 173         {
 174           ed_key.proto = s->in2out.port;
 175           ed_key.r_port = 0;
 176           ed_key.l_port = 0;
 177         }
 178       else
 179         {
 180           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 181           ed_key.l_port = s->out2in.port;
 182           ed_key.r_port = s->ext_host_port;
 183         }
 184       ed_kv.key[0] = ed_key.as_u64[0];
 185       ed_kv.key[1] = ed_key.as_u64[1];
 186       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 187         nat_log_warn ("out2in_ed key del failed");
 188
 189       if (snat_is_unk_proto_session (s))
 190         goto delete;
 191
 192       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 193                                            s->out2in.addr.as_u32,
 194                                            s->in2out.protocol,
 195                                            s->in2out.port,
 196                                            s->out2in.port,
 197                                            s->in2out.fib_index);
 198
 199       if (is_twice_nat_session (s))
 200         {
 201           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 202             {
 203               key.protocol = s->in2out.protocol;
 204               key.port = s->ext_host_nat_port;
 205               a = sm->twice_nat_addresses + i;
 206               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 207                 {
 208                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 209                                                       ctx->thread_index,
 210                                                       &key);
 211                   break;
 212                 }
 213             }
 214         }
 215
 216       if (snat_is_session_static (s))
 217         goto delete;
 218
 219       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 220                                           &s->out2in);
 221     delete:
 222       nat44_delete_session (sm, s, ctx->thread_index);
 223       return 1;
 224     }
 225
 226   return 0;
 227 }
 228
 229 static inline u32
 230 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 231                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 232                           u32 sw_if_index0, u32 rx_fib_index0,
 233                           vlib_node_runtime_t * node, u32 next0, f64 now,
 234                           u32 thread_index, snat_session_t ** p_s0)
 235 {
 236   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 237                        next0, thread_index, p_s0, 0);
 238   snat_session_t *s0 = *p_s0;
 239   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
 240     {
 241       /* Accounting */
 242       nat44_session_update_counters (s0, now,
 243                                      vlib_buffer_length_in_chain
 244                                      (sm->vlib_main, b0));
 245     }
 246   return next0;
 247 }
 248
 249 static u32
 250 slow_path_ed (snat_main_t * sm,
 251               vlib_buffer_t * b,
 252               u32 rx_fib_index,
 253               clib_bihash_kv_16_8_t * kv,
 254               snat_session_t ** sessionp,
 255               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 256 {
 257   snat_session_t *s;
 258   snat_user_t *u;
 259   snat_session_key_t key0, key1;
 260   lb_nat_type_t lb = 0, is_sm = 0;
 261   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 262   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 263   u32 proto = ip_proto_to_snat_proto (key->proto);
 264   nat_outside_fib_t *outside_fib;
 265   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 266   fib_prefix_t pfx = {
 267     .fp_proto = FIB_PROTOCOL_IP4,
 268     .fp_len = 32,
 269     .fp_addr = {
 270                 .ip4.as_u32 = key->r_addr.as_u32,
 271                 },
 272   };
 273   nat44_is_idle_session_ctx_t ctx;
 274
 275   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 276     {
 277       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 278       nat_ipfix_logging_max_sessions (sm->max_translations);
 279       nat_log_notice ("maximum sessions exceeded");
 280       return NAT_IN2OUT_ED_NEXT_DROP;
 281     }
 282
 283   key0.addr = key->l_addr;
 284   key0.port = key->l_port;
 285   key1.protocol = key0.protocol = proto;
 286   key0.fib_index = rx_fib_index;
 287   key1.fib_index = sm->outside_fib_index;
 288   /* First try to match static mapping by local address and port */
 289   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0))
 290     {
 291       /* Try to create dynamic translation */
 292       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 293                                                thread_index, &key1,
 294                                                sm->port_per_thread,
 295                                                tsm->snat_thread_index))
 296         {
 297           nat_log_notice ("addresses exhausted");
 298           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 299           return NAT_IN2OUT_ED_NEXT_DROP;
 300         }
 301     }
 302   else
 303     is_sm = 1;
 304
 305   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 306   if (!u)
 307     {
 308       nat_log_warn ("create NAT user failed");
 309       if (!is_sm)
 310         snat_free_outside_address_and_port (sm->addresses,
 311                                             thread_index, &key1);
 312       return NAT_IN2OUT_ED_NEXT_DROP;
 313     }
 314
 315   s = nat_ed_session_alloc (sm, u, thread_index, now);
 316   if (!s)
 317     {
 318       nat44_delete_user_with_no_session (sm, u, thread_index);
 319       nat_log_warn ("create NAT session failed");
 320       if (!is_sm)
 321         snat_free_outside_address_and_port (sm->addresses,
 322                                             thread_index, &key1);
 323       return NAT_IN2OUT_ED_NEXT_DROP;
 324     }
 325
 326   user_session_increment (sm, u, is_sm);
 327   if (is_sm)
 328     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 329   if (lb)
 330     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 331   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 332   s->ext_host_addr = key->r_addr;
 333   s->ext_host_port = key->r_port;
 334   s->in2out = key0;
 335   s->out2in = key1;
 336   s->out2in.protocol = key0.protocol;
 337
 338   switch (vec_len (sm->outside_fibs))
 339     {
 340     case 0:
 341       s->out2in.fib_index = sm->outside_fib_index;
 342       break;
 343     case 1:
 344       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 345       break;
 346     default:
 347       /* *INDENT-OFF* */
 348       vec_foreach (outside_fib, sm->outside_fibs)
 349        {
 350           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 351           if (FIB_NODE_INDEX_INVALID != fei)
 352             {
 353               if (fib_entry_get_resolving_interface (fei) != ~0)
 354                 {
 355                   s->out2in.fib_index = outside_fib->fib_index;
 356                   break;
 357                 }
 358             }
 359         }
 360       /* *INDENT-ON* */
 361       break;
 362     }
 363
 364   /* Add to lookup tables */
 365   kv->value = s - tsm->sessions;
 366   ctx.now = now;
 367   ctx.thread_index = thread_index;
 368   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 369                                                nat44_i2o_ed_is_idle_session_cb,
 370                                                &ctx))
 371     nat_log_notice ("in2out-ed key add failed");
 372
 373   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 374               key1.port, key->r_port);
 375   kv->value = s - tsm->sessions;
 376   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 377                                                nat44_o2i_ed_is_idle_session_cb,
 378                                                &ctx))
 379     nat_log_notice ("out2in-ed key add failed");
 380
 381   *sessionp = s;
 382
 383   /* log NAT event */
 384   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 385                                        s->out2in.addr.as_u32,
 386                                        s->in2out.protocol,
 387                                        s->in2out.port,
 388                                        s->out2in.port, s->in2out.fib_index);
 389   return next;
 390 }
 391
 392 static_always_inline int
 393 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 394                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 395                         u32 rx_fib_index, u32 thread_index)
 396 {
 397   udp_header_t *udp = ip4_next_header (ip);
 398   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 399   clib_bihash_kv_16_8_t kv, value;
 400   snat_session_key_t key0, key1;
 401
 402   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 403               sm->outside_fib_index, udp->dst_port, udp->src_port);
 404
 405   /* NAT packet aimed at external address if */
 406   /* has active sessions */
 407   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 408     {
 409       key0.addr = ip->dst_address;
 410       key0.port = udp->dst_port;
 411       key0.protocol = proto;
 412       key0.fib_index = sm->outside_fib_index;
 413       /* or is static mappings */
 414       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0))
 415         return 0;
 416     }
 417   else
 418     return 0;
 419
 420   if (sm->forwarding_enabled)
 421     return 1;
 422
 423   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 424                                   rx_fib_index);
 425 }
 426
 427 static_always_inline int
 428 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 429                                       u32 thread_index, f64 now,
 430                                       vlib_main_t * vm, vlib_buffer_t * b)
 431 {
 432   nat_ed_ses_key_t key;
 433   clib_bihash_kv_16_8_t kv, value;
 434   udp_header_t *udp;
 435   snat_session_t *s = 0;
 436   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 437
 438   if (!sm->forwarding_enabled)
 439     return 0;
 440
 441   if (ip->protocol == IP_PROTOCOL_ICMP)
 442     {
 443       key.as_u64[0] = key.as_u64[1] = 0;
 444       if (icmp_get_ed_key (ip, &key))
 445         return 0;
 446       key.fib_index = 0;
 447       kv.key[0] = key.as_u64[0];
 448       kv.key[1] = key.as_u64[1];
 449     }
 450   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 451     {
 452       udp = ip4_next_header (ip);
 453       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 454                   udp->src_port, udp->dst_port);
 455     }
 456   else
 457     {
 458       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 459                   0);
 460     }
 461
 462   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 463     {
 464       s = pool_elt_at_index (tsm->sessions, value.value);
 465       if (is_fwd_bypass_session (s))
 466         {
 467           if (ip->protocol == IP_PROTOCOL_TCP)
 468             {
 469               tcp_header_t *tcp = ip4_next_header (ip);
 470               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 471                 return 1;
 472             }
 473           /* Accounting */
 474           nat44_session_update_counters (s, now,
 475                                          vlib_buffer_length_in_chain (vm, b));
 476           return 1;
 477         }
 478       else
 479         return 0;
 480     }
 481
 482   return 0;
 483 }
 484
 485 static_always_inline int
 486 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 487                                        u8 proto, u16 src_port, u16 dst_port,
 488                                        u32 thread_index, u32 rx_sw_if_index,
 489                                        u32 tx_sw_if_index)
 490 {
 491   clib_bihash_kv_16_8_t kv, value;
 492   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 493   snat_interface_t *i;
 494   snat_session_t *s;
 495   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 496   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 497
 498   /* src NAT check */
 499   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 500               src_port, dst_port);
 501   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 502     return 1;
 503
 504   /* dst NAT check */
 505   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 506               dst_port, src_port);
 507   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 508     {
 509       s = pool_elt_at_index (tsm->sessions, value.value);
 510       if (is_fwd_bypass_session (s))
 511         return 0;
 512
 513       /* hairpinning */
 514       /* *INDENT-OFF* */
 515       pool_foreach (i, sm->output_feature_interfaces,
 516       ({
 517         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 518            return 0;
 519       }));
 520       /* *INDENT-ON* */
 521       return 1;
 522     }
 523
 524   return 0;
 525 }
 526
 527 u32
 528 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 529                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 530                       u8 * p_proto, snat_session_key_t * p_value,
 531                       u8 * p_dont_translate, void *d, void *e)
 532 {
 533   icmp46_header_t *icmp;
 534   u32 sw_if_index;
 535   u32 rx_fib_index;
 536   nat_ed_ses_key_t key;
 537   snat_session_t *s = 0;
 538   u8 dont_translate = 0;
 539   clib_bihash_kv_16_8_t kv, value;
 540   u32 next = ~0;
 541   int err;
 542   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 543
 544   icmp = (icmp46_header_t *) ip4_next_header (ip);
 545   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 546   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 547
 548   key.as_u64[0] = key.as_u64[1] = 0;
 549   err = icmp_get_ed_key (ip, &key);
 550   if (err != 0)
 551     {
 552       b->error = node->errors[err];
 553       next = NAT_IN2OUT_ED_NEXT_DROP;
 554       goto out;
 555     }
 556   key.fib_index = rx_fib_index;
 557
 558   kv.key[0] = key.as_u64[0];
 559   kv.key[1] = key.as_u64[1];
 560
 561   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 562     {
 563       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 564         {
 565           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 566                                                                     key.proto,
 567                                                                     key.
 568                                                                     l_port,
 569                                                                     key.
 570                                                                     r_port,
 571                                                                     thread_index,
 572                                                                     sw_if_index,
 573                                                                     vnet_buffer
 574                                                                     (b)->
 575                                                                     sw_if_index
 576                                                                     [VLIB_TX])))
 577             {
 578               dont_translate = 1;
 579               goto out;
 580             }
 581         }
 582       else
 583         {
 584           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 585                                                      ip, SNAT_PROTOCOL_ICMP,
 586                                                      rx_fib_index,
 587                                                      thread_index)))
 588             {
 589               dont_translate = 1;
 590               goto out;
 591             }
 592         }
 593
 594       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 595         {
 596           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 597           next = NAT_IN2OUT_ED_NEXT_DROP;
 598           goto out;
 599         }
 600
 601       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 602                            thread_index, vlib_time_now (sm->vlib_main));
 603
 604       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
 605         goto out;
 606     }
 607   else
 608     {
 609       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 610                          icmp->type != ICMP4_echo_reply &&
 611                          !icmp_is_error_message (icmp)))
 612         {
 613           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 614           next = NAT_IN2OUT_ED_NEXT_DROP;
 615           goto out;
 616         }
 617
 618       s = pool_elt_at_index (tsm->sessions, value.value);
 619     }
 620
 621   *p_proto = ip_proto_to_snat_proto (key.proto);
 622 out:
 623   if (s)
 624     *p_value = s->out2in;
 625   *p_dont_translate = dont_translate;
 626   if (d)
 627     *(snat_session_t **) d = s;
 628   return next;
 629 }
 630
 631 static snat_session_t *
 632 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 633                                vlib_buffer_t * b,
 634                                ip4_header_t * ip,
 635                                u32 rx_fib_index,
 636                                u32 thread_index,
 637                                f64 now,
 638                                vlib_main_t * vm, vlib_node_runtime_t * node)
 639 {
 640   clib_bihash_kv_8_8_t kv, value;
 641   clib_bihash_kv_16_8_t s_kv, s_value;
 642   snat_static_mapping_t *m;
 643   u32 old_addr, new_addr = 0;
 644   ip_csum_t sum;
 645   snat_user_t *u;
 646   dlist_elt_t *head, *elt;
 647   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 648   u32 elt_index, head_index, ses_index;
 649   snat_session_t *s;
 650   u32 outside_fib_index = sm->outside_fib_index;
 651   int i;
 652   u8 is_sm = 0;
 653   nat_outside_fib_t *outside_fib;
 654   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 655   fib_prefix_t pfx = {
 656     .fp_proto = FIB_PROTOCOL_IP4,
 657     .fp_len = 32,
 658     .fp_addr = {
 659                 .ip4.as_u32 = ip->dst_address.as_u32,
 660                 },
 661   };
 662
 663   switch (vec_len (sm->outside_fibs))
 664     {
 665     case 0:
 666       outside_fib_index = sm->outside_fib_index;
 667       break;
 668     case 1:
 669       outside_fib_index = sm->outside_fibs[0].fib_index;
 670       break;
 671     default:
 672       /* *INDENT-OFF* */
 673       vec_foreach (outside_fib, sm->outside_fibs)
 674         {
 675           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 676           if (FIB_NODE_INDEX_INVALID != fei)
 677             {
 678               if (fib_entry_get_resolving_interface (fei) != ~0)
 679                 {
 680                   outside_fib_index = outside_fib->fib_index;
 681                   break;
 682                 }
 683             }
 684         }
 685       /* *INDENT-ON* */
 686       break;
 687     }
 688   old_addr = ip->src_address.as_u32;
 689
 690   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 691               rx_fib_index, 0, 0);
 692
 693   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 694     {
 695       s = pool_elt_at_index (tsm->sessions, s_value.value);
 696       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 697     }
 698   else
 699     {
 700       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 701         {
 702           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 703           nat_ipfix_logging_max_sessions (sm->max_translations);
 704           nat_log_notice ("maximum sessions exceeded");
 705           return 0;
 706         }
 707
 708       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 709                                   thread_index);
 710       if (!u)
 711         {
 712           nat_log_warn ("create NAT user failed");
 713           return 0;
 714         }
 715
 716       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 717
 718       /* Try to find static mapping first */
 719       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 720         {
 721           m = pool_elt_at_index (sm->static_mappings, value.value);
 722           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 723           is_sm = 1;
 724           goto create_ses;
 725         }
 726       /* Fallback to 3-tuple key */
 727       else
 728         {
 729           /* Choose same out address as for TCP/UDP session to same destination */
 730           head_index = u->sessions_per_user_list_head_index;
 731           head = pool_elt_at_index (tsm->list_pool, head_index);
 732           elt_index = head->next;
 733           if (PREDICT_FALSE (elt_index == ~0))
 734             ses_index = ~0;
 735           else
 736             {
 737               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 738               ses_index = elt->value;
 739             }
 740
 741           while (ses_index != ~0)
 742             {
 743               s = pool_elt_at_index (tsm->sessions, ses_index);
 744               elt_index = elt->next;
 745               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 746               ses_index = elt->value;
 747
 748               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 749                 {
 750                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 751
 752                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 753                               ip->protocol, outside_fib_index, 0, 0);
 754                   if (clib_bihash_search_16_8
 755                       (&tsm->out2in_ed, &s_kv, &s_value))
 756                     goto create_ses;
 757
 758                   break;
 759                 }
 760             }
 761
 762           for (i = 0; i < vec_len (sm->addresses); i++)
 763             {
 764               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 765                           ip->protocol, outside_fib_index, 0, 0);
 766               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 767                 {
 768                   new_addr = ip->src_address.as_u32 =
 769                     sm->addresses[i].addr.as_u32;
 770                   goto create_ses;
 771                 }
 772             }
 773           return 0;
 774         }
 775
 776     create_ses:
 777       s = nat_ed_session_alloc (sm, u, thread_index, now);
 778       if (!s)
 779         {
 780           nat44_delete_user_with_no_session (sm, u, thread_index);
 781           nat_log_warn ("create NAT session failed");
 782           return 0;
 783         }
 784
 785       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 786       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 787       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 788       s->out2in.addr.as_u32 = new_addr;
 789       s->out2in.fib_index = outside_fib_index;
 790       s->in2out.addr.as_u32 = old_addr;
 791       s->in2out.fib_index = rx_fib_index;
 792       s->in2out.port = s->out2in.port = ip->protocol;
 793       if (is_sm)
 794         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 795       user_session_increment (sm, u, is_sm);
 796
 797       /* Add to lookup tables */
 798       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 799                   rx_fib_index, 0, 0);
 800       s_kv.value = s - tsm->sessions;
 801       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 802         nat_log_notice ("in2out key add failed");
 803
 804       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 805                   outside_fib_index, 0, 0);
 806       s_kv.value = s - tsm->sessions;
 807       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 808         nat_log_notice ("out2in key add failed");
 809     }
 810
 811   /* Update IP checksum */
 812   sum = ip->checksum;
 813   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 814   ip->checksum = ip_csum_fold (sum);
 815
 816   /* Accounting */
 817   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 818
 819   /* Hairpinning */
 820   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 821     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 822
 823   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 824     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 825
 826   return s;
 827 }
 828
 829 static inline uword
 830 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 831                                 vlib_node_runtime_t * node,
 832                                 vlib_frame_t * frame, int is_slow_path,
 833                                 int is_output_feature)
 834 {
 835   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 836   nat_in2out_ed_next_t next_index;
 837   snat_main_t *sm = &snat_main;
 838   f64 now = vlib_time_now (vm);
 839   u32 thread_index = vm->thread_index;
 840   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 841
 842   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
 843     nat44_ed_in2out_node.index;
 844
 845   from = vlib_frame_vector_args (frame);
 846   n_left_from = frame->n_vectors;
 847   next_index = node->cached_next_index;
 848
 849   while (n_left_from > 0)
 850     {
 851       u32 n_left_to_next;
 852
 853       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 854
 855       while (n_left_from >= 4 && n_left_to_next >= 2)
 856         {
 857           u32 bi0, bi1;
 858           vlib_buffer_t *b0, *b1;
 859           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 860             new_addr0, old_addr0;
 861           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 862             new_addr1, old_addr1;
 863           u16 old_port0, new_port0, old_port1, new_port1;
 864           ip4_header_t *ip0, *ip1;
 865           udp_header_t *udp0, *udp1;
 866           tcp_header_t *tcp0, *tcp1;
 867           icmp46_header_t *icmp0, *icmp1;
 868           snat_session_t *s0 = 0, *s1 = 0;
 869           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 870           ip_csum_t sum0, sum1;
 871
 872           /* Prefetch next iteration. */
 873           {
 874             vlib_buffer_t *p2, *p3;
 875
 876             p2 = vlib_get_buffer (vm, from[2]);
 877             p3 = vlib_get_buffer (vm, from[3]);
 878
 879             vlib_prefetch_buffer_header (p2, LOAD);
 880             vlib_prefetch_buffer_header (p3, LOAD);
 881
 882             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 883             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 884           }
 885
 886           /* speculatively enqueue b0 and b1 to the current next frame */
 887           to_next[0] = bi0 = from[0];
 888           to_next[1] = bi1 = from[1];
 889           from += 2;
 890           to_next += 2;
 891           n_left_from -= 2;
 892           n_left_to_next -= 2;
 893
 894           b0 = vlib_get_buffer (vm, bi0);
 895           b1 = vlib_get_buffer (vm, bi1);
 896
 897           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
 898
 899           if (is_output_feature)
 900             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 901
 902           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 903                                   iph_offset0);
 904
 905           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 906           rx_fib_index0 =
 907             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 908                                                  sw_if_index0);
 909
 910           if (PREDICT_FALSE (ip0->ttl == 1))
 911             {
 912               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 913               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 914                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 915                                            0);
 916               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
 917               goto trace00;
 918             }
 919
 920           udp0 = ip4_next_header (ip0);
 921           tcp0 = (tcp_header_t *) udp0;
 922           icmp0 = (icmp46_header_t *) udp0;
 923           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 924
 925           if (is_slow_path)
 926             {
 927               if (PREDICT_FALSE (proto0 == ~0))
 928                 {
 929                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
 930                                                       rx_fib_index0,
 931                                                       thread_index, now, vm,
 932                                                       node);
 933                   if (!s0)
 934                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
 935                   goto trace00;
 936                 }
 937
 938               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 939                 {
 940                   next0 = icmp_in2out_ed_slow_path
 941                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 942                      next0, now, thread_index, &s0);
 943                   goto trace00;
 944                 }
 945             }
 946           else
 947             {
 948               if (PREDICT_FALSE (proto0 == ~0))
 949                 {
 950                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 951                   goto trace00;
 952                 }
 953
 954               if (ip4_is_fragment (ip0))
 955                 {
 956                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
 957                   goto trace00;
 958                 }
 959
 960               if (is_output_feature)
 961                 {
 962                   if (PREDICT_FALSE
 963                       (nat_not_translate_output_feature_fwd
 964                        (sm, ip0, thread_index, now, vm, b0)))
 965                     goto trace00;
 966                 }
 967
 968               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 969                 {
 970                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 971                   goto trace00;
 972                 }
 973             }
 974
 975           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
 976                       ip0->protocol, rx_fib_index0, udp0->src_port,
 977                       udp0->dst_port);
 978
 979           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
 980             {
 981               if (is_slow_path)
 982                 {
 983                   if (is_output_feature)
 984                     {
 985                       if (PREDICT_FALSE
 986                           (nat44_ed_not_translate_output_feature
 987                            (sm, ip0, ip0->protocol, udp0->src_port,
 988                             udp0->dst_port, thread_index, sw_if_index0,
 989                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
 990                         goto trace00;
 991                     }
 992                   else
 993                     {
 994                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
 995                                                                  sw_if_index0,
 996                                                                  ip0, proto0,
 997                                                                  rx_fib_index0,
 998                                                                  thread_index)))
 999                         goto trace00;
1000                     }
1001
1002                   next0 =
1003                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1004                                   next0, thread_index, now);
1005
1006                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1007                     goto trace00;
1008                 }
1009               else
1010                 {
1011                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1012                   goto trace00;
1013                 }
1014             }
1015           else
1016             {
1017               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1018             }
1019
1020           b0->flags |= VNET_BUFFER_F_IS_NATED;
1021
1022           if (!is_output_feature)
1023             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1024
1025           old_addr0 = ip0->src_address.as_u32;
1026           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1027           sum0 = ip0->checksum;
1028           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1029                                  src_address);
1030           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1031             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1032                                    s0->ext_host_addr.as_u32, ip4_header_t,
1033                                    dst_address);
1034           ip0->checksum = ip_csum_fold (sum0);
1035
1036           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1037             {
1038               old_port0 = tcp0->src_port;
1039               new_port0 = tcp0->src_port = s0->out2in.port;
1040
1041               sum0 = tcp0->checksum;
1042               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1043                                      dst_address);
1044               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1045                                      length);
1046               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1047                 {
1048                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1049                                          s0->ext_host_addr.as_u32,
1050                                          ip4_header_t, dst_address);
1051                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1052                                          s0->ext_host_port, ip4_header_t,
1053                                          length);
1054                   tcp0->dst_port = s0->ext_host_port;
1055                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1056                 }
1057               mss_clamping (sm, tcp0, &sum0);
1058               tcp0->checksum = ip_csum_fold (sum0);
1059               if (nat44_set_tcp_session_state_i2o
1060                   (sm, s0, tcp0, thread_index))
1061                 goto trace00;
1062             }
1063           else
1064             {
1065               udp0->src_port = s0->out2in.port;
1066               udp0->checksum = 0;
1067               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1068                 {
1069                   udp0->dst_port = s0->ext_host_port;
1070                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1071                 }
1072             }
1073
1074           /* Accounting */
1075           nat44_session_update_counters (s0, now,
1076                                          vlib_buffer_length_in_chain (vm,
1077                                                                       b0));
1078
1079         trace00:
1080           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1081                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1082             {
1083               nat_in2out_ed_trace_t *t =
1084                 vlib_add_trace (vm, node, b0, sizeof (*t));
1085               t->is_slow_path = is_slow_path;
1086               t->sw_if_index = sw_if_index0;
1087               t->next_index = next0;
1088               t->session_index = ~0;
1089               if (s0)
1090                 t->session_index = s0 - tsm->sessions;
1091             }
1092
1093           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1094
1095
1096           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1097
1098           if (is_output_feature)
1099             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1100
1101           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1102                                   iph_offset1);
1103
1104           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1105           rx_fib_index1 =
1106             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1107                                                  sw_if_index1);
1108
1109           if (PREDICT_FALSE (ip1->ttl == 1))
1110             {
1111               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1112               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1113                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1114                                            0);
1115               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1116               goto trace01;
1117             }
1118
1119           udp1 = ip4_next_header (ip1);
1120           tcp1 = (tcp_header_t *) udp1;
1121           icmp1 = (icmp46_header_t *) udp1;
1122           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1123
1124           if (is_slow_path)
1125             {
1126               if (PREDICT_FALSE (proto1 == ~0))
1127                 {
1128                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1129                                                       rx_fib_index1,
1130                                                       thread_index, now, vm,
1131                                                       node);
1132                   if (!s1)
1133                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1134                   goto trace01;
1135                 }
1136
1137               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1138                 {
1139                   next1 = icmp_in2out_ed_slow_path
1140                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1141                      next1, now, thread_index, &s1);
1142                   goto trace01;
1143                 }
1144             }
1145           else
1146             {
1147               if (PREDICT_FALSE (proto1 == ~0))
1148                 {
1149                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1150                   goto trace01;
1151                 }
1152
1153               if (ip4_is_fragment (ip1))
1154                 {
1155                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1156                   goto trace01;
1157                 }
1158
1159               if (is_output_feature)
1160                 {
1161                   if (PREDICT_FALSE
1162                       (nat_not_translate_output_feature_fwd
1163                        (sm, ip1, thread_index, now, vm, b1)))
1164                     goto trace01;
1165                 }
1166
1167               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1168                 {
1169                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1170                   goto trace01;
1171                 }
1172             }
1173
1174           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1175                       ip1->protocol, rx_fib_index1, udp1->src_port,
1176                       udp1->dst_port);
1177
1178           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1179             {
1180               if (is_slow_path)
1181                 {
1182                   if (is_output_feature)
1183                     {
1184                       if (PREDICT_FALSE
1185                           (nat44_ed_not_translate_output_feature
1186                            (sm, ip1, ip1->protocol, udp1->src_port,
1187                             udp1->dst_port, thread_index, sw_if_index1,
1188                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1189                         goto trace01;
1190                     }
1191                   else
1192                     {
1193                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1194                                                                  sw_if_index1,
1195                                                                  ip1, proto1,
1196                                                                  rx_fib_index1,
1197                                                                  thread_index)))
1198                         goto trace01;
1199                     }
1200
1201                   next1 =
1202                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1203                                   next1, thread_index, now);
1204
1205                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1206                     goto trace01;
1207                 }
1208               else
1209                 {
1210                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1211                   goto trace01;
1212                 }
1213             }
1214           else
1215             {
1216               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1217             }
1218
1219           b1->flags |= VNET_BUFFER_F_IS_NATED;
1220
1221           if (!is_output_feature)
1222             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1223
1224           old_addr1 = ip1->src_address.as_u32;
1225           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1226           sum1 = ip1->checksum;
1227           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1228                                  src_address);
1229           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1230             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1231                                    s1->ext_host_addr.as_u32, ip4_header_t,
1232                                    dst_address);
1233           ip1->checksum = ip_csum_fold (sum1);
1234
1235           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1236             {
1237               old_port1 = tcp1->src_port;
1238               new_port1 = tcp1->src_port = s1->out2in.port;
1239
1240               sum1 = tcp1->checksum;
1241               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1242                                      dst_address);
1243               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1244                                      length);
1245               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1246                 {
1247                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1248                                          s1->ext_host_addr.as_u32,
1249                                          ip4_header_t, dst_address);
1250                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1251                                          s1->ext_host_port, ip4_header_t,
1252                                          length);
1253                   tcp1->dst_port = s1->ext_host_port;
1254                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1255                 }
1256               tcp1->checksum = ip_csum_fold (sum1);
1257               mss_clamping (sm, tcp1, &sum1);
1258               if (nat44_set_tcp_session_state_i2o
1259                   (sm, s1, tcp1, thread_index))
1260                 goto trace01;
1261             }
1262           else
1263             {
1264               udp1->src_port = s1->out2in.port;
1265               udp1->checksum = 0;
1266               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1267                 {
1268                   udp1->dst_port = s1->ext_host_port;
1269                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1270                 }
1271             }
1272
1273           /* Accounting */
1274           nat44_session_update_counters (s1, now,
1275                                          vlib_buffer_length_in_chain (vm,
1276                                                                       b1));
1277
1278         trace01:
1279           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1280                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1281             {
1282               nat_in2out_ed_trace_t *t =
1283                 vlib_add_trace (vm, node, b1, sizeof (*t));
1284               t->is_slow_path = is_slow_path;
1285               t->sw_if_index = sw_if_index1;
1286               t->next_index = next1;
1287               t->session_index = ~0;
1288               if (s1)
1289                 t->session_index = s1 - tsm->sessions;
1290             }
1291
1292           pkts_processed += next1 != NAT_IN2OUT_ED_NEXT_DROP;
1293
1294           /* verify speculative enqueues, maybe switch current next frame */
1295           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1296                                            to_next, n_left_to_next,
1297                                            bi0, bi1, next0, next1);
1298         }
1299
1300       while (n_left_from > 0 && n_left_to_next > 0)
1301         {
1302           u32 bi0;
1303           vlib_buffer_t *b0;
1304           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1305             new_addr0, old_addr0;
1306           u16 old_port0, new_port0;
1307           ip4_header_t *ip0;
1308           udp_header_t *udp0;
1309           tcp_header_t *tcp0;
1310           icmp46_header_t *icmp0;
1311           snat_session_t *s0 = 0;
1312           clib_bihash_kv_16_8_t kv0, value0;
1313           ip_csum_t sum0;
1314
1315           /* speculatively enqueue b0 to the current next frame */
1316           bi0 = from[0];
1317           to_next[0] = bi0;
1318           from += 1;
1319           to_next += 1;
1320           n_left_from -= 1;
1321           n_left_to_next -= 1;
1322
1323           b0 = vlib_get_buffer (vm, bi0);
1324           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1325
1326           if (is_output_feature)
1327             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1328
1329           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1330                                   iph_offset0);
1331
1332           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1333           rx_fib_index0 =
1334             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1335                                                  sw_if_index0);
1336
1337           if (PREDICT_FALSE (ip0->ttl == 1))
1338             {
1339               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1340               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1341                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1342                                            0);
1343               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1344               goto trace0;
1345             }
1346
1347           udp0 = ip4_next_header (ip0);
1348           tcp0 = (tcp_header_t *) udp0;
1349           icmp0 = (icmp46_header_t *) udp0;
1350           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1351
1352           if (is_slow_path)
1353             {
1354               if (PREDICT_FALSE (proto0 == ~0))
1355                 {
1356                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1357                                                       rx_fib_index0,
1358                                                       thread_index, now, vm,
1359                                                       node);
1360                   if (!s0)
1361                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1362                   goto trace0;
1363                 }
1364
1365               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1366                 {
1367                   next0 = icmp_in2out_ed_slow_path
1368                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1369                      next0, now, thread_index, &s0);
1370                   goto trace0;
1371                 }
1372             }
1373           else
1374             {
1375               if (PREDICT_FALSE (proto0 == ~0))
1376                 {
1377                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1378                   goto trace0;
1379                 }
1380
1381               if (ip4_is_fragment (ip0))
1382                 {
1383                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1384                   goto trace0;
1385                 }
1386
1387               if (is_output_feature)
1388                 {
1389                   if (PREDICT_FALSE
1390                       (nat_not_translate_output_feature_fwd
1391                        (sm, ip0, thread_index, now, vm, b0)))
1392                     goto trace0;
1393                 }
1394
1395               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1396                 {
1397                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1398                   goto trace0;
1399                 }
1400             }
1401
1402           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1403                       ip0->protocol, rx_fib_index0, udp0->src_port,
1404                       udp0->dst_port);
1405
1406           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1407             {
1408               if (is_slow_path)
1409                 {
1410                   if (is_output_feature)
1411                     {
1412                       if (PREDICT_FALSE
1413                           (nat44_ed_not_translate_output_feature
1414                            (sm, ip0, ip0->protocol, udp0->src_port,
1415                             udp0->dst_port, thread_index, sw_if_index0,
1416                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1417                         goto trace0;
1418                     }
1419                   else
1420                     {
1421                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1422                                                                  sw_if_index0,
1423                                                                  ip0, proto0,
1424                                                                  rx_fib_index0,
1425                                                                  thread_index)))
1426                         goto trace0;
1427                     }
1428
1429                   next0 =
1430                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1431                                   next0, thread_index, now);
1432
1433                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1434                     goto trace0;
1435                 }
1436               else
1437                 {
1438                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1439                   goto trace0;
1440                 }
1441             }
1442           else
1443             {
1444               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1445             }
1446
1447           b0->flags |= VNET_BUFFER_F_IS_NATED;
1448
1449           if (!is_output_feature)
1450             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1451
1452           old_addr0 = ip0->src_address.as_u32;
1453           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1454           sum0 = ip0->checksum;
1455           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1456                                  src_address);
1457           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1458             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1459                                    s0->ext_host_addr.as_u32, ip4_header_t,
1460                                    dst_address);
1461           ip0->checksum = ip_csum_fold (sum0);
1462
1463           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1464             {
1465               old_port0 = tcp0->src_port;
1466               new_port0 = tcp0->src_port = s0->out2in.port;
1467
1468               sum0 = tcp0->checksum;
1469               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1470                                      dst_address);
1471               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1472                                      length);
1473               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1474                 {
1475                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1476                                          s0->ext_host_addr.as_u32,
1477                                          ip4_header_t, dst_address);
1478                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1479                                          s0->ext_host_port, ip4_header_t,
1480                                          length);
1481                   tcp0->dst_port = s0->ext_host_port;
1482                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1483                 }
1484               mss_clamping (sm, tcp0, &sum0);
1485               tcp0->checksum = ip_csum_fold (sum0);
1486               if (nat44_set_tcp_session_state_i2o
1487                   (sm, s0, tcp0, thread_index))
1488                 goto trace0;
1489             }
1490           else
1491             {
1492               udp0->src_port = s0->out2in.port;
1493               udp0->checksum = 0;
1494               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1495                 {
1496                   udp0->dst_port = s0->ext_host_port;
1497                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1498                 }
1499             }
1500
1501           /* Accounting */
1502           nat44_session_update_counters (s0, now,
1503                                          vlib_buffer_length_in_chain (vm,
1504                                                                       b0));
1505
1506         trace0:
1507           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1508                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1509             {
1510               nat_in2out_ed_trace_t *t =
1511                 vlib_add_trace (vm, node, b0, sizeof (*t));
1512               t->is_slow_path = is_slow_path;
1513               t->sw_if_index = sw_if_index0;
1514               t->next_index = next0;
1515               t->session_index = ~0;
1516               if (s0)
1517                 t->session_index = s0 - tsm->sessions;
1518             }
1519
1520           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1521
1522           /* verify speculative enqueue, maybe switch current next frame */
1523           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1524                                            to_next, n_left_to_next,
1525                                            bi0, next0);
1526         }
1527
1528       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1529     }
1530
1531   vlib_node_increment_counter (vm, stats_node_index,
1532                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1533                                pkts_processed);
1534   return frame->n_vectors;
1535 }
1536
1537 static uword
1538 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1539                               vlib_node_runtime_t * node,
1540                               vlib_frame_t * frame)
1541 {
1542   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1543 }
1544
1545 /* *INDENT-OFF* */
1546 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1547   .function = nat44_ed_in2out_fast_path_fn,
1548   .name = "nat44-ed-in2out",
1549   .vector_size = sizeof (u32),
1550   .format_trace = format_nat_in2out_ed_trace,
1551   .type = VLIB_NODE_TYPE_INTERNAL,
1552   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1553   .error_strings = nat_in2out_ed_error_strings,
1554   .runtime_data_bytes = sizeof (snat_runtime_t),
1555   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1556   .next_nodes = {
1557     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1558     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1559     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1560     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1561     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1562   },
1563 };
1564 /* *INDENT-ON* */
1565
1566 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1567                               nat44_ed_in2out_fast_path_fn);
1568
1569 static uword
1570 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1571                                      vlib_node_runtime_t * node,
1572                                      vlib_frame_t * frame)
1573 {
1574   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1575 }
1576
1577 /* *INDENT-OFF* */
1578 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1579   .function = nat44_ed_in2out_output_fast_path_fn,
1580   .name = "nat44-ed-in2out-output",
1581   .vector_size = sizeof (u32),
1582   .format_trace = format_nat_in2out_ed_trace,
1583   .type = VLIB_NODE_TYPE_INTERNAL,
1584   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1585   .error_strings = nat_in2out_ed_error_strings,
1586   .runtime_data_bytes = sizeof (snat_runtime_t),
1587   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1588   .next_nodes = {
1589     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1590     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1591     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1592     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1593     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1594   },
1595 };
1596 /* *INDENT-ON* */
1597
1598 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1599                               nat44_ed_in2out_output_fast_path_fn);
1600
1601 static uword
1602 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1603                               vlib_node_runtime_t * node,
1604                               vlib_frame_t * frame)
1605 {
1606   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1607 }
1608
1609 /* *INDENT-OFF* */
1610 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1611   .function = nat44_ed_in2out_slow_path_fn,
1612   .name = "nat44-ed-in2out-slowpath",
1613   .vector_size = sizeof (u32),
1614   .format_trace = format_nat_in2out_ed_trace,
1615   .type = VLIB_NODE_TYPE_INTERNAL,
1616   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1617   .error_strings = nat_in2out_ed_error_strings,
1618   .runtime_data_bytes = sizeof (snat_runtime_t),
1619   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1620   .next_nodes = {
1621     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1622     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1623     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1624     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1625     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1626   },
1627 };
1628 /* *INDENT-ON* */
1629
1630 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1631                               nat44_ed_in2out_slow_path_fn);
1632
1633 static uword
1634 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1635                                      vlib_node_runtime_t * node,
1636                                      vlib_frame_t * frame)
1637 {
1638   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1639 }
1640
1641 /* *INDENT-OFF* */
1642 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1643   .function = nat44_ed_in2out_output_slow_path_fn,
1644   .name = "nat44-ed-in2out-output-slowpath",
1645   .vector_size = sizeof (u32),
1646   .format_trace = format_nat_in2out_ed_trace,
1647   .type = VLIB_NODE_TYPE_INTERNAL,
1648   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1649   .error_strings = nat_in2out_ed_error_strings,
1650   .runtime_data_bytes = sizeof (snat_runtime_t),
1651   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1652   .next_nodes = {
1653     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1654     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1655     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1656     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1657     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1658   },
1659 };
1660 /* *INDENT-ON* */
1661
1662 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1663                               nat44_ed_in2out_output_slow_path_fn);
1664
1665 static inline uword
1666 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1667                                       vlib_node_runtime_t * node,
1668                                       vlib_frame_t * frame,
1669                                       int is_output_feature)
1670 {
1671   u32 n_left_from, *from, *to_next;
1672   nat_in2out_ed_next_t next_index;
1673   u32 pkts_processed = 0;
1674   snat_main_t *sm = &snat_main;
1675   f64 now = vlib_time_now (vm);
1676   u32 thread_index = vm->thread_index;
1677   snat_main_per_thread_data_t *per_thread_data =
1678     &sm->per_thread_data[thread_index];
1679   u32 *fragments_to_drop = 0;
1680   u32 *fragments_to_loopback = 0;
1681
1682   from = vlib_frame_vector_args (frame);
1683   n_left_from = frame->n_vectors;
1684   next_index = node->cached_next_index;
1685
1686   while (n_left_from > 0)
1687     {
1688       u32 n_left_to_next;
1689
1690       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1691
1692       while (n_left_from > 0 && n_left_to_next > 0)
1693         {
1694           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1695           u32 iph_offset0 = 0;
1696           vlib_buffer_t *b0;
1697           u32 next0;
1698           u8 cached0 = 0;
1699           ip4_header_t *ip0 = 0;
1700           nat_reass_ip4_t *reass0;
1701           udp_header_t *udp0;
1702           tcp_header_t *tcp0;
1703           icmp46_header_t *icmp0;
1704           clib_bihash_kv_16_8_t kv0, value0;
1705           snat_session_t *s0 = 0;
1706           u16 old_port0, new_port0;
1707           ip_csum_t sum0;
1708
1709           /* speculatively enqueue b0 to the current next frame */
1710           bi0 = from[0];
1711           to_next[0] = bi0;
1712           from += 1;
1713           to_next += 1;
1714           n_left_from -= 1;
1715           n_left_to_next -= 1;
1716
1717           b0 = vlib_get_buffer (vm, bi0);
1718
1719           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1720
1721           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1722           rx_fib_index0 =
1723             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1724                                                  sw_if_index0);
1725
1726           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1727             {
1728               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1729               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1730               goto trace0;
1731             }
1732
1733           if (is_output_feature)
1734             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1735
1736           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1737                                   iph_offset0);
1738
1739           udp0 = ip4_next_header (ip0);
1740           tcp0 = (tcp_header_t *) udp0;
1741           icmp0 = (icmp46_header_t *) udp0;
1742           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1743
1744           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1745                                                  ip0->dst_address,
1746                                                  ip0->fragment_id,
1747                                                  ip0->protocol,
1748                                                  1, &fragments_to_drop);
1749
1750           if (PREDICT_FALSE (!reass0))
1751             {
1752               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1753               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1754               nat_log_notice ("maximum reassemblies exceeded");
1755               goto trace0;
1756             }
1757
1758           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1759             {
1760               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1761                 {
1762                   if (is_output_feature)
1763                     {
1764                       if (PREDICT_FALSE
1765                           (nat_not_translate_output_feature_fwd
1766                            (sm, ip0, thread_index, now, vm, b0)))
1767                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1768                       goto trace0;
1769                     }
1770
1771                   next0 = icmp_in2out_ed_slow_path
1772                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1773                      next0, now, thread_index, &s0);
1774
1775                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1776                     {
1777                       if (s0)
1778                         reass0->sess_index = s0 - per_thread_data->sessions;
1779                       else
1780                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1781                       nat_ip4_reass_get_frags (reass0,
1782                                                &fragments_to_loopback);
1783                     }
1784
1785                   goto trace0;
1786                 }
1787
1788               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1789                           ip0->protocol, rx_fib_index0, udp0->src_port,
1790                           udp0->dst_port);
1791
1792               if (clib_bihash_search_16_8
1793                   (&per_thread_data->in2out_ed, &kv0, &value0))
1794                 {
1795                   if (is_output_feature)
1796                     {
1797                       if (PREDICT_FALSE
1798                           (nat44_ed_not_translate_output_feature
1799                            (sm, ip0, ip0->protocol, udp0->src_port,
1800                             udp0->dst_port, thread_index, sw_if_index0,
1801                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1802                         {
1803                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1804                           nat_ip4_reass_get_frags (reass0,
1805                                                    &fragments_to_loopback);
1806                           goto trace0;
1807                         }
1808                     }
1809                   else
1810                     {
1811                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1812                                                                  sw_if_index0,
1813                                                                  ip0, proto0,
1814                                                                  rx_fib_index0,
1815                                                                  thread_index)))
1816                         {
1817                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1818                           nat_ip4_reass_get_frags (reass0,
1819                                                    &fragments_to_loopback);
1820                           goto trace0;
1821                         }
1822                     }
1823
1824                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1825                                         &s0, node, next0, thread_index, now);
1826
1827                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1828                     goto trace0;
1829
1830                   reass0->sess_index = s0 - per_thread_data->sessions;
1831                 }
1832               else
1833                 {
1834                   s0 = pool_elt_at_index (per_thread_data->sessions,
1835                                           value0.value);
1836                   reass0->sess_index = value0.value;
1837                 }
1838               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1839             }
1840           else
1841             {
1842               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1843                 goto trace0;
1844               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1845                 {
1846                   if (nat_ip4_reass_add_fragment
1847                       (reass0, bi0, &fragments_to_drop))
1848                     {
1849                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1850                       nat_log_notice
1851                         ("maximum fragments per reassembly exceeded");
1852                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1853                       goto trace0;
1854                     }
1855                   cached0 = 1;
1856                   goto trace0;
1857                 }
1858               s0 = pool_elt_at_index (per_thread_data->sessions,
1859                                       reass0->sess_index);
1860             }
1861
1862           old_addr0 = ip0->src_address.as_u32;
1863           ip0->src_address = s0->out2in.addr;
1864           new_addr0 = ip0->src_address.as_u32;
1865           if (!is_output_feature)
1866             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1867
1868           sum0 = ip0->checksum;
1869           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1870                                  ip4_header_t,
1871                                  src_address /* changed member */ );
1872           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1873             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1874                                    s0->ext_host_addr.as_u32, ip4_header_t,
1875                                    dst_address);
1876           ip0->checksum = ip_csum_fold (sum0);
1877
1878           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1879             {
1880               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1881                 {
1882                   old_port0 = tcp0->src_port;
1883                   tcp0->src_port = s0->out2in.port;
1884                   new_port0 = tcp0->src_port;
1885
1886                   sum0 = tcp0->checksum;
1887                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1888                                          ip4_header_t,
1889                                          dst_address /* changed member */ );
1890                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1891                                          ip4_header_t /* cheat */ ,
1892                                          length /* changed member */ );
1893                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1894                     {
1895                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1896                                              s0->ext_host_addr.as_u32,
1897                                              ip4_header_t, dst_address);
1898                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1899                                              s0->ext_host_port, ip4_header_t,
1900                                              length);
1901                       tcp0->dst_port = s0->ext_host_port;
1902                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1903                     }
1904                   tcp0->checksum = ip_csum_fold (sum0);
1905                 }
1906               else
1907                 {
1908                   old_port0 = udp0->src_port;
1909                   udp0->src_port = s0->out2in.port;
1910                   udp0->checksum = 0;
1911                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1912                     {
1913                       udp0->dst_port = s0->ext_host_port;
1914                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1915                     }
1916                 }
1917             }
1918
1919           /* Hairpinning */
1920           if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_ICMP))
1921             nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1922                                      s0->ext_host_port, proto0, 1);
1923           else
1924             snat_icmp_hairpinning (sm, b0, ip0, icmp0, 1);
1925
1926           /* Accounting */
1927           nat44_session_update_counters (s0, now,
1928                                          vlib_buffer_length_in_chain (vm,
1929                                                                       b0));
1930
1931         trace0:
1932           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1933                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1934             {
1935               nat44_reass_trace_t *t =
1936                 vlib_add_trace (vm, node, b0, sizeof (*t));
1937               t->cached = cached0;
1938               t->sw_if_index = sw_if_index0;
1939               t->next_index = next0;
1940             }
1941
1942           if (cached0)
1943             {
1944               n_left_to_next++;
1945               to_next--;
1946             }
1947           else
1948             {
1949               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1950
1951               /* verify speculative enqueue, maybe switch current next frame */
1952               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1953                                                to_next, n_left_to_next,
1954                                                bi0, next0);
1955             }
1956
1957           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1958             {
1959               from = vlib_frame_vector_args (frame);
1960               u32 len = vec_len (fragments_to_loopback);
1961               if (len <= VLIB_FRAME_SIZE)
1962                 {
1963                   clib_memcpy (from, fragments_to_loopback,
1964                                sizeof (u32) * len);
1965                   n_left_from = len;
1966                   vec_reset_length (fragments_to_loopback);
1967                 }
1968               else
1969                 {
1970                   clib_memcpy (from,
1971                                fragments_to_loopback + (len -
1972                                                         VLIB_FRAME_SIZE),
1973                                sizeof (u32) * VLIB_FRAME_SIZE);
1974                   n_left_from = VLIB_FRAME_SIZE;
1975                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1976                 }
1977             }
1978         }
1979
1980       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1981     }
1982
1983   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
1984                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1985                                pkts_processed);
1986
1987   nat_send_all_to_node (vm, fragments_to_drop, node,
1988                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
1989                         NAT_IN2OUT_ED_NEXT_DROP);
1990
1991   vec_free (fragments_to_drop);
1992   vec_free (fragments_to_loopback);
1993   return frame->n_vectors;
1994 }
1995
1996 static uword
1997 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
1998                                vlib_node_runtime_t * node,
1999                                vlib_frame_t * frame)
2000 {
2001   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2002 }
2003
2004 /* *INDENT-OFF* */
2005 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2006   .function = nat44_ed_in2out_reass_node_fn,
2007   .name = "nat44-ed-in2out-reass",
2008   .vector_size = sizeof (u32),
2009   .format_trace = format_nat44_reass_trace,
2010   .type = VLIB_NODE_TYPE_INTERNAL,
2011   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2012   .error_strings = nat_in2out_ed_error_strings,
2013   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2014   .next_nodes = {
2015     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2016     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2017     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2018     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2019     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2020   },
2021 };
2022 /* *INDENT-ON* */
2023
2024 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2025                               nat44_ed_in2out_reass_node_fn);
2026
2027 static uword
2028 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2029                                       vlib_node_runtime_t * node,
2030                                       vlib_frame_t * frame)
2031 {
2032   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2033 }
2034
2035 /* *INDENT-OFF* */
2036 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2037   .function = nat44_ed_in2out_reass_output_node_fn,
2038   .name = "nat44-ed-in2out-reass-output",
2039   .vector_size = sizeof (u32),
2040   .format_trace = format_nat44_reass_trace,
2041   .type = VLIB_NODE_TYPE_INTERNAL,
2042   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2043   .error_strings = nat_in2out_ed_error_strings,
2044   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2045   .next_nodes = {
2046     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2047     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2048     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2049     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2050     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2051   },
2052 };
2053 /* *INDENT-ON* */
2054
2055 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2056                               nat44_ed_in2out_reass_output_node_fn);
2057
2058 /*
2059  * fd.io coding-style-patch-verification: ON
2060  *
2061  * Local Variables:
2062  * eval: (c-set-style "gnu")
2063  * End:
2064  */