src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #define foreach_nat_in2out_ed_error                       \
  33 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  34 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  35 _(OUT_OF_PORTS, "Out of ports")                         \
  36 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  37 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  38 _(DROP_FRAGMENT, "Drop fragment")                       \
  39 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  40 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  41
  42 typedef enum
  43 {
  44 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
  45   foreach_nat_in2out_ed_error
  46 #undef _
  47     NAT_IN2OUT_ED_N_ERROR,
  48 } nat_in2out_ed_error_t;
  49
  50 static char *nat_in2out_ed_error_strings[] = {
  51 #define _(sym,string) string,
  52   foreach_nat_in2out_ed_error
  53 #undef _
  54 };
  55
  56 typedef enum
  57 {
  58   NAT_IN2OUT_ED_NEXT_LOOKUP,
  59   NAT_IN2OUT_ED_NEXT_DROP,
  60   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
  61   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
  62   NAT_IN2OUT_ED_NEXT_REASS,
  63   NAT_IN2OUT_ED_N_NEXT,
  64 } nat_in2out_ed_next_t;
  65
  66 typedef struct
  67 {
  68   u32 sw_if_index;
  69   u32 next_index;
  70   u32 session_index;
  71   u32 is_slow_path;
  72 } nat_in2out_ed_trace_t;
  73
  74 vlib_node_registration_t nat44_ed_in2out_node;
  75 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
  76 vlib_node_registration_t nat44_ed_in2out_output_node;
  77 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
  78 vlib_node_registration_t nat44_ed_in2out_reass_node;
  79
  80 static u8 *
  81 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  82 {
  83   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  84   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  85   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  86   char *tag;
  87
  88   tag =
  89     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  90     "NAT44_IN2OUT_ED_FAST_PATH";
  91
  92   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  93               t->sw_if_index, t->next_index, t->session_index);
  94
  95   return s;
  96 }
  97
  98 static_always_inline int
  99 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 100 {
 101   icmp46_header_t *icmp0;
 102   nat_ed_ses_key_t key0;
 103   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 104   ip4_header_t *inner_ip0 = 0;
 105   void *l4_header = 0;
 106   icmp46_header_t *inner_icmp0;
 107
 108   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 109   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 110
 111   if (!icmp_is_error_message (icmp0))
 112     {
 113       key0.proto = IP_PROTOCOL_ICMP;
 114       key0.l_addr = ip0->src_address;
 115       key0.r_addr = ip0->dst_address;
 116       key0.l_port = echo0->identifier;
 117       key0.r_port = 0;
 118     }
 119   else
 120     {
 121       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 122       l4_header = ip4_next_header (inner_ip0);
 123       key0.proto = inner_ip0->protocol;
 124       key0.r_addr = inner_ip0->src_address;
 125       key0.l_addr = inner_ip0->dst_address;
 126       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 127         {
 128         case SNAT_PROTOCOL_ICMP:
 129           inner_icmp0 = (icmp46_header_t *) l4_header;
 130           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 131           key0.r_port = 0;
 132           key0.l_port = inner_echo0->identifier;
 133           break;
 134         case SNAT_PROTOCOL_UDP:
 135         case SNAT_PROTOCOL_TCP:
 136           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 137           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
 138           break;
 139         default:
 140           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
 141         }
 142     }
 143   *p_key0 = key0;
 144   return 0;
 145 }
 146
 147 int
 148 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 149 {
 150   snat_main_t *sm = &snat_main;
 151   nat44_is_idle_session_ctx_t *ctx = arg;
 152   snat_session_t *s;
 153   u64 sess_timeout_time;
 154   nat_ed_ses_key_t ed_key;
 155   clib_bihash_kv_16_8_t ed_kv;
 156   int i;
 157   snat_address_t *a;
 158   snat_session_key_t key;
 159   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 160                                                        ctx->thread_index);
 161
 162   s = pool_elt_at_index (tsm->sessions, kv->value);
 163   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 164   if (ctx->now >= sess_timeout_time)
 165     {
 166       if (is_fwd_bypass_session (s))
 167         goto delete;
 168
 169       ed_key.l_addr = s->out2in.addr;
 170       ed_key.r_addr = s->ext_host_addr;
 171       ed_key.fib_index = s->out2in.fib_index;
 172       if (snat_is_unk_proto_session (s))
 173         {
 174           ed_key.proto = s->in2out.port;
 175           ed_key.r_port = 0;
 176           ed_key.l_port = 0;
 177         }
 178       else
 179         {
 180           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 181           ed_key.l_port = s->out2in.port;
 182           ed_key.r_port = s->ext_host_port;
 183         }
 184       ed_kv.key[0] = ed_key.as_u64[0];
 185       ed_kv.key[1] = ed_key.as_u64[1];
 186       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 187         nat_log_warn ("out2in_ed key del failed");
 188
 189       if (snat_is_unk_proto_session (s))
 190         goto delete;
 191
 192       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 193                                            s->out2in.addr.as_u32,
 194                                            s->in2out.protocol,
 195                                            s->in2out.port,
 196                                            s->out2in.port,
 197                                            s->in2out.fib_index);
 198
 199       if (is_twice_nat_session (s))
 200         {
 201           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 202             {
 203               key.protocol = s->in2out.protocol;
 204               key.port = s->ext_host_nat_port;
 205               a = sm->twice_nat_addresses + i;
 206               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 207                 {
 208                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 209                                                       ctx->thread_index,
 210                                                       &key);
 211                   break;
 212                 }
 213             }
 214         }
 215
 216       if (snat_is_session_static (s))
 217         goto delete;
 218
 219       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 220                                           &s->out2in);
 221     delete:
 222       nat44_delete_session (sm, s, ctx->thread_index);
 223       return 1;
 224     }
 225
 226   return 0;
 227 }
 228
 229 static inline u32
 230 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 231                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 232                           u32 sw_if_index0, u32 rx_fib_index0,
 233                           vlib_node_runtime_t * node, u32 next0, f64 now,
 234                           u32 thread_index, snat_session_t ** p_s0)
 235 {
 236   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 237                        next0, thread_index, p_s0, 0);
 238   snat_session_t *s0 = *p_s0;
 239   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
 240     {
 241       /* Accounting */
 242       nat44_session_update_counters (s0, now,
 243                                      vlib_buffer_length_in_chain
 244                                      (sm->vlib_main, b0));
 245       /* Per-user LRU list maintenance */
 246       nat44_session_update_lru (sm, s0, thread_index);
 247     }
 248   return next0;
 249 }
 250
 251 static u32
 252 slow_path_ed (snat_main_t * sm,
 253               vlib_buffer_t * b,
 254               u32 rx_fib_index,
 255               clib_bihash_kv_16_8_t * kv,
 256               snat_session_t ** sessionp,
 257               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 258 {
 259   snat_session_t *s;
 260   snat_user_t *u;
 261   snat_session_key_t key0, key1;
 262   lb_nat_type_t lb = 0, is_sm = 0;
 263   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 264   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 265   u32 proto = ip_proto_to_snat_proto (key->proto);
 266   nat_outside_fib_t *outside_fib;
 267   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 268   fib_prefix_t pfx = {
 269     .fp_proto = FIB_PROTOCOL_IP4,
 270     .fp_len = 32,
 271     .fp_addr = {
 272                 .ip4.as_u32 = key->r_addr.as_u32,
 273                 },
 274   };
 275   nat44_is_idle_session_ctx_t ctx;
 276
 277   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 278     {
 279       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 280       nat_ipfix_logging_max_sessions (sm->max_translations);
 281       nat_log_notice ("maximum sessions exceeded");
 282       return NAT_IN2OUT_ED_NEXT_DROP;
 283     }
 284
 285   key0.addr = key->l_addr;
 286   key0.port = key->l_port;
 287   key1.protocol = key0.protocol = proto;
 288   key0.fib_index = rx_fib_index;
 289   key1.fib_index = sm->outside_fib_index;
 290   /* First try to match static mapping by local address and port */
 291   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0))
 292     {
 293       /* Try to create dynamic translation */
 294       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 295                                                thread_index, &key1,
 296                                                sm->port_per_thread,
 297                                                tsm->snat_thread_index))
 298         {
 299           nat_log_notice ("addresses exhausted");
 300           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 301           return NAT_IN2OUT_ED_NEXT_DROP;
 302         }
 303     }
 304   else
 305     is_sm = 1;
 306
 307   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 308   if (!u)
 309     {
 310       nat_log_warn ("create NAT user failed");
 311       if (!is_sm)
 312         snat_free_outside_address_and_port (sm->addresses,
 313                                             thread_index, &key1);
 314       return NAT_IN2OUT_ED_NEXT_DROP;
 315     }
 316
 317   s = nat_ed_session_alloc (sm, u, thread_index, now);
 318   if (!s)
 319     {
 320       nat44_delete_user_with_no_session (sm, u, thread_index);
 321       nat_log_warn ("create NAT session failed");
 322       if (!is_sm)
 323         snat_free_outside_address_and_port (sm->addresses,
 324                                             thread_index, &key1);
 325       return NAT_IN2OUT_ED_NEXT_DROP;
 326     }
 327
 328   user_session_increment (sm, u, is_sm);
 329   if (is_sm)
 330     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 331   if (lb)
 332     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 333   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 334   s->ext_host_addr = key->r_addr;
 335   s->ext_host_port = key->r_port;
 336   s->in2out = key0;
 337   s->out2in = key1;
 338   s->out2in.protocol = key0.protocol;
 339
 340   switch (vec_len (sm->outside_fibs))
 341     {
 342     case 0:
 343       s->out2in.fib_index = sm->outside_fib_index;
 344       break;
 345     case 1:
 346       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 347       break;
 348     default:
 349       /* *INDENT-OFF* */
 350       vec_foreach (outside_fib, sm->outside_fibs)
 351        {
 352           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 353           if (FIB_NODE_INDEX_INVALID != fei)
 354             {
 355               if (fib_entry_get_resolving_interface (fei) != ~0)
 356                 {
 357                   s->out2in.fib_index = outside_fib->fib_index;
 358                   break;
 359                 }
 360             }
 361         }
 362       /* *INDENT-ON* */
 363       break;
 364     }
 365
 366   /* Add to lookup tables */
 367   kv->value = s - tsm->sessions;
 368   ctx.now = now;
 369   ctx.thread_index = thread_index;
 370   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 371                                                nat44_i2o_ed_is_idle_session_cb,
 372                                                &ctx))
 373     nat_log_notice ("in2out-ed key add failed");
 374
 375   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 376               key1.port, key->r_port);
 377   kv->value = s - tsm->sessions;
 378   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 379                                                nat44_o2i_ed_is_idle_session_cb,
 380                                                &ctx))
 381     nat_log_notice ("out2in-ed key add failed");
 382
 383   *sessionp = s;
 384
 385   /* log NAT event */
 386   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 387                                        s->out2in.addr.as_u32,
 388                                        s->in2out.protocol,
 389                                        s->in2out.port,
 390                                        s->out2in.port, s->in2out.fib_index);
 391   return next;
 392 }
 393
 394 static_always_inline int
 395 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 396                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 397                         u32 rx_fib_index, u32 thread_index)
 398 {
 399   udp_header_t *udp = ip4_next_header (ip);
 400   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 401   clib_bihash_kv_16_8_t kv, value;
 402   snat_session_key_t key0, key1;
 403
 404   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 405               sm->outside_fib_index, udp->dst_port, udp->src_port);
 406
 407   /* NAT packet aimed at external address if */
 408   /* has active sessions */
 409   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 410     {
 411       key0.addr = ip->dst_address;
 412       key0.port = udp->dst_port;
 413       key0.protocol = proto;
 414       key0.fib_index = sm->outside_fib_index;
 415       /* or is static mappings */
 416       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0))
 417         return 0;
 418     }
 419   else
 420     return 0;
 421
 422   if (sm->forwarding_enabled)
 423     return 1;
 424
 425   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 426                                   rx_fib_index);
 427 }
 428
 429 static_always_inline int
 430 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 431                                       u32 thread_index, f64 now,
 432                                       vlib_main_t * vm, vlib_buffer_t * b)
 433 {
 434   nat_ed_ses_key_t key;
 435   clib_bihash_kv_16_8_t kv, value;
 436   udp_header_t *udp;
 437   snat_session_t *s = 0;
 438   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 439
 440   if (!sm->forwarding_enabled)
 441     return 0;
 442
 443   if (ip->protocol == IP_PROTOCOL_ICMP)
 444     {
 445       key.as_u64[0] = key.as_u64[1] = 0;
 446       if (icmp_get_ed_key (ip, &key))
 447         return 0;
 448       key.fib_index = 0;
 449       kv.key[0] = key.as_u64[0];
 450       kv.key[1] = key.as_u64[1];
 451     }
 452   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 453     {
 454       udp = ip4_next_header (ip);
 455       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 456                   udp->src_port, udp->dst_port);
 457     }
 458   else
 459     {
 460       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 461                   0);
 462     }
 463
 464   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 465     {
 466       s = pool_elt_at_index (tsm->sessions, value.value);
 467       if (is_fwd_bypass_session (s))
 468         {
 469           if (ip->protocol == IP_PROTOCOL_TCP)
 470             {
 471               tcp_header_t *tcp = ip4_next_header (ip);
 472               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 473                 return 1;
 474             }
 475           /* Accounting */
 476           nat44_session_update_counters (s, now,
 477                                          vlib_buffer_length_in_chain (vm, b));
 478           /* Per-user LRU list maintenance */
 479           nat44_session_update_lru (sm, s, thread_index);
 480           return 1;
 481         }
 482       else
 483         return 0;
 484     }
 485
 486   return 0;
 487 }
 488
 489 static_always_inline int
 490 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 491                                        u8 proto, u16 src_port, u16 dst_port,
 492                                        u32 thread_index, u32 rx_sw_if_index,
 493                                        u32 tx_sw_if_index)
 494 {
 495   clib_bihash_kv_16_8_t kv, value;
 496   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 497   snat_interface_t *i;
 498   snat_session_t *s;
 499   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 500   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 501
 502   /* src NAT check */
 503   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 504               src_port, dst_port);
 505   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 506     return 1;
 507
 508   /* dst NAT check */
 509   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 510               dst_port, src_port);
 511   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 512     {
 513       s = pool_elt_at_index (tsm->sessions, value.value);
 514       if (is_fwd_bypass_session (s))
 515         return 0;
 516
 517       /* hairpinning */
 518       /* *INDENT-OFF* */
 519       pool_foreach (i, sm->output_feature_interfaces,
 520       ({
 521         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 522            return 0;
 523       }));
 524       /* *INDENT-ON* */
 525       return 1;
 526     }
 527
 528   return 0;
 529 }
 530
 531 u32
 532 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 533                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 534                       u8 * p_proto, snat_session_key_t * p_value,
 535                       u8 * p_dont_translate, void *d, void *e)
 536 {
 537   icmp46_header_t *icmp;
 538   u32 sw_if_index;
 539   u32 rx_fib_index;
 540   nat_ed_ses_key_t key;
 541   snat_session_t *s = 0;
 542   u8 dont_translate = 0;
 543   clib_bihash_kv_16_8_t kv, value;
 544   u32 next = ~0;
 545   int err;
 546   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 547
 548   icmp = (icmp46_header_t *) ip4_next_header (ip);
 549   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 550   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 551
 552   key.as_u64[0] = key.as_u64[1] = 0;
 553   err = icmp_get_ed_key (ip, &key);
 554   if (err != 0)
 555     {
 556       b->error = node->errors[err];
 557       next = NAT_IN2OUT_ED_NEXT_DROP;
 558       goto out;
 559     }
 560   key.fib_index = rx_fib_index;
 561
 562   kv.key[0] = key.as_u64[0];
 563   kv.key[1] = key.as_u64[1];
 564
 565   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 566     {
 567       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 568         {
 569           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 570                                                                     key.proto,
 571                                                                     key.
 572                                                                     l_port,
 573                                                                     key.
 574                                                                     r_port,
 575                                                                     thread_index,
 576                                                                     sw_if_index,
 577                                                                     vnet_buffer
 578                                                                     (b)->
 579                                                                     sw_if_index
 580                                                                     [VLIB_TX])))
 581             {
 582               dont_translate = 1;
 583               goto out;
 584             }
 585         }
 586       else
 587         {
 588           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 589                                                      ip, SNAT_PROTOCOL_ICMP,
 590                                                      rx_fib_index,
 591                                                      thread_index)))
 592             {
 593               dont_translate = 1;
 594               goto out;
 595             }
 596         }
 597
 598       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 599         {
 600           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 601           next = NAT_IN2OUT_ED_NEXT_DROP;
 602           goto out;
 603         }
 604
 605       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 606                            thread_index, vlib_time_now (sm->vlib_main));
 607
 608       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
 609         goto out;
 610     }
 611   else
 612     {
 613       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 614                          icmp->type != ICMP4_echo_reply &&
 615                          !icmp_is_error_message (icmp)))
 616         {
 617           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 618           next = NAT_IN2OUT_ED_NEXT_DROP;
 619           goto out;
 620         }
 621
 622       s = pool_elt_at_index (tsm->sessions, value.value);
 623     }
 624
 625   *p_proto = ip_proto_to_snat_proto (key.proto);
 626 out:
 627   if (s)
 628     *p_value = s->out2in;
 629   *p_dont_translate = dont_translate;
 630   if (d)
 631     *(snat_session_t **) d = s;
 632   return next;
 633 }
 634
 635 static snat_session_t *
 636 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 637                                vlib_buffer_t * b,
 638                                ip4_header_t * ip,
 639                                u32 rx_fib_index,
 640                                u32 thread_index,
 641                                f64 now,
 642                                vlib_main_t * vm, vlib_node_runtime_t * node)
 643 {
 644   clib_bihash_kv_8_8_t kv, value;
 645   clib_bihash_kv_16_8_t s_kv, s_value;
 646   snat_static_mapping_t *m;
 647   u32 old_addr, new_addr = 0;
 648   ip_csum_t sum;
 649   snat_user_t *u;
 650   dlist_elt_t *head, *elt;
 651   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 652   u32 elt_index, head_index, ses_index;
 653   snat_session_t *s;
 654   u32 outside_fib_index = sm->outside_fib_index;
 655   int i;
 656   u8 is_sm = 0;
 657   nat_outside_fib_t *outside_fib;
 658   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 659   fib_prefix_t pfx = {
 660     .fp_proto = FIB_PROTOCOL_IP4,
 661     .fp_len = 32,
 662     .fp_addr = {
 663                 .ip4.as_u32 = ip->dst_address.as_u32,
 664                 },
 665   };
 666
 667   switch (vec_len (sm->outside_fibs))
 668     {
 669     case 0:
 670       outside_fib_index = sm->outside_fib_index;
 671       break;
 672     case 1:
 673       outside_fib_index = sm->outside_fibs[0].fib_index;
 674       break;
 675     default:
 676       /* *INDENT-OFF* */
 677       vec_foreach (outside_fib, sm->outside_fibs)
 678         {
 679           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 680           if (FIB_NODE_INDEX_INVALID != fei)
 681             {
 682               if (fib_entry_get_resolving_interface (fei) != ~0)
 683                 {
 684                   outside_fib_index = outside_fib->fib_index;
 685                   break;
 686                 }
 687             }
 688         }
 689       /* *INDENT-ON* */
 690       break;
 691     }
 692   old_addr = ip->src_address.as_u32;
 693
 694   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 695               rx_fib_index, 0, 0);
 696
 697   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 698     {
 699       s = pool_elt_at_index (tsm->sessions, s_value.value);
 700       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 701     }
 702   else
 703     {
 704       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 705         {
 706           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 707           nat_ipfix_logging_max_sessions (sm->max_translations);
 708           nat_log_notice ("maximum sessions exceeded");
 709           return 0;
 710         }
 711
 712       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 713                                   thread_index);
 714       if (!u)
 715         {
 716           nat_log_warn ("create NAT user failed");
 717           return 0;
 718         }
 719
 720       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 721
 722       /* Try to find static mapping first */
 723       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 724         {
 725           m = pool_elt_at_index (sm->static_mappings, value.value);
 726           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 727           is_sm = 1;
 728           goto create_ses;
 729         }
 730       /* Fallback to 3-tuple key */
 731       else
 732         {
 733           /* Choose same out address as for TCP/UDP session to same destination */
 734           head_index = u->sessions_per_user_list_head_index;
 735           head = pool_elt_at_index (tsm->list_pool, head_index);
 736           elt_index = head->next;
 737           if (PREDICT_FALSE (elt_index == ~0))
 738             ses_index = ~0;
 739           else
 740             {
 741               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 742               ses_index = elt->value;
 743             }
 744
 745           while (ses_index != ~0)
 746             {
 747               s = pool_elt_at_index (tsm->sessions, ses_index);
 748               elt_index = elt->next;
 749               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 750               ses_index = elt->value;
 751
 752               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 753                 {
 754                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 755
 756                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 757                               ip->protocol, outside_fib_index, 0, 0);
 758                   if (clib_bihash_search_16_8
 759                       (&tsm->out2in_ed, &s_kv, &s_value))
 760                     goto create_ses;
 761
 762                   break;
 763                 }
 764             }
 765
 766           for (i = 0; i < vec_len (sm->addresses); i++)
 767             {
 768               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 769                           ip->protocol, outside_fib_index, 0, 0);
 770               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 771                 {
 772                   new_addr = ip->src_address.as_u32 =
 773                     sm->addresses[i].addr.as_u32;
 774                   goto create_ses;
 775                 }
 776             }
 777           return 0;
 778         }
 779
 780     create_ses:
 781       s = nat_ed_session_alloc (sm, u, thread_index, now);
 782       if (!s)
 783         {
 784           nat44_delete_user_with_no_session (sm, u, thread_index);
 785           nat_log_warn ("create NAT session failed");
 786           return 0;
 787         }
 788
 789       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 790       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 791       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 792       s->out2in.addr.as_u32 = new_addr;
 793       s->out2in.fib_index = outside_fib_index;
 794       s->in2out.addr.as_u32 = old_addr;
 795       s->in2out.fib_index = rx_fib_index;
 796       s->in2out.port = s->out2in.port = ip->protocol;
 797       if (is_sm)
 798         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 799       user_session_increment (sm, u, is_sm);
 800
 801       /* Add to lookup tables */
 802       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 803                   rx_fib_index, 0, 0);
 804       s_kv.value = s - tsm->sessions;
 805       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 806         nat_log_notice ("in2out key add failed");
 807
 808       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 809                   outside_fib_index, 0, 0);
 810       s_kv.value = s - tsm->sessions;
 811       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 812         nat_log_notice ("out2in key add failed");
 813     }
 814
 815   /* Update IP checksum */
 816   sum = ip->checksum;
 817   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 818   ip->checksum = ip_csum_fold (sum);
 819
 820   /* Accounting */
 821   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 822   /* Per-user LRU list maintenance */
 823   nat44_session_update_lru (sm, s, thread_index);
 824
 825   /* Hairpinning */
 826   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 827     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 828
 829   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 830     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 831
 832   return s;
 833 }
 834
 835 static inline uword
 836 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 837                                 vlib_node_runtime_t * node,
 838                                 vlib_frame_t * frame, int is_slow_path,
 839                                 int is_output_feature)
 840 {
 841   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 842   nat_in2out_ed_next_t next_index;
 843   snat_main_t *sm = &snat_main;
 844   f64 now = vlib_time_now (vm);
 845   u32 thread_index = vm->thread_index;
 846   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 847
 848   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
 849     nat44_ed_in2out_node.index;
 850
 851   from = vlib_frame_vector_args (frame);
 852   n_left_from = frame->n_vectors;
 853   next_index = node->cached_next_index;
 854
 855   while (n_left_from > 0)
 856     {
 857       u32 n_left_to_next;
 858
 859       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 860
 861       while (n_left_from >= 4 && n_left_to_next >= 2)
 862         {
 863           u32 bi0, bi1;
 864           vlib_buffer_t *b0, *b1;
 865           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 866             new_addr0, old_addr0;
 867           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 868             new_addr1, old_addr1;
 869           u16 old_port0, new_port0, old_port1, new_port1;
 870           ip4_header_t *ip0, *ip1;
 871           udp_header_t *udp0, *udp1;
 872           tcp_header_t *tcp0, *tcp1;
 873           icmp46_header_t *icmp0, *icmp1;
 874           snat_session_t *s0 = 0, *s1 = 0;
 875           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 876           ip_csum_t sum0, sum1;
 877
 878           /* Prefetch next iteration. */
 879           {
 880             vlib_buffer_t *p2, *p3;
 881
 882             p2 = vlib_get_buffer (vm, from[2]);
 883             p3 = vlib_get_buffer (vm, from[3]);
 884
 885             vlib_prefetch_buffer_header (p2, LOAD);
 886             vlib_prefetch_buffer_header (p3, LOAD);
 887
 888             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 889             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 890           }
 891
 892           /* speculatively enqueue b0 and b1 to the current next frame */
 893           to_next[0] = bi0 = from[0];
 894           to_next[1] = bi1 = from[1];
 895           from += 2;
 896           to_next += 2;
 897           n_left_from -= 2;
 898           n_left_to_next -= 2;
 899
 900           b0 = vlib_get_buffer (vm, bi0);
 901           b1 = vlib_get_buffer (vm, bi1);
 902
 903           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
 904
 905           if (is_output_feature)
 906             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 907
 908           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 909                                   iph_offset0);
 910
 911           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 912           rx_fib_index0 =
 913             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 914                                                  sw_if_index0);
 915
 916           if (PREDICT_FALSE (ip0->ttl == 1))
 917             {
 918               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 919               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 920                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 921                                            0);
 922               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
 923               goto trace00;
 924             }
 925
 926           udp0 = ip4_next_header (ip0);
 927           tcp0 = (tcp_header_t *) udp0;
 928           icmp0 = (icmp46_header_t *) udp0;
 929           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 930
 931           if (is_slow_path)
 932             {
 933               if (PREDICT_FALSE (proto0 == ~0))
 934                 {
 935                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
 936                                                       rx_fib_index0,
 937                                                       thread_index, now, vm,
 938                                                       node);
 939                   if (!s0)
 940                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
 941                   goto trace00;
 942                 }
 943
 944               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 945                 {
 946                   next0 = icmp_in2out_ed_slow_path
 947                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 948                      next0, now, thread_index, &s0);
 949                   goto trace00;
 950                 }
 951             }
 952           else
 953             {
 954               if (PREDICT_FALSE (proto0 == ~0))
 955                 {
 956                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 957                   goto trace00;
 958                 }
 959
 960               if (ip4_is_fragment (ip0))
 961                 {
 962                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
 963                   goto trace00;
 964                 }
 965
 966               if (is_output_feature)
 967                 {
 968                   if (PREDICT_FALSE
 969                       (nat_not_translate_output_feature_fwd
 970                        (sm, ip0, thread_index, now, vm, b0)))
 971                     goto trace00;
 972                 }
 973
 974               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 975                 {
 976                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 977                   goto trace00;
 978                 }
 979             }
 980
 981           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
 982                       ip0->protocol, rx_fib_index0, udp0->src_port,
 983                       udp0->dst_port);
 984
 985           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
 986             {
 987               if (is_slow_path)
 988                 {
 989                   if (is_output_feature)
 990                     {
 991                       if (PREDICT_FALSE
 992                           (nat44_ed_not_translate_output_feature
 993                            (sm, ip0, ip0->protocol, udp0->src_port,
 994                             udp0->dst_port, thread_index, sw_if_index0,
 995                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
 996                         goto trace00;
 997                     }
 998                   else
 999                     {
1000                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1001                                                                  sw_if_index0,
1002                                                                  ip0, proto0,
1003                                                                  rx_fib_index0,
1004                                                                  thread_index)))
1005                         goto trace00;
1006                     }
1007
1008                   next0 =
1009                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1010                                   next0, thread_index, now);
1011
1012                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1013                     goto trace00;
1014                 }
1015               else
1016                 {
1017                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1018                   goto trace00;
1019                 }
1020             }
1021           else
1022             {
1023               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1024             }
1025
1026           b0->flags |= VNET_BUFFER_F_IS_NATED;
1027
1028           if (!is_output_feature)
1029             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1030
1031           old_addr0 = ip0->src_address.as_u32;
1032           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1033           sum0 = ip0->checksum;
1034           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1035                                  src_address);
1036           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1037             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1038                                    s0->ext_host_addr.as_u32, ip4_header_t,
1039                                    dst_address);
1040           ip0->checksum = ip_csum_fold (sum0);
1041
1042           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1043             {
1044               old_port0 = tcp0->src_port;
1045               new_port0 = tcp0->src_port = s0->out2in.port;
1046
1047               sum0 = tcp0->checksum;
1048               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1049                                      dst_address);
1050               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1051                                      length);
1052               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1053                 {
1054                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1055                                          s0->ext_host_addr.as_u32,
1056                                          ip4_header_t, dst_address);
1057                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1058                                          s0->ext_host_port, ip4_header_t,
1059                                          length);
1060                   tcp0->dst_port = s0->ext_host_port;
1061                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1062                 }
1063               mss_clamping (sm, tcp0, &sum0);
1064               tcp0->checksum = ip_csum_fold (sum0);
1065               if (nat44_set_tcp_session_state_i2o
1066                   (sm, s0, tcp0, thread_index))
1067                 goto trace00;
1068             }
1069           else
1070             {
1071               udp0->src_port = s0->out2in.port;
1072               udp0->checksum = 0;
1073               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1074                 {
1075                   udp0->dst_port = s0->ext_host_port;
1076                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1077                 }
1078             }
1079
1080           /* Accounting */
1081           nat44_session_update_counters (s0, now,
1082                                          vlib_buffer_length_in_chain (vm,
1083                                                                       b0));
1084           /* Per-user LRU list maintenance */
1085           nat44_session_update_lru (sm, s0, thread_index);
1086
1087         trace00:
1088           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1089                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1090             {
1091               nat_in2out_ed_trace_t *t =
1092                 vlib_add_trace (vm, node, b0, sizeof (*t));
1093               t->is_slow_path = is_slow_path;
1094               t->sw_if_index = sw_if_index0;
1095               t->next_index = next0;
1096               t->session_index = ~0;
1097               if (s0)
1098                 t->session_index = s0 - tsm->sessions;
1099             }
1100
1101           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1102
1103
1104           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1105
1106           if (is_output_feature)
1107             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1108
1109           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1110                                   iph_offset1);
1111
1112           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1113           rx_fib_index1 =
1114             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1115                                                  sw_if_index1);
1116
1117           if (PREDICT_FALSE (ip1->ttl == 1))
1118             {
1119               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1120               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1121                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1122                                            0);
1123               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1124               goto trace01;
1125             }
1126
1127           udp1 = ip4_next_header (ip1);
1128           tcp1 = (tcp_header_t *) udp1;
1129           icmp1 = (icmp46_header_t *) udp1;
1130           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1131
1132           if (is_slow_path)
1133             {
1134               if (PREDICT_FALSE (proto1 == ~0))
1135                 {
1136                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1137                                                       rx_fib_index1,
1138                                                       thread_index, now, vm,
1139                                                       node);
1140                   if (!s1)
1141                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1142                   goto trace01;
1143                 }
1144
1145               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1146                 {
1147                   next1 = icmp_in2out_ed_slow_path
1148                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1149                      next1, now, thread_index, &s1);
1150                   goto trace01;
1151                 }
1152             }
1153           else
1154             {
1155               if (PREDICT_FALSE (proto1 == ~0))
1156                 {
1157                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1158                   goto trace01;
1159                 }
1160
1161               if (ip4_is_fragment (ip1))
1162                 {
1163                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1164                   goto trace01;
1165                 }
1166
1167               if (is_output_feature)
1168                 {
1169                   if (PREDICT_FALSE
1170                       (nat_not_translate_output_feature_fwd
1171                        (sm, ip1, thread_index, now, vm, b1)))
1172                     goto trace01;
1173                 }
1174
1175               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1176                 {
1177                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1178                   goto trace01;
1179                 }
1180             }
1181
1182           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1183                       ip1->protocol, rx_fib_index1, udp1->src_port,
1184                       udp1->dst_port);
1185
1186           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1187             {
1188               if (is_slow_path)
1189                 {
1190                   if (is_output_feature)
1191                     {
1192                       if (PREDICT_FALSE
1193                           (nat44_ed_not_translate_output_feature
1194                            (sm, ip1, ip1->protocol, udp1->src_port,
1195                             udp1->dst_port, thread_index, sw_if_index1,
1196                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1197                         goto trace01;
1198                     }
1199                   else
1200                     {
1201                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1202                                                                  sw_if_index1,
1203                                                                  ip1, proto1,
1204                                                                  rx_fib_index1,
1205                                                                  thread_index)))
1206                         goto trace01;
1207                     }
1208
1209                   next1 =
1210                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1211                                   next1, thread_index, now);
1212
1213                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1214                     goto trace01;
1215                 }
1216               else
1217                 {
1218                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1219                   goto trace01;
1220                 }
1221             }
1222           else
1223             {
1224               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1225             }
1226
1227           b1->flags |= VNET_BUFFER_F_IS_NATED;
1228
1229           if (!is_output_feature)
1230             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1231
1232           old_addr1 = ip1->src_address.as_u32;
1233           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1234           sum1 = ip1->checksum;
1235           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1236                                  src_address);
1237           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1238             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1239                                    s1->ext_host_addr.as_u32, ip4_header_t,
1240                                    dst_address);
1241           ip1->checksum = ip_csum_fold (sum1);
1242
1243           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1244             {
1245               old_port1 = tcp1->src_port;
1246               new_port1 = tcp1->src_port = s1->out2in.port;
1247
1248               sum1 = tcp1->checksum;
1249               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1250                                      dst_address);
1251               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1252                                      length);
1253               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1254                 {
1255                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1256                                          s1->ext_host_addr.as_u32,
1257                                          ip4_header_t, dst_address);
1258                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1259                                          s1->ext_host_port, ip4_header_t,
1260                                          length);
1261                   tcp1->dst_port = s1->ext_host_port;
1262                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1263                 }
1264               tcp1->checksum = ip_csum_fold (sum1);
1265               mss_clamping (sm, tcp1, &sum1);
1266               if (nat44_set_tcp_session_state_i2o
1267                   (sm, s1, tcp1, thread_index))
1268                 goto trace01;
1269             }
1270           else
1271             {
1272               udp1->src_port = s1->out2in.port;
1273               udp1->checksum = 0;
1274               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1275                 {
1276                   udp1->dst_port = s1->ext_host_port;
1277                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1278                 }
1279             }
1280
1281           /* Accounting */
1282           nat44_session_update_counters (s1, now,
1283                                          vlib_buffer_length_in_chain (vm,
1284                                                                       b1));
1285           /* Per-user LRU list maintenance */
1286           nat44_session_update_lru (sm, s1, thread_index);
1287
1288         trace01:
1289           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1290                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1291             {
1292               nat_in2out_ed_trace_t *t =
1293                 vlib_add_trace (vm, node, b1, sizeof (*t));
1294               t->is_slow_path = is_slow_path;
1295               t->sw_if_index = sw_if_index1;
1296               t->next_index = next1;
1297               t->session_index = ~0;
1298               if (s1)
1299                 t->session_index = s1 - tsm->sessions;
1300             }
1301
1302           pkts_processed += next1 != NAT_IN2OUT_ED_NEXT_DROP;
1303
1304           /* verify speculative enqueues, maybe switch current next frame */
1305           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1306                                            to_next, n_left_to_next,
1307                                            bi0, bi1, next0, next1);
1308         }
1309
1310       while (n_left_from > 0 && n_left_to_next > 0)
1311         {
1312           u32 bi0;
1313           vlib_buffer_t *b0;
1314           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1315             new_addr0, old_addr0;
1316           u16 old_port0, new_port0;
1317           ip4_header_t *ip0;
1318           udp_header_t *udp0;
1319           tcp_header_t *tcp0;
1320           icmp46_header_t *icmp0;
1321           snat_session_t *s0 = 0;
1322           clib_bihash_kv_16_8_t kv0, value0;
1323           ip_csum_t sum0;
1324
1325           /* speculatively enqueue b0 to the current next frame */
1326           bi0 = from[0];
1327           to_next[0] = bi0;
1328           from += 1;
1329           to_next += 1;
1330           n_left_from -= 1;
1331           n_left_to_next -= 1;
1332
1333           b0 = vlib_get_buffer (vm, bi0);
1334           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1335
1336           if (is_output_feature)
1337             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1338
1339           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1340                                   iph_offset0);
1341
1342           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1343           rx_fib_index0 =
1344             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1345                                                  sw_if_index0);
1346
1347           if (PREDICT_FALSE (ip0->ttl == 1))
1348             {
1349               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1350               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1351                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1352                                            0);
1353               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1354               goto trace0;
1355             }
1356
1357           udp0 = ip4_next_header (ip0);
1358           tcp0 = (tcp_header_t *) udp0;
1359           icmp0 = (icmp46_header_t *) udp0;
1360           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1361
1362           if (is_slow_path)
1363             {
1364               if (PREDICT_FALSE (proto0 == ~0))
1365                 {
1366                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1367                                                       rx_fib_index0,
1368                                                       thread_index, now, vm,
1369                                                       node);
1370                   if (!s0)
1371                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1372                   goto trace0;
1373                 }
1374
1375               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1376                 {
1377                   next0 = icmp_in2out_ed_slow_path
1378                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1379                      next0, now, thread_index, &s0);
1380                   goto trace0;
1381                 }
1382             }
1383           else
1384             {
1385               if (PREDICT_FALSE (proto0 == ~0))
1386                 {
1387                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1388                   goto trace0;
1389                 }
1390
1391               if (ip4_is_fragment (ip0))
1392                 {
1393                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1394                   goto trace0;
1395                 }
1396
1397               if (is_output_feature)
1398                 {
1399                   if (PREDICT_FALSE
1400                       (nat_not_translate_output_feature_fwd
1401                        (sm, ip0, thread_index, now, vm, b0)))
1402                     goto trace0;
1403                 }
1404
1405               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1406                 {
1407                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1408                   goto trace0;
1409                 }
1410             }
1411
1412           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1413                       ip0->protocol, rx_fib_index0, udp0->src_port,
1414                       udp0->dst_port);
1415
1416           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1417             {
1418               if (is_slow_path)
1419                 {
1420                   if (is_output_feature)
1421                     {
1422                       if (PREDICT_FALSE
1423                           (nat44_ed_not_translate_output_feature
1424                            (sm, ip0, ip0->protocol, udp0->src_port,
1425                             udp0->dst_port, thread_index, sw_if_index0,
1426                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1427                         goto trace0;
1428                     }
1429                   else
1430                     {
1431                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1432                                                                  sw_if_index0,
1433                                                                  ip0, proto0,
1434                                                                  rx_fib_index0,
1435                                                                  thread_index)))
1436                         goto trace0;
1437                     }
1438
1439                   next0 =
1440                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1441                                   next0, thread_index, now);
1442
1443                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1444                     goto trace0;
1445                 }
1446               else
1447                 {
1448                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1449                   goto trace0;
1450                 }
1451             }
1452           else
1453             {
1454               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1455             }
1456
1457           b0->flags |= VNET_BUFFER_F_IS_NATED;
1458
1459           if (!is_output_feature)
1460             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1461
1462           old_addr0 = ip0->src_address.as_u32;
1463           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1464           sum0 = ip0->checksum;
1465           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1466                                  src_address);
1467           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1468             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1469                                    s0->ext_host_addr.as_u32, ip4_header_t,
1470                                    dst_address);
1471           ip0->checksum = ip_csum_fold (sum0);
1472
1473           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1474             {
1475               old_port0 = tcp0->src_port;
1476               new_port0 = tcp0->src_port = s0->out2in.port;
1477
1478               sum0 = tcp0->checksum;
1479               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1480                                      dst_address);
1481               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1482                                      length);
1483               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1484                 {
1485                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1486                                          s0->ext_host_addr.as_u32,
1487                                          ip4_header_t, dst_address);
1488                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1489                                          s0->ext_host_port, ip4_header_t,
1490                                          length);
1491                   tcp0->dst_port = s0->ext_host_port;
1492                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1493                 }
1494               mss_clamping (sm, tcp0, &sum0);
1495               tcp0->checksum = ip_csum_fold (sum0);
1496               if (nat44_set_tcp_session_state_i2o
1497                   (sm, s0, tcp0, thread_index))
1498                 goto trace0;
1499             }
1500           else
1501             {
1502               udp0->src_port = s0->out2in.port;
1503               udp0->checksum = 0;
1504               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1505                 {
1506                   udp0->dst_port = s0->ext_host_port;
1507                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1508                 }
1509             }
1510
1511           /* Accounting */
1512           nat44_session_update_counters (s0, now,
1513                                          vlib_buffer_length_in_chain (vm,
1514                                                                       b0));
1515           /* Per-user LRU list maintenance */
1516           nat44_session_update_lru (sm, s0, thread_index);
1517
1518         trace0:
1519           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1520                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1521             {
1522               nat_in2out_ed_trace_t *t =
1523                 vlib_add_trace (vm, node, b0, sizeof (*t));
1524               t->is_slow_path = is_slow_path;
1525               t->sw_if_index = sw_if_index0;
1526               t->next_index = next0;
1527               t->session_index = ~0;
1528               if (s0)
1529                 t->session_index = s0 - tsm->sessions;
1530             }
1531
1532           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1533
1534           /* verify speculative enqueue, maybe switch current next frame */
1535           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1536                                            to_next, n_left_to_next,
1537                                            bi0, next0);
1538         }
1539
1540       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1541     }
1542
1543   vlib_node_increment_counter (vm, stats_node_index,
1544                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1545                                pkts_processed);
1546   return frame->n_vectors;
1547 }
1548
1549 static uword
1550 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1551                               vlib_node_runtime_t * node,
1552                               vlib_frame_t * frame)
1553 {
1554   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1555 }
1556
1557 /* *INDENT-OFF* */
1558 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1559   .function = nat44_ed_in2out_fast_path_fn,
1560   .name = "nat44-ed-in2out",
1561   .vector_size = sizeof (u32),
1562   .format_trace = format_nat_in2out_ed_trace,
1563   .type = VLIB_NODE_TYPE_INTERNAL,
1564   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1565   .error_strings = nat_in2out_ed_error_strings,
1566   .runtime_data_bytes = sizeof (snat_runtime_t),
1567   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1568   .next_nodes = {
1569     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1570     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1571     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1572     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1573     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1574   },
1575 };
1576 /* *INDENT-ON* */
1577
1578 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1579                               nat44_ed_in2out_fast_path_fn);
1580
1581 static uword
1582 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1583                                      vlib_node_runtime_t * node,
1584                                      vlib_frame_t * frame)
1585 {
1586   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1587 }
1588
1589 /* *INDENT-OFF* */
1590 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1591   .function = nat44_ed_in2out_output_fast_path_fn,
1592   .name = "nat44-ed-in2out-output",
1593   .vector_size = sizeof (u32),
1594   .format_trace = format_nat_in2out_ed_trace,
1595   .type = VLIB_NODE_TYPE_INTERNAL,
1596   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1597   .error_strings = nat_in2out_ed_error_strings,
1598   .runtime_data_bytes = sizeof (snat_runtime_t),
1599   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1600   .next_nodes = {
1601     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1602     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1603     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1604     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1605     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1606   },
1607 };
1608 /* *INDENT-ON* */
1609
1610 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1611                               nat44_ed_in2out_output_fast_path_fn);
1612
1613 static uword
1614 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1615                               vlib_node_runtime_t * node,
1616                               vlib_frame_t * frame)
1617 {
1618   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1619 }
1620
1621 /* *INDENT-OFF* */
1622 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1623   .function = nat44_ed_in2out_slow_path_fn,
1624   .name = "nat44-ed-in2out-slowpath",
1625   .vector_size = sizeof (u32),
1626   .format_trace = format_nat_in2out_ed_trace,
1627   .type = VLIB_NODE_TYPE_INTERNAL,
1628   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1629   .error_strings = nat_in2out_ed_error_strings,
1630   .runtime_data_bytes = sizeof (snat_runtime_t),
1631   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1632   .next_nodes = {
1633     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1634     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1635     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1636     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1637     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1638   },
1639 };
1640 /* *INDENT-ON* */
1641
1642 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1643                               nat44_ed_in2out_slow_path_fn);
1644
1645 static uword
1646 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1647                                      vlib_node_runtime_t * node,
1648                                      vlib_frame_t * frame)
1649 {
1650   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1651 }
1652
1653 /* *INDENT-OFF* */
1654 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1655   .function = nat44_ed_in2out_output_slow_path_fn,
1656   .name = "nat44-ed-in2out-output-slowpath",
1657   .vector_size = sizeof (u32),
1658   .format_trace = format_nat_in2out_ed_trace,
1659   .type = VLIB_NODE_TYPE_INTERNAL,
1660   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1661   .error_strings = nat_in2out_ed_error_strings,
1662   .runtime_data_bytes = sizeof (snat_runtime_t),
1663   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1664   .next_nodes = {
1665     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1666     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1667     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1668     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1669     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1670   },
1671 };
1672 /* *INDENT-ON* */
1673
1674 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1675                               nat44_ed_in2out_output_slow_path_fn);
1676
1677 static inline uword
1678 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1679                                       vlib_node_runtime_t * node,
1680                                       vlib_frame_t * frame,
1681                                       int is_output_feature)
1682 {
1683   u32 n_left_from, *from, *to_next;
1684   nat_in2out_ed_next_t next_index;
1685   u32 pkts_processed = 0;
1686   snat_main_t *sm = &snat_main;
1687   f64 now = vlib_time_now (vm);
1688   u32 thread_index = vm->thread_index;
1689   snat_main_per_thread_data_t *per_thread_data =
1690     &sm->per_thread_data[thread_index];
1691   u32 *fragments_to_drop = 0;
1692   u32 *fragments_to_loopback = 0;
1693
1694   from = vlib_frame_vector_args (frame);
1695   n_left_from = frame->n_vectors;
1696   next_index = node->cached_next_index;
1697
1698   while (n_left_from > 0)
1699     {
1700       u32 n_left_to_next;
1701
1702       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1703
1704       while (n_left_from > 0 && n_left_to_next > 0)
1705         {
1706           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1707           u32 iph_offset0 = 0;
1708           vlib_buffer_t *b0;
1709           u32 next0;
1710           u8 cached0 = 0;
1711           ip4_header_t *ip0 = 0;
1712           nat_reass_ip4_t *reass0;
1713           udp_header_t *udp0;
1714           tcp_header_t *tcp0;
1715           icmp46_header_t *icmp0;
1716           clib_bihash_kv_16_8_t kv0, value0;
1717           snat_session_t *s0 = 0;
1718           u16 old_port0, new_port0;
1719           ip_csum_t sum0;
1720
1721           /* speculatively enqueue b0 to the current next frame */
1722           bi0 = from[0];
1723           to_next[0] = bi0;
1724           from += 1;
1725           to_next += 1;
1726           n_left_from -= 1;
1727           n_left_to_next -= 1;
1728
1729           b0 = vlib_get_buffer (vm, bi0);
1730
1731           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1732
1733           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1734           rx_fib_index0 =
1735             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1736                                                  sw_if_index0);
1737
1738           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1739             {
1740               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1741               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1742               goto trace0;
1743             }
1744
1745           if (is_output_feature)
1746             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1747
1748           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1749                                   iph_offset0);
1750
1751           udp0 = ip4_next_header (ip0);
1752           tcp0 = (tcp_header_t *) udp0;
1753           icmp0 = (icmp46_header_t *) udp0;
1754           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1755
1756           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1757                                                  ip0->dst_address,
1758                                                  ip0->fragment_id,
1759                                                  ip0->protocol,
1760                                                  1, &fragments_to_drop);
1761
1762           if (PREDICT_FALSE (!reass0))
1763             {
1764               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1765               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1766               nat_log_notice ("maximum reassemblies exceeded");
1767               goto trace0;
1768             }
1769
1770           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1771             {
1772               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1773                 {
1774                   if (is_output_feature)
1775                     {
1776                       if (PREDICT_FALSE
1777                           (nat_not_translate_output_feature_fwd
1778                            (sm, ip0, thread_index, now, vm, b0)))
1779                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1780                       goto trace0;
1781                     }
1782
1783                   next0 = icmp_in2out_ed_slow_path
1784                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1785                      next0, now, thread_index, &s0);
1786
1787                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1788                     {
1789                       if (s0)
1790                         reass0->sess_index = s0 - per_thread_data->sessions;
1791                       else
1792                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1793                       nat_ip4_reass_get_frags (reass0,
1794                                                &fragments_to_loopback);
1795                     }
1796
1797                   goto trace0;
1798                 }
1799
1800               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1801                           ip0->protocol, rx_fib_index0, udp0->src_port,
1802                           udp0->dst_port);
1803
1804               if (clib_bihash_search_16_8
1805                   (&per_thread_data->in2out_ed, &kv0, &value0))
1806                 {
1807                   if (is_output_feature)
1808                     {
1809                       if (PREDICT_FALSE
1810                           (nat44_ed_not_translate_output_feature
1811                            (sm, ip0, ip0->protocol, udp0->src_port,
1812                             udp0->dst_port, thread_index, sw_if_index0,
1813                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1814                         {
1815                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1816                           nat_ip4_reass_get_frags (reass0,
1817                                                    &fragments_to_loopback);
1818                           goto trace0;
1819                         }
1820                     }
1821                   else
1822                     {
1823                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1824                                                                  sw_if_index0,
1825                                                                  ip0, proto0,
1826                                                                  rx_fib_index0,
1827                                                                  thread_index)))
1828                         {
1829                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1830                           nat_ip4_reass_get_frags (reass0,
1831                                                    &fragments_to_loopback);
1832                           goto trace0;
1833                         }
1834                     }
1835
1836                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1837                                         &s0, node, next0, thread_index, now);
1838
1839                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1840                     goto trace0;
1841
1842                   reass0->sess_index = s0 - per_thread_data->sessions;
1843                 }
1844               else
1845                 {
1846                   s0 = pool_elt_at_index (per_thread_data->sessions,
1847                                           value0.value);
1848                   reass0->sess_index = value0.value;
1849                 }
1850               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1851             }
1852           else
1853             {
1854               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1855                 goto trace0;
1856               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1857                 {
1858                   if (nat_ip4_reass_add_fragment
1859                       (reass0, bi0, &fragments_to_drop))
1860                     {
1861                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1862                       nat_log_notice
1863                         ("maximum fragments per reassembly exceeded");
1864                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1865                       goto trace0;
1866                     }
1867                   cached0 = 1;
1868                   goto trace0;
1869                 }
1870               s0 = pool_elt_at_index (per_thread_data->sessions,
1871                                       reass0->sess_index);
1872             }
1873
1874           old_addr0 = ip0->src_address.as_u32;
1875           ip0->src_address = s0->out2in.addr;
1876           new_addr0 = ip0->src_address.as_u32;
1877           if (!is_output_feature)
1878             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1879
1880           sum0 = ip0->checksum;
1881           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1882                                  ip4_header_t,
1883                                  src_address /* changed member */ );
1884           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1885             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1886                                    s0->ext_host_addr.as_u32, ip4_header_t,
1887                                    dst_address);
1888           ip0->checksum = ip_csum_fold (sum0);
1889
1890           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1891             {
1892               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1893                 {
1894                   old_port0 = tcp0->src_port;
1895                   tcp0->src_port = s0->out2in.port;
1896                   new_port0 = tcp0->src_port;
1897
1898                   sum0 = tcp0->checksum;
1899                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1900                                          ip4_header_t,
1901                                          dst_address /* changed member */ );
1902                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1903                                          ip4_header_t /* cheat */ ,
1904                                          length /* changed member */ );
1905                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1906                     {
1907                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1908                                              s0->ext_host_addr.as_u32,
1909                                              ip4_header_t, dst_address);
1910                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1911                                              s0->ext_host_port, ip4_header_t,
1912                                              length);
1913                       tcp0->dst_port = s0->ext_host_port;
1914                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1915                     }
1916                   tcp0->checksum = ip_csum_fold (sum0);
1917                 }
1918               else
1919                 {
1920                   old_port0 = udp0->src_port;
1921                   udp0->src_port = s0->out2in.port;
1922                   udp0->checksum = 0;
1923                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1924                     {
1925                       udp0->dst_port = s0->ext_host_port;
1926                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1927                     }
1928                 }
1929             }
1930
1931           /* Hairpinning */
1932           if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_ICMP))
1933             nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1934                                      s0->ext_host_port, proto0, 1);
1935           else
1936             snat_icmp_hairpinning (sm, b0, ip0, icmp0, 1);
1937
1938           /* Accounting */
1939           nat44_session_update_counters (s0, now,
1940                                          vlib_buffer_length_in_chain (vm,
1941                                                                       b0));
1942           /* Per-user LRU list maintenance */
1943           nat44_session_update_lru (sm, s0, thread_index);
1944
1945         trace0:
1946           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1947                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1948             {
1949               nat44_reass_trace_t *t =
1950                 vlib_add_trace (vm, node, b0, sizeof (*t));
1951               t->cached = cached0;
1952               t->sw_if_index = sw_if_index0;
1953               t->next_index = next0;
1954             }
1955
1956           if (cached0)
1957             {
1958               n_left_to_next++;
1959               to_next--;
1960             }
1961           else
1962             {
1963               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1964
1965               /* verify speculative enqueue, maybe switch current next frame */
1966               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1967                                                to_next, n_left_to_next,
1968                                                bi0, next0);
1969             }
1970
1971           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1972             {
1973               from = vlib_frame_vector_args (frame);
1974               u32 len = vec_len (fragments_to_loopback);
1975               if (len <= VLIB_FRAME_SIZE)
1976                 {
1977                   clib_memcpy (from, fragments_to_loopback,
1978                                sizeof (u32) * len);
1979                   n_left_from = len;
1980                   vec_reset_length (fragments_to_loopback);
1981                 }
1982               else
1983                 {
1984                   clib_memcpy (from,
1985                                fragments_to_loopback + (len -
1986                                                         VLIB_FRAME_SIZE),
1987                                sizeof (u32) * VLIB_FRAME_SIZE);
1988                   n_left_from = VLIB_FRAME_SIZE;
1989                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1990                 }
1991             }
1992         }
1993
1994       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1995     }
1996
1997   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
1998                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1999                                pkts_processed);
2000
2001   nat_send_all_to_node (vm, fragments_to_drop, node,
2002                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
2003                         NAT_IN2OUT_ED_NEXT_DROP);
2004
2005   vec_free (fragments_to_drop);
2006   vec_free (fragments_to_loopback);
2007   return frame->n_vectors;
2008 }
2009
2010 static uword
2011 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
2012                                vlib_node_runtime_t * node,
2013                                vlib_frame_t * frame)
2014 {
2015   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2016 }
2017
2018 /* *INDENT-OFF* */
2019 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2020   .function = nat44_ed_in2out_reass_node_fn,
2021   .name = "nat44-ed-in2out-reass",
2022   .vector_size = sizeof (u32),
2023   .format_trace = format_nat44_reass_trace,
2024   .type = VLIB_NODE_TYPE_INTERNAL,
2025   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2026   .error_strings = nat_in2out_ed_error_strings,
2027   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2028   .next_nodes = {
2029     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2030     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2031     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2032     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2033     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2034   },
2035 };
2036 /* *INDENT-ON* */
2037
2038 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2039                               nat44_ed_in2out_reass_node_fn);
2040
2041 static uword
2042 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2043                                       vlib_node_runtime_t * node,
2044                                       vlib_frame_t * frame)
2045 {
2046   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2047 }
2048
2049 /* *INDENT-OFF* */
2050 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2051   .function = nat44_ed_in2out_reass_output_node_fn,
2052   .name = "nat44-ed-in2out-reass-output",
2053   .vector_size = sizeof (u32),
2054   .format_trace = format_nat44_reass_trace,
2055   .type = VLIB_NODE_TYPE_INTERNAL,
2056   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2057   .error_strings = nat_in2out_ed_error_strings,
2058   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2059   .next_nodes = {
2060     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2061     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2062     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2063     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2064     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2065   },
2066 };
2067 /* *INDENT-ON* */
2068
2069 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2070                               nat44_ed_in2out_reass_output_node_fn);
2071
2072 /*
2073  * fd.io coding-style-patch-verification: ON
2074  *
2075  * Local Variables:
2076  * eval: (c-set-style "gnu")
2077  * End:
2078  */