src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #define foreach_nat_in2out_ed_error                       \
  33 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  34 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  35 _(OUT_OF_PORTS, "Out of ports")                         \
  36 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  37 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  38 _(DROP_FRAGMENT, "Drop fragment")                       \
  39 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  40 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  41
  42 typedef enum
  43 {
  44 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
  45   foreach_nat_in2out_ed_error
  46 #undef _
  47     NAT_IN2OUT_ED_N_ERROR,
  48 } nat_in2out_ed_error_t;
  49
  50 static char *nat_in2out_ed_error_strings[] = {
  51 #define _(sym,string) string,
  52   foreach_nat_in2out_ed_error
  53 #undef _
  54 };
  55
  56 typedef enum
  57 {
  58   NAT_IN2OUT_ED_NEXT_LOOKUP,
  59   NAT_IN2OUT_ED_NEXT_DROP,
  60   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
  61   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
  62   NAT_IN2OUT_ED_NEXT_REASS,
  63   NAT_IN2OUT_ED_N_NEXT,
  64 } nat_in2out_ed_next_t;
  65
  66 typedef struct
  67 {
  68   u32 sw_if_index;
  69   u32 next_index;
  70   u32 session_index;
  71   u32 is_slow_path;
  72 } nat_in2out_ed_trace_t;
  73
  74 vlib_node_registration_t nat44_ed_in2out_node;
  75 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
  76 vlib_node_registration_t nat44_ed_in2out_output_node;
  77 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
  78 vlib_node_registration_t nat44_ed_in2out_reass_node;
  79
  80 static u8 *
  81 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  82 {
  83   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  84   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  85   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  86   char *tag;
  87
  88   tag =
  89     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  90     "NAT44_IN2OUT_ED_FAST_PATH";
  91
  92   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  93               t->sw_if_index, t->next_index, t->session_index);
  94
  95   return s;
  96 }
  97
  98 static_always_inline int
  99 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 100 {
 101   icmp46_header_t *icmp0;
 102   nat_ed_ses_key_t key0;
 103   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 104   ip4_header_t *inner_ip0 = 0;
 105   void *l4_header = 0;
 106   icmp46_header_t *inner_icmp0;
 107
 108   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 109   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 110
 111   if (!icmp_is_error_message (icmp0))
 112     {
 113       key0.proto = IP_PROTOCOL_ICMP;
 114       key0.l_addr = ip0->src_address;
 115       key0.r_addr = ip0->dst_address;
 116       key0.l_port = echo0->identifier;
 117       key0.r_port = 0;
 118     }
 119   else
 120     {
 121       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 122       l4_header = ip4_next_header (inner_ip0);
 123       key0.proto = inner_ip0->protocol;
 124       key0.r_addr = inner_ip0->src_address;
 125       key0.l_addr = inner_ip0->dst_address;
 126       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 127         {
 128         case SNAT_PROTOCOL_ICMP:
 129           inner_icmp0 = (icmp46_header_t *) l4_header;
 130           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 131           key0.r_port = 0;
 132           key0.l_port = inner_echo0->identifier;
 133           break;
 134         case SNAT_PROTOCOL_UDP:
 135         case SNAT_PROTOCOL_TCP:
 136           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 137           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
 138           break;
 139         default:
 140           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
 141         }
 142     }
 143   *p_key0 = key0;
 144   return 0;
 145 }
 146
 147 int
 148 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 149 {
 150   snat_main_t *sm = &snat_main;
 151   nat44_is_idle_session_ctx_t *ctx = arg;
 152   snat_session_t *s;
 153   u64 sess_timeout_time;
 154   nat_ed_ses_key_t ed_key;
 155   clib_bihash_kv_16_8_t ed_kv;
 156   int i;
 157   snat_address_t *a;
 158   snat_session_key_t key;
 159   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 160                                                        ctx->thread_index);
 161
 162   s = pool_elt_at_index (tsm->sessions, kv->value);
 163   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 164   if (ctx->now >= sess_timeout_time)
 165     {
 166       if (is_fwd_bypass_session (s))
 167         goto delete;
 168
 169       ed_key.l_addr = s->out2in.addr;
 170       ed_key.r_addr = s->ext_host_addr;
 171       ed_key.fib_index = s->out2in.fib_index;
 172       if (snat_is_unk_proto_session (s))
 173         {
 174           ed_key.proto = s->in2out.port;
 175           ed_key.r_port = 0;
 176           ed_key.l_port = 0;
 177         }
 178       else
 179         {
 180           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 181           ed_key.l_port = s->out2in.port;
 182           ed_key.r_port = s->ext_host_port;
 183         }
 184       ed_kv.key[0] = ed_key.as_u64[0];
 185       ed_kv.key[1] = ed_key.as_u64[1];
 186       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 187         nat_log_warn ("out2in_ed key del failed");
 188
 189       if (snat_is_unk_proto_session (s))
 190         goto delete;
 191
 192       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 193                                            s->out2in.addr.as_u32,
 194                                            s->in2out.protocol,
 195                                            s->in2out.port,
 196                                            s->out2in.port,
 197                                            s->in2out.fib_index);
 198
 199       if (is_twice_nat_session (s))
 200         {
 201           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 202             {
 203               key.protocol = s->in2out.protocol;
 204               key.port = s->ext_host_nat_port;
 205               a = sm->twice_nat_addresses + i;
 206               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 207                 {
 208                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 209                                                       ctx->thread_index,
 210                                                       &key);
 211                   break;
 212                 }
 213             }
 214         }
 215
 216       if (snat_is_session_static (s))
 217         goto delete;
 218
 219       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 220                                           &s->out2in);
 221     delete:
 222       nat44_delete_session (sm, s, ctx->thread_index);
 223       return 1;
 224     }
 225
 226   return 0;
 227 }
 228
 229 static inline u32
 230 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 231                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 232                           u32 sw_if_index0, u32 rx_fib_index0,
 233                           vlib_node_runtime_t * node, u32 next0, f64 now,
 234                           u32 thread_index, snat_session_t ** p_s0)
 235 {
 236   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 237                        next0, thread_index, p_s0, 0);
 238   snat_session_t *s0 = *p_s0;
 239   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
 240     {
 241       /* Accounting */
 242       nat44_session_update_counters (s0, now,
 243                                      vlib_buffer_length_in_chain
 244                                      (sm->vlib_main, b0));
 245       /* Per-user LRU list maintenance */
 246       nat44_session_update_lru (sm, s0, thread_index);
 247     }
 248   return next0;
 249 }
 250
 251 static u32
 252 slow_path_ed (snat_main_t * sm,
 253               vlib_buffer_t * b,
 254               u32 rx_fib_index,
 255               clib_bihash_kv_16_8_t * kv,
 256               snat_session_t ** sessionp,
 257               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 258 {
 259   snat_session_t *s = 0;
 260   snat_user_t *u;
 261   snat_session_key_t key0, key1;
 262   lb_nat_type_t lb = 0, is_sm = 0;
 263   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 264   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 265   u32 proto = ip_proto_to_snat_proto (key->proto);
 266   nat_outside_fib_t *outside_fib;
 267   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 268   u8 identity_nat;
 269   fib_prefix_t pfx = {
 270     .fp_proto = FIB_PROTOCOL_IP4,
 271     .fp_len = 32,
 272     .fp_addr = {
 273                 .ip4.as_u32 = key->r_addr.as_u32,
 274                 },
 275   };
 276   nat44_is_idle_session_ctx_t ctx;
 277
 278   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 279     {
 280       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 281       nat_ipfix_logging_max_sessions (sm->max_translations);
 282       nat_log_notice ("maximum sessions exceeded");
 283       return NAT_IN2OUT_ED_NEXT_DROP;
 284     }
 285
 286   key0.addr = key->l_addr;
 287   key0.port = key->l_port;
 288   key1.protocol = key0.protocol = proto;
 289   key0.fib_index = rx_fib_index;
 290   key1.fib_index = sm->outside_fib_index;
 291   /* First try to match static mapping by local address and port */
 292   if (snat_static_mapping_match
 293       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 294     {
 295       /* Try to create dynamic translation */
 296       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 297                                                thread_index, &key1,
 298                                                sm->port_per_thread,
 299                                                tsm->snat_thread_index))
 300         {
 301           nat_log_notice ("addresses exhausted");
 302           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 303           return NAT_IN2OUT_ED_NEXT_DROP;
 304         }
 305     }
 306   else
 307     {
 308       if (PREDICT_FALSE (identity_nat))
 309         {
 310           *sessionp = s;
 311           return next;
 312         }
 313
 314       is_sm = 1;
 315     }
 316
 317   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 318   if (!u)
 319     {
 320       nat_log_warn ("create NAT user failed");
 321       if (!is_sm)
 322         snat_free_outside_address_and_port (sm->addresses,
 323                                             thread_index, &key1);
 324       return NAT_IN2OUT_ED_NEXT_DROP;
 325     }
 326
 327   s = nat_ed_session_alloc (sm, u, thread_index, now);
 328   if (!s)
 329     {
 330       nat44_delete_user_with_no_session (sm, u, thread_index);
 331       nat_log_warn ("create NAT session failed");
 332       if (!is_sm)
 333         snat_free_outside_address_and_port (sm->addresses,
 334                                             thread_index, &key1);
 335       return NAT_IN2OUT_ED_NEXT_DROP;
 336     }
 337
 338   user_session_increment (sm, u, is_sm);
 339   if (is_sm)
 340     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 341   if (lb)
 342     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 343   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 344   s->ext_host_addr = key->r_addr;
 345   s->ext_host_port = key->r_port;
 346   s->in2out = key0;
 347   s->out2in = key1;
 348   s->out2in.protocol = key0.protocol;
 349
 350   switch (vec_len (sm->outside_fibs))
 351     {
 352     case 0:
 353       s->out2in.fib_index = sm->outside_fib_index;
 354       break;
 355     case 1:
 356       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 357       break;
 358     default:
 359       /* *INDENT-OFF* */
 360       vec_foreach (outside_fib, sm->outside_fibs)
 361        {
 362           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 363           if (FIB_NODE_INDEX_INVALID != fei)
 364             {
 365               if (fib_entry_get_resolving_interface (fei) != ~0)
 366                 {
 367                   s->out2in.fib_index = outside_fib->fib_index;
 368                   break;
 369                 }
 370             }
 371         }
 372       /* *INDENT-ON* */
 373       break;
 374     }
 375
 376   /* Add to lookup tables */
 377   kv->value = s - tsm->sessions;
 378   ctx.now = now;
 379   ctx.thread_index = thread_index;
 380   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 381                                                nat44_i2o_ed_is_idle_session_cb,
 382                                                &ctx))
 383     nat_log_notice ("in2out-ed key add failed");
 384
 385   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 386               key1.port, key->r_port);
 387   kv->value = s - tsm->sessions;
 388   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 389                                                nat44_o2i_ed_is_idle_session_cb,
 390                                                &ctx))
 391     nat_log_notice ("out2in-ed key add failed");
 392
 393   *sessionp = s;
 394
 395   /* log NAT event */
 396   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 397                                        s->out2in.addr.as_u32,
 398                                        s->in2out.protocol,
 399                                        s->in2out.port,
 400                                        s->out2in.port, s->in2out.fib_index);
 401   return next;
 402 }
 403
 404 static_always_inline int
 405 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 406                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 407                         u32 rx_fib_index, u32 thread_index)
 408 {
 409   udp_header_t *udp = ip4_next_header (ip);
 410   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 411   clib_bihash_kv_16_8_t kv, value;
 412   snat_session_key_t key0, key1;
 413
 414   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 415               sm->outside_fib_index, udp->dst_port, udp->src_port);
 416
 417   /* NAT packet aimed at external address if */
 418   /* has active sessions */
 419   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 420     {
 421       key0.addr = ip->dst_address;
 422       key0.port = udp->dst_port;
 423       key0.protocol = proto;
 424       key0.fib_index = sm->outside_fib_index;
 425       /* or is static mappings */
 426       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 427         return 0;
 428     }
 429   else
 430     return 0;
 431
 432   if (sm->forwarding_enabled)
 433     return 1;
 434
 435   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 436                                   rx_fib_index);
 437 }
 438
 439 static_always_inline int
 440 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 441                                       u32 thread_index, f64 now,
 442                                       vlib_main_t * vm, vlib_buffer_t * b)
 443 {
 444   nat_ed_ses_key_t key;
 445   clib_bihash_kv_16_8_t kv, value;
 446   udp_header_t *udp;
 447   snat_session_t *s = 0;
 448   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 449
 450   if (!sm->forwarding_enabled)
 451     return 0;
 452
 453   if (ip->protocol == IP_PROTOCOL_ICMP)
 454     {
 455       key.as_u64[0] = key.as_u64[1] = 0;
 456       if (icmp_get_ed_key (ip, &key))
 457         return 0;
 458       key.fib_index = 0;
 459       kv.key[0] = key.as_u64[0];
 460       kv.key[1] = key.as_u64[1];
 461     }
 462   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 463     {
 464       udp = ip4_next_header (ip);
 465       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 466                   udp->src_port, udp->dst_port);
 467     }
 468   else
 469     {
 470       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 471                   0);
 472     }
 473
 474   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 475     {
 476       s = pool_elt_at_index (tsm->sessions, value.value);
 477       if (is_fwd_bypass_session (s))
 478         {
 479           if (ip->protocol == IP_PROTOCOL_TCP)
 480             {
 481               tcp_header_t *tcp = ip4_next_header (ip);
 482               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 483                 return 1;
 484             }
 485           /* Accounting */
 486           nat44_session_update_counters (s, now,
 487                                          vlib_buffer_length_in_chain (vm, b));
 488           /* Per-user LRU list maintenance */
 489           nat44_session_update_lru (sm, s, thread_index);
 490           return 1;
 491         }
 492       else
 493         return 0;
 494     }
 495
 496   return 0;
 497 }
 498
 499 static_always_inline int
 500 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 501                                        u8 proto, u16 src_port, u16 dst_port,
 502                                        u32 thread_index, u32 rx_sw_if_index,
 503                                        u32 tx_sw_if_index)
 504 {
 505   clib_bihash_kv_16_8_t kv, value;
 506   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 507   snat_interface_t *i;
 508   snat_session_t *s;
 509   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 510   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 511
 512   /* src NAT check */
 513   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 514               src_port, dst_port);
 515   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 516     return 1;
 517
 518   /* dst NAT check */
 519   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 520               dst_port, src_port);
 521   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 522     {
 523       s = pool_elt_at_index (tsm->sessions, value.value);
 524       if (is_fwd_bypass_session (s))
 525         return 0;
 526
 527       /* hairpinning */
 528       /* *INDENT-OFF* */
 529       pool_foreach (i, sm->output_feature_interfaces,
 530       ({
 531         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 532            return 0;
 533       }));
 534       /* *INDENT-ON* */
 535       return 1;
 536     }
 537
 538   return 0;
 539 }
 540
 541 u32
 542 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 543                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 544                       u8 * p_proto, snat_session_key_t * p_value,
 545                       u8 * p_dont_translate, void *d, void *e)
 546 {
 547   icmp46_header_t *icmp;
 548   u32 sw_if_index;
 549   u32 rx_fib_index;
 550   nat_ed_ses_key_t key;
 551   snat_session_t *s = 0;
 552   u8 dont_translate = 0;
 553   clib_bihash_kv_16_8_t kv, value;
 554   u32 next = ~0;
 555   int err;
 556   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 557
 558   icmp = (icmp46_header_t *) ip4_next_header (ip);
 559   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 560   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 561
 562   key.as_u64[0] = key.as_u64[1] = 0;
 563   err = icmp_get_ed_key (ip, &key);
 564   if (err != 0)
 565     {
 566       b->error = node->errors[err];
 567       next = NAT_IN2OUT_ED_NEXT_DROP;
 568       goto out;
 569     }
 570   key.fib_index = rx_fib_index;
 571
 572   kv.key[0] = key.as_u64[0];
 573   kv.key[1] = key.as_u64[1];
 574
 575   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 576     {
 577       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 578         {
 579           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 580                                                                     key.proto,
 581                                                                     key.
 582                                                                     l_port,
 583                                                                     key.
 584                                                                     r_port,
 585                                                                     thread_index,
 586                                                                     sw_if_index,
 587                                                                     vnet_buffer
 588                                                                     (b)->
 589                                                                     sw_if_index
 590                                                                     [VLIB_TX])))
 591             {
 592               dont_translate = 1;
 593               goto out;
 594             }
 595         }
 596       else
 597         {
 598           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 599                                                      ip, SNAT_PROTOCOL_ICMP,
 600                                                      rx_fib_index,
 601                                                      thread_index)))
 602             {
 603               dont_translate = 1;
 604               goto out;
 605             }
 606         }
 607
 608       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 609         {
 610           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 611           next = NAT_IN2OUT_ED_NEXT_DROP;
 612           goto out;
 613         }
 614
 615       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 616                            thread_index, vlib_time_now (sm->vlib_main));
 617
 618       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
 619         goto out;
 620
 621       if (!s)
 622         {
 623           dont_translate = 1;
 624           goto out;
 625         }
 626     }
 627   else
 628     {
 629       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 630                          icmp->type != ICMP4_echo_reply &&
 631                          !icmp_is_error_message (icmp)))
 632         {
 633           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 634           next = NAT_IN2OUT_ED_NEXT_DROP;
 635           goto out;
 636         }
 637
 638       s = pool_elt_at_index (tsm->sessions, value.value);
 639     }
 640
 641   *p_proto = ip_proto_to_snat_proto (key.proto);
 642 out:
 643   if (s)
 644     *p_value = s->out2in;
 645   *p_dont_translate = dont_translate;
 646   if (d)
 647     *(snat_session_t **) d = s;
 648   return next;
 649 }
 650
 651 static snat_session_t *
 652 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 653                                vlib_buffer_t * b,
 654                                ip4_header_t * ip,
 655                                u32 rx_fib_index,
 656                                u32 thread_index,
 657                                f64 now,
 658                                vlib_main_t * vm, vlib_node_runtime_t * node)
 659 {
 660   clib_bihash_kv_8_8_t kv, value;
 661   clib_bihash_kv_16_8_t s_kv, s_value;
 662   snat_static_mapping_t *m;
 663   u32 old_addr, new_addr = 0;
 664   ip_csum_t sum;
 665   snat_user_t *u;
 666   dlist_elt_t *head, *elt;
 667   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 668   u32 elt_index, head_index, ses_index;
 669   snat_session_t *s;
 670   u32 outside_fib_index = sm->outside_fib_index;
 671   int i;
 672   u8 is_sm = 0;
 673   nat_outside_fib_t *outside_fib;
 674   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 675   fib_prefix_t pfx = {
 676     .fp_proto = FIB_PROTOCOL_IP4,
 677     .fp_len = 32,
 678     .fp_addr = {
 679                 .ip4.as_u32 = ip->dst_address.as_u32,
 680                 },
 681   };
 682
 683   switch (vec_len (sm->outside_fibs))
 684     {
 685     case 0:
 686       outside_fib_index = sm->outside_fib_index;
 687       break;
 688     case 1:
 689       outside_fib_index = sm->outside_fibs[0].fib_index;
 690       break;
 691     default:
 692       /* *INDENT-OFF* */
 693       vec_foreach (outside_fib, sm->outside_fibs)
 694         {
 695           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 696           if (FIB_NODE_INDEX_INVALID != fei)
 697             {
 698               if (fib_entry_get_resolving_interface (fei) != ~0)
 699                 {
 700                   outside_fib_index = outside_fib->fib_index;
 701                   break;
 702                 }
 703             }
 704         }
 705       /* *INDENT-ON* */
 706       break;
 707     }
 708   old_addr = ip->src_address.as_u32;
 709
 710   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 711               rx_fib_index, 0, 0);
 712
 713   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 714     {
 715       s = pool_elt_at_index (tsm->sessions, s_value.value);
 716       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 717     }
 718   else
 719     {
 720       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 721         {
 722           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 723           nat_ipfix_logging_max_sessions (sm->max_translations);
 724           nat_log_notice ("maximum sessions exceeded");
 725           return 0;
 726         }
 727
 728       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 729                                   thread_index);
 730       if (!u)
 731         {
 732           nat_log_warn ("create NAT user failed");
 733           return 0;
 734         }
 735
 736       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 737
 738       /* Try to find static mapping first */
 739       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 740         {
 741           m = pool_elt_at_index (sm->static_mappings, value.value);
 742           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 743           is_sm = 1;
 744           goto create_ses;
 745         }
 746       /* Fallback to 3-tuple key */
 747       else
 748         {
 749           /* Choose same out address as for TCP/UDP session to same destination */
 750           head_index = u->sessions_per_user_list_head_index;
 751           head = pool_elt_at_index (tsm->list_pool, head_index);
 752           elt_index = head->next;
 753           if (PREDICT_FALSE (elt_index == ~0))
 754             ses_index = ~0;
 755           else
 756             {
 757               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 758               ses_index = elt->value;
 759             }
 760
 761           while (ses_index != ~0)
 762             {
 763               s = pool_elt_at_index (tsm->sessions, ses_index);
 764               elt_index = elt->next;
 765               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 766               ses_index = elt->value;
 767
 768               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 769                 {
 770                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 771
 772                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 773                               ip->protocol, outside_fib_index, 0, 0);
 774                   if (clib_bihash_search_16_8
 775                       (&tsm->out2in_ed, &s_kv, &s_value))
 776                     goto create_ses;
 777
 778                   break;
 779                 }
 780             }
 781
 782           for (i = 0; i < vec_len (sm->addresses); i++)
 783             {
 784               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 785                           ip->protocol, outside_fib_index, 0, 0);
 786               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 787                 {
 788                   new_addr = ip->src_address.as_u32 =
 789                     sm->addresses[i].addr.as_u32;
 790                   goto create_ses;
 791                 }
 792             }
 793           return 0;
 794         }
 795
 796     create_ses:
 797       s = nat_ed_session_alloc (sm, u, thread_index, now);
 798       if (!s)
 799         {
 800           nat44_delete_user_with_no_session (sm, u, thread_index);
 801           nat_log_warn ("create NAT session failed");
 802           return 0;
 803         }
 804
 805       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 806       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 807       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 808       s->out2in.addr.as_u32 = new_addr;
 809       s->out2in.fib_index = outside_fib_index;
 810       s->in2out.addr.as_u32 = old_addr;
 811       s->in2out.fib_index = rx_fib_index;
 812       s->in2out.port = s->out2in.port = ip->protocol;
 813       if (is_sm)
 814         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 815       user_session_increment (sm, u, is_sm);
 816
 817       /* Add to lookup tables */
 818       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 819                   rx_fib_index, 0, 0);
 820       s_kv.value = s - tsm->sessions;
 821       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 822         nat_log_notice ("in2out key add failed");
 823
 824       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 825                   outside_fib_index, 0, 0);
 826       s_kv.value = s - tsm->sessions;
 827       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 828         nat_log_notice ("out2in key add failed");
 829     }
 830
 831   /* Update IP checksum */
 832   sum = ip->checksum;
 833   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 834   ip->checksum = ip_csum_fold (sum);
 835
 836   /* Accounting */
 837   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 838   /* Per-user LRU list maintenance */
 839   nat44_session_update_lru (sm, s, thread_index);
 840
 841   /* Hairpinning */
 842   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 843     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 844
 845   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 846     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 847
 848   return s;
 849 }
 850
 851 static inline uword
 852 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 853                                 vlib_node_runtime_t * node,
 854                                 vlib_frame_t * frame, int is_slow_path,
 855                                 int is_output_feature)
 856 {
 857   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 858   nat_in2out_ed_next_t next_index;
 859   snat_main_t *sm = &snat_main;
 860   f64 now = vlib_time_now (vm);
 861   u32 thread_index = vm->thread_index;
 862   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 863
 864   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
 865     nat44_ed_in2out_node.index;
 866
 867   from = vlib_frame_vector_args (frame);
 868   n_left_from = frame->n_vectors;
 869   next_index = node->cached_next_index;
 870
 871   while (n_left_from > 0)
 872     {
 873       u32 n_left_to_next;
 874
 875       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 876
 877       while (n_left_from >= 4 && n_left_to_next >= 2)
 878         {
 879           u32 bi0, bi1;
 880           vlib_buffer_t *b0, *b1;
 881           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 882             new_addr0, old_addr0;
 883           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 884             new_addr1, old_addr1;
 885           u16 old_port0, new_port0, old_port1, new_port1;
 886           ip4_header_t *ip0, *ip1;
 887           udp_header_t *udp0, *udp1;
 888           tcp_header_t *tcp0, *tcp1;
 889           icmp46_header_t *icmp0, *icmp1;
 890           snat_session_t *s0 = 0, *s1 = 0;
 891           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 892           ip_csum_t sum0, sum1;
 893
 894           /* Prefetch next iteration. */
 895           {
 896             vlib_buffer_t *p2, *p3;
 897
 898             p2 = vlib_get_buffer (vm, from[2]);
 899             p3 = vlib_get_buffer (vm, from[3]);
 900
 901             vlib_prefetch_buffer_header (p2, LOAD);
 902             vlib_prefetch_buffer_header (p3, LOAD);
 903
 904             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 905             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 906           }
 907
 908           /* speculatively enqueue b0 and b1 to the current next frame */
 909           to_next[0] = bi0 = from[0];
 910           to_next[1] = bi1 = from[1];
 911           from += 2;
 912           to_next += 2;
 913           n_left_from -= 2;
 914           n_left_to_next -= 2;
 915
 916           b0 = vlib_get_buffer (vm, bi0);
 917           b1 = vlib_get_buffer (vm, bi1);
 918
 919           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
 920
 921           if (is_output_feature)
 922             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 923
 924           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 925                                   iph_offset0);
 926
 927           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 928           rx_fib_index0 =
 929             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 930                                                  sw_if_index0);
 931
 932           if (PREDICT_FALSE (ip0->ttl == 1))
 933             {
 934               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 935               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 936                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 937                                            0);
 938               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
 939               goto trace00;
 940             }
 941
 942           udp0 = ip4_next_header (ip0);
 943           tcp0 = (tcp_header_t *) udp0;
 944           icmp0 = (icmp46_header_t *) udp0;
 945           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 946
 947           if (is_slow_path)
 948             {
 949               if (PREDICT_FALSE (proto0 == ~0))
 950                 {
 951                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
 952                                                       rx_fib_index0,
 953                                                       thread_index, now, vm,
 954                                                       node);
 955                   if (!s0)
 956                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
 957                   goto trace00;
 958                 }
 959
 960               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 961                 {
 962                   next0 = icmp_in2out_ed_slow_path
 963                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 964                      next0, now, thread_index, &s0);
 965                   goto trace00;
 966                 }
 967             }
 968           else
 969             {
 970               if (PREDICT_FALSE (proto0 == ~0))
 971                 {
 972                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 973                   goto trace00;
 974                 }
 975
 976               if (ip4_is_fragment (ip0))
 977                 {
 978                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
 979                   goto trace00;
 980                 }
 981
 982               if (is_output_feature)
 983                 {
 984                   if (PREDICT_FALSE
 985                       (nat_not_translate_output_feature_fwd
 986                        (sm, ip0, thread_index, now, vm, b0)))
 987                     goto trace00;
 988                 }
 989
 990               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 991                 {
 992                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 993                   goto trace00;
 994                 }
 995             }
 996
 997           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
 998                       ip0->protocol, rx_fib_index0, udp0->src_port,
 999                       udp0->dst_port);
1000
1001           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1002             {
1003               if (is_slow_path)
1004                 {
1005                   if (is_output_feature)
1006                     {
1007                       if (PREDICT_FALSE
1008                           (nat44_ed_not_translate_output_feature
1009                            (sm, ip0, ip0->protocol, udp0->src_port,
1010                             udp0->dst_port, thread_index, sw_if_index0,
1011                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1012                         goto trace00;
1013                     }
1014                   else
1015                     {
1016                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1017                                                                  sw_if_index0,
1018                                                                  ip0, proto0,
1019                                                                  rx_fib_index0,
1020                                                                  thread_index)))
1021                         goto trace00;
1022                     }
1023
1024                   next0 =
1025                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1026                                   next0, thread_index, now);
1027
1028                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1029                     goto trace00;
1030
1031                   if (PREDICT_FALSE (!s0))
1032                     goto trace00;
1033                 }
1034               else
1035                 {
1036                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1037                   goto trace00;
1038                 }
1039             }
1040           else
1041             {
1042               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1043             }
1044
1045           b0->flags |= VNET_BUFFER_F_IS_NATED;
1046
1047           if (!is_output_feature)
1048             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1049
1050           old_addr0 = ip0->src_address.as_u32;
1051           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1052           sum0 = ip0->checksum;
1053           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1054                                  src_address);
1055           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1056             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1057                                    s0->ext_host_addr.as_u32, ip4_header_t,
1058                                    dst_address);
1059           ip0->checksum = ip_csum_fold (sum0);
1060
1061           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1062             {
1063               old_port0 = tcp0->src_port;
1064               new_port0 = tcp0->src_port = s0->out2in.port;
1065
1066               sum0 = tcp0->checksum;
1067               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1068                                      dst_address);
1069               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1070                                      length);
1071               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1072                 {
1073                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1074                                          s0->ext_host_addr.as_u32,
1075                                          ip4_header_t, dst_address);
1076                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1077                                          s0->ext_host_port, ip4_header_t,
1078                                          length);
1079                   tcp0->dst_port = s0->ext_host_port;
1080                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1081                 }
1082               mss_clamping (sm, tcp0, &sum0);
1083               tcp0->checksum = ip_csum_fold (sum0);
1084               if (nat44_set_tcp_session_state_i2o
1085                   (sm, s0, tcp0, thread_index))
1086                 goto trace00;
1087             }
1088           else
1089             {
1090               udp0->src_port = s0->out2in.port;
1091               udp0->checksum = 0;
1092               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1093                 {
1094                   udp0->dst_port = s0->ext_host_port;
1095                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1096                 }
1097             }
1098
1099           /* Accounting */
1100           nat44_session_update_counters (s0, now,
1101                                          vlib_buffer_length_in_chain (vm,
1102                                                                       b0));
1103           /* Per-user LRU list maintenance */
1104           nat44_session_update_lru (sm, s0, thread_index);
1105
1106         trace00:
1107           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1108                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1109             {
1110               nat_in2out_ed_trace_t *t =
1111                 vlib_add_trace (vm, node, b0, sizeof (*t));
1112               t->is_slow_path = is_slow_path;
1113               t->sw_if_index = sw_if_index0;
1114               t->next_index = next0;
1115               t->session_index = ~0;
1116               if (s0)
1117                 t->session_index = s0 - tsm->sessions;
1118             }
1119
1120           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1121
1122
1123           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1124
1125           if (is_output_feature)
1126             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1127
1128           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1129                                   iph_offset1);
1130
1131           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1132           rx_fib_index1 =
1133             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1134                                                  sw_if_index1);
1135
1136           if (PREDICT_FALSE (ip1->ttl == 1))
1137             {
1138               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1139               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1140                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1141                                            0);
1142               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1143               goto trace01;
1144             }
1145
1146           udp1 = ip4_next_header (ip1);
1147           tcp1 = (tcp_header_t *) udp1;
1148           icmp1 = (icmp46_header_t *) udp1;
1149           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1150
1151           if (is_slow_path)
1152             {
1153               if (PREDICT_FALSE (proto1 == ~0))
1154                 {
1155                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1156                                                       rx_fib_index1,
1157                                                       thread_index, now, vm,
1158                                                       node);
1159                   if (!s1)
1160                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1161                   goto trace01;
1162                 }
1163
1164               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1165                 {
1166                   next1 = icmp_in2out_ed_slow_path
1167                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1168                      next1, now, thread_index, &s1);
1169                   goto trace01;
1170                 }
1171             }
1172           else
1173             {
1174               if (PREDICT_FALSE (proto1 == ~0))
1175                 {
1176                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1177                   goto trace01;
1178                 }
1179
1180               if (ip4_is_fragment (ip1))
1181                 {
1182                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1183                   goto trace01;
1184                 }
1185
1186               if (is_output_feature)
1187                 {
1188                   if (PREDICT_FALSE
1189                       (nat_not_translate_output_feature_fwd
1190                        (sm, ip1, thread_index, now, vm, b1)))
1191                     goto trace01;
1192                 }
1193
1194               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1195                 {
1196                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1197                   goto trace01;
1198                 }
1199             }
1200
1201           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1202                       ip1->protocol, rx_fib_index1, udp1->src_port,
1203                       udp1->dst_port);
1204
1205           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1206             {
1207               if (is_slow_path)
1208                 {
1209                   if (is_output_feature)
1210                     {
1211                       if (PREDICT_FALSE
1212                           (nat44_ed_not_translate_output_feature
1213                            (sm, ip1, ip1->protocol, udp1->src_port,
1214                             udp1->dst_port, thread_index, sw_if_index1,
1215                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1216                         goto trace01;
1217                     }
1218                   else
1219                     {
1220                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1221                                                                  sw_if_index1,
1222                                                                  ip1, proto1,
1223                                                                  rx_fib_index1,
1224                                                                  thread_index)))
1225                         goto trace01;
1226                     }
1227
1228                   next1 =
1229                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1230                                   next1, thread_index, now);
1231
1232                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1233                     goto trace01;
1234
1235                   if (PREDICT_FALSE (!s1))
1236                     goto trace01;
1237                 }
1238               else
1239                 {
1240                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1241                   goto trace01;
1242                 }
1243             }
1244           else
1245             {
1246               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1247             }
1248
1249           b1->flags |= VNET_BUFFER_F_IS_NATED;
1250
1251           if (!is_output_feature)
1252             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1253
1254           old_addr1 = ip1->src_address.as_u32;
1255           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1256           sum1 = ip1->checksum;
1257           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1258                                  src_address);
1259           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1260             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1261                                    s1->ext_host_addr.as_u32, ip4_header_t,
1262                                    dst_address);
1263           ip1->checksum = ip_csum_fold (sum1);
1264
1265           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1266             {
1267               old_port1 = tcp1->src_port;
1268               new_port1 = tcp1->src_port = s1->out2in.port;
1269
1270               sum1 = tcp1->checksum;
1271               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1272                                      dst_address);
1273               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1274                                      length);
1275               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1276                 {
1277                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1278                                          s1->ext_host_addr.as_u32,
1279                                          ip4_header_t, dst_address);
1280                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1281                                          s1->ext_host_port, ip4_header_t,
1282                                          length);
1283                   tcp1->dst_port = s1->ext_host_port;
1284                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1285                 }
1286               tcp1->checksum = ip_csum_fold (sum1);
1287               mss_clamping (sm, tcp1, &sum1);
1288               if (nat44_set_tcp_session_state_i2o
1289                   (sm, s1, tcp1, thread_index))
1290                 goto trace01;
1291             }
1292           else
1293             {
1294               udp1->src_port = s1->out2in.port;
1295               udp1->checksum = 0;
1296               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1297                 {
1298                   udp1->dst_port = s1->ext_host_port;
1299                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1300                 }
1301             }
1302
1303           /* Accounting */
1304           nat44_session_update_counters (s1, now,
1305                                          vlib_buffer_length_in_chain (vm,
1306                                                                       b1));
1307           /* Per-user LRU list maintenance */
1308           nat44_session_update_lru (sm, s1, thread_index);
1309
1310         trace01:
1311           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1312                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1313             {
1314               nat_in2out_ed_trace_t *t =
1315                 vlib_add_trace (vm, node, b1, sizeof (*t));
1316               t->is_slow_path = is_slow_path;
1317               t->sw_if_index = sw_if_index1;
1318               t->next_index = next1;
1319               t->session_index = ~0;
1320               if (s1)
1321                 t->session_index = s1 - tsm->sessions;
1322             }
1323
1324           pkts_processed += next1 != NAT_IN2OUT_ED_NEXT_DROP;
1325
1326           /* verify speculative enqueues, maybe switch current next frame */
1327           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1328                                            to_next, n_left_to_next,
1329                                            bi0, bi1, next0, next1);
1330         }
1331
1332       while (n_left_from > 0 && n_left_to_next > 0)
1333         {
1334           u32 bi0;
1335           vlib_buffer_t *b0;
1336           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1337             new_addr0, old_addr0;
1338           u16 old_port0, new_port0;
1339           ip4_header_t *ip0;
1340           udp_header_t *udp0;
1341           tcp_header_t *tcp0;
1342           icmp46_header_t *icmp0;
1343           snat_session_t *s0 = 0;
1344           clib_bihash_kv_16_8_t kv0, value0;
1345           ip_csum_t sum0;
1346
1347           /* speculatively enqueue b0 to the current next frame */
1348           bi0 = from[0];
1349           to_next[0] = bi0;
1350           from += 1;
1351           to_next += 1;
1352           n_left_from -= 1;
1353           n_left_to_next -= 1;
1354
1355           b0 = vlib_get_buffer (vm, bi0);
1356           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1357
1358           if (is_output_feature)
1359             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1360
1361           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1362                                   iph_offset0);
1363
1364           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1365           rx_fib_index0 =
1366             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1367                                                  sw_if_index0);
1368
1369           if (PREDICT_FALSE (ip0->ttl == 1))
1370             {
1371               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1372               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1373                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1374                                            0);
1375               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1376               goto trace0;
1377             }
1378
1379           udp0 = ip4_next_header (ip0);
1380           tcp0 = (tcp_header_t *) udp0;
1381           icmp0 = (icmp46_header_t *) udp0;
1382           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1383
1384           if (is_slow_path)
1385             {
1386               if (PREDICT_FALSE (proto0 == ~0))
1387                 {
1388                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1389                                                       rx_fib_index0,
1390                                                       thread_index, now, vm,
1391                                                       node);
1392                   if (!s0)
1393                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1394                   goto trace0;
1395                 }
1396
1397               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1398                 {
1399                   next0 = icmp_in2out_ed_slow_path
1400                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1401                      next0, now, thread_index, &s0);
1402                   goto trace0;
1403                 }
1404             }
1405           else
1406             {
1407               if (PREDICT_FALSE (proto0 == ~0))
1408                 {
1409                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1410                   goto trace0;
1411                 }
1412
1413               if (ip4_is_fragment (ip0))
1414                 {
1415                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1416                   goto trace0;
1417                 }
1418
1419               if (is_output_feature)
1420                 {
1421                   if (PREDICT_FALSE
1422                       (nat_not_translate_output_feature_fwd
1423                        (sm, ip0, thread_index, now, vm, b0)))
1424                     goto trace0;
1425                 }
1426
1427               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1428                 {
1429                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1430                   goto trace0;
1431                 }
1432             }
1433
1434           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1435                       ip0->protocol, rx_fib_index0, udp0->src_port,
1436                       udp0->dst_port);
1437
1438           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1439             {
1440               if (is_slow_path)
1441                 {
1442                   if (is_output_feature)
1443                     {
1444                       if (PREDICT_FALSE
1445                           (nat44_ed_not_translate_output_feature
1446                            (sm, ip0, ip0->protocol, udp0->src_port,
1447                             udp0->dst_port, thread_index, sw_if_index0,
1448                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1449                         goto trace0;
1450                     }
1451                   else
1452                     {
1453                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1454                                                                  sw_if_index0,
1455                                                                  ip0, proto0,
1456                                                                  rx_fib_index0,
1457                                                                  thread_index)))
1458                         goto trace0;
1459                     }
1460
1461                   next0 =
1462                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1463                                   next0, thread_index, now);
1464
1465                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1466                     goto trace0;
1467
1468                   if (PREDICT_FALSE (!s0))
1469                     goto trace0;
1470                 }
1471               else
1472                 {
1473                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1474                   goto trace0;
1475                 }
1476             }
1477           else
1478             {
1479               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1480             }
1481
1482           b0->flags |= VNET_BUFFER_F_IS_NATED;
1483
1484           if (!is_output_feature)
1485             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1486
1487           old_addr0 = ip0->src_address.as_u32;
1488           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1489           sum0 = ip0->checksum;
1490           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1491                                  src_address);
1492           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1493             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1494                                    s0->ext_host_addr.as_u32, ip4_header_t,
1495                                    dst_address);
1496           ip0->checksum = ip_csum_fold (sum0);
1497
1498           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1499             {
1500               old_port0 = tcp0->src_port;
1501               new_port0 = tcp0->src_port = s0->out2in.port;
1502
1503               sum0 = tcp0->checksum;
1504               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1505                                      dst_address);
1506               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1507                                      length);
1508               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1509                 {
1510                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1511                                          s0->ext_host_addr.as_u32,
1512                                          ip4_header_t, dst_address);
1513                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1514                                          s0->ext_host_port, ip4_header_t,
1515                                          length);
1516                   tcp0->dst_port = s0->ext_host_port;
1517                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1518                 }
1519               mss_clamping (sm, tcp0, &sum0);
1520               tcp0->checksum = ip_csum_fold (sum0);
1521               if (nat44_set_tcp_session_state_i2o
1522                   (sm, s0, tcp0, thread_index))
1523                 goto trace0;
1524             }
1525           else
1526             {
1527               udp0->src_port = s0->out2in.port;
1528               udp0->checksum = 0;
1529               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1530                 {
1531                   udp0->dst_port = s0->ext_host_port;
1532                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1533                 }
1534             }
1535
1536           /* Accounting */
1537           nat44_session_update_counters (s0, now,
1538                                          vlib_buffer_length_in_chain (vm,
1539                                                                       b0));
1540           /* Per-user LRU list maintenance */
1541           nat44_session_update_lru (sm, s0, thread_index);
1542
1543         trace0:
1544           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1545                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1546             {
1547               nat_in2out_ed_trace_t *t =
1548                 vlib_add_trace (vm, node, b0, sizeof (*t));
1549               t->is_slow_path = is_slow_path;
1550               t->sw_if_index = sw_if_index0;
1551               t->next_index = next0;
1552               t->session_index = ~0;
1553               if (s0)
1554                 t->session_index = s0 - tsm->sessions;
1555             }
1556
1557           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1558
1559           /* verify speculative enqueue, maybe switch current next frame */
1560           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1561                                            to_next, n_left_to_next,
1562                                            bi0, next0);
1563         }
1564
1565       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1566     }
1567
1568   vlib_node_increment_counter (vm, stats_node_index,
1569                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1570                                pkts_processed);
1571   return frame->n_vectors;
1572 }
1573
1574 static uword
1575 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1576                               vlib_node_runtime_t * node,
1577                               vlib_frame_t * frame)
1578 {
1579   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1580 }
1581
1582 /* *INDENT-OFF* */
1583 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1584   .function = nat44_ed_in2out_fast_path_fn,
1585   .name = "nat44-ed-in2out",
1586   .vector_size = sizeof (u32),
1587   .format_trace = format_nat_in2out_ed_trace,
1588   .type = VLIB_NODE_TYPE_INTERNAL,
1589   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1590   .error_strings = nat_in2out_ed_error_strings,
1591   .runtime_data_bytes = sizeof (snat_runtime_t),
1592   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1593   .next_nodes = {
1594     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1595     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1596     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1597     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1598     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1599   },
1600 };
1601 /* *INDENT-ON* */
1602
1603 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1604                               nat44_ed_in2out_fast_path_fn);
1605
1606 static uword
1607 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1608                                      vlib_node_runtime_t * node,
1609                                      vlib_frame_t * frame)
1610 {
1611   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1612 }
1613
1614 /* *INDENT-OFF* */
1615 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1616   .function = nat44_ed_in2out_output_fast_path_fn,
1617   .name = "nat44-ed-in2out-output",
1618   .vector_size = sizeof (u32),
1619   .format_trace = format_nat_in2out_ed_trace,
1620   .type = VLIB_NODE_TYPE_INTERNAL,
1621   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1622   .error_strings = nat_in2out_ed_error_strings,
1623   .runtime_data_bytes = sizeof (snat_runtime_t),
1624   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1625   .next_nodes = {
1626     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1627     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1628     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1629     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1630     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1631   },
1632 };
1633 /* *INDENT-ON* */
1634
1635 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1636                               nat44_ed_in2out_output_fast_path_fn);
1637
1638 static uword
1639 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1640                               vlib_node_runtime_t * node,
1641                               vlib_frame_t * frame)
1642 {
1643   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1644 }
1645
1646 /* *INDENT-OFF* */
1647 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1648   .function = nat44_ed_in2out_slow_path_fn,
1649   .name = "nat44-ed-in2out-slowpath",
1650   .vector_size = sizeof (u32),
1651   .format_trace = format_nat_in2out_ed_trace,
1652   .type = VLIB_NODE_TYPE_INTERNAL,
1653   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1654   .error_strings = nat_in2out_ed_error_strings,
1655   .runtime_data_bytes = sizeof (snat_runtime_t),
1656   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1657   .next_nodes = {
1658     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1659     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1660     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1661     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1662     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1663   },
1664 };
1665 /* *INDENT-ON* */
1666
1667 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1668                               nat44_ed_in2out_slow_path_fn);
1669
1670 static uword
1671 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1672                                      vlib_node_runtime_t * node,
1673                                      vlib_frame_t * frame)
1674 {
1675   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1676 }
1677
1678 /* *INDENT-OFF* */
1679 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1680   .function = nat44_ed_in2out_output_slow_path_fn,
1681   .name = "nat44-ed-in2out-output-slowpath",
1682   .vector_size = sizeof (u32),
1683   .format_trace = format_nat_in2out_ed_trace,
1684   .type = VLIB_NODE_TYPE_INTERNAL,
1685   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1686   .error_strings = nat_in2out_ed_error_strings,
1687   .runtime_data_bytes = sizeof (snat_runtime_t),
1688   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1689   .next_nodes = {
1690     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1691     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1692     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1693     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1694     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1695   },
1696 };
1697 /* *INDENT-ON* */
1698
1699 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1700                               nat44_ed_in2out_output_slow_path_fn);
1701
1702 static inline uword
1703 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1704                                       vlib_node_runtime_t * node,
1705                                       vlib_frame_t * frame,
1706                                       int is_output_feature)
1707 {
1708   u32 n_left_from, *from, *to_next;
1709   nat_in2out_ed_next_t next_index;
1710   u32 pkts_processed = 0;
1711   snat_main_t *sm = &snat_main;
1712   f64 now = vlib_time_now (vm);
1713   u32 thread_index = vm->thread_index;
1714   snat_main_per_thread_data_t *per_thread_data =
1715     &sm->per_thread_data[thread_index];
1716   u32 *fragments_to_drop = 0;
1717   u32 *fragments_to_loopback = 0;
1718
1719   from = vlib_frame_vector_args (frame);
1720   n_left_from = frame->n_vectors;
1721   next_index = node->cached_next_index;
1722
1723   while (n_left_from > 0)
1724     {
1725       u32 n_left_to_next;
1726
1727       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1728
1729       while (n_left_from > 0 && n_left_to_next > 0)
1730         {
1731           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1732           u32 iph_offset0 = 0;
1733           vlib_buffer_t *b0;
1734           u32 next0;
1735           u8 cached0 = 0;
1736           ip4_header_t *ip0 = 0;
1737           nat_reass_ip4_t *reass0;
1738           udp_header_t *udp0;
1739           tcp_header_t *tcp0;
1740           icmp46_header_t *icmp0;
1741           clib_bihash_kv_16_8_t kv0, value0;
1742           snat_session_t *s0 = 0;
1743           u16 old_port0, new_port0;
1744           ip_csum_t sum0;
1745
1746           /* speculatively enqueue b0 to the current next frame */
1747           bi0 = from[0];
1748           to_next[0] = bi0;
1749           from += 1;
1750           to_next += 1;
1751           n_left_from -= 1;
1752           n_left_to_next -= 1;
1753
1754           b0 = vlib_get_buffer (vm, bi0);
1755
1756           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1757
1758           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1759           rx_fib_index0 =
1760             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1761                                                  sw_if_index0);
1762
1763           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1764             {
1765               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1766               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1767               goto trace0;
1768             }
1769
1770           if (is_output_feature)
1771             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1772
1773           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1774                                   iph_offset0);
1775
1776           udp0 = ip4_next_header (ip0);
1777           tcp0 = (tcp_header_t *) udp0;
1778           icmp0 = (icmp46_header_t *) udp0;
1779           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1780
1781           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1782                                                  ip0->dst_address,
1783                                                  ip0->fragment_id,
1784                                                  ip0->protocol,
1785                                                  1, &fragments_to_drop);
1786
1787           if (PREDICT_FALSE (!reass0))
1788             {
1789               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1790               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1791               nat_log_notice ("maximum reassemblies exceeded");
1792               goto trace0;
1793             }
1794
1795           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1796             {
1797               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1798                 {
1799                   if (is_output_feature)
1800                     {
1801                       if (PREDICT_FALSE
1802                           (nat_not_translate_output_feature_fwd
1803                            (sm, ip0, thread_index, now, vm, b0)))
1804                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1805                       goto trace0;
1806                     }
1807
1808                   next0 = icmp_in2out_ed_slow_path
1809                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1810                      next0, now, thread_index, &s0);
1811
1812                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1813                     {
1814                       if (s0)
1815                         reass0->sess_index = s0 - per_thread_data->sessions;
1816                       else
1817                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1818                       nat_ip4_reass_get_frags (reass0,
1819                                                &fragments_to_loopback);
1820                     }
1821
1822                   goto trace0;
1823                 }
1824
1825               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1826                           ip0->protocol, rx_fib_index0, udp0->src_port,
1827                           udp0->dst_port);
1828
1829               if (clib_bihash_search_16_8
1830                   (&per_thread_data->in2out_ed, &kv0, &value0))
1831                 {
1832                   if (is_output_feature)
1833                     {
1834                       if (PREDICT_FALSE
1835                           (nat44_ed_not_translate_output_feature
1836                            (sm, ip0, ip0->protocol, udp0->src_port,
1837                             udp0->dst_port, thread_index, sw_if_index0,
1838                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1839                         {
1840                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1841                           nat_ip4_reass_get_frags (reass0,
1842                                                    &fragments_to_loopback);
1843                           goto trace0;
1844                         }
1845                     }
1846                   else
1847                     {
1848                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1849                                                                  sw_if_index0,
1850                                                                  ip0, proto0,
1851                                                                  rx_fib_index0,
1852                                                                  thread_index)))
1853                         {
1854                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1855                           nat_ip4_reass_get_frags (reass0,
1856                                                    &fragments_to_loopback);
1857                           goto trace0;
1858                         }
1859                     }
1860
1861                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1862                                         &s0, node, next0, thread_index, now);
1863
1864                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1865                     goto trace0;
1866
1867                   if (PREDICT_FALSE (!s0))
1868                     {
1869                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1870                       goto trace0;
1871                     }
1872
1873                   reass0->sess_index = s0 - per_thread_data->sessions;
1874                 }
1875               else
1876                 {
1877                   s0 = pool_elt_at_index (per_thread_data->sessions,
1878                                           value0.value);
1879                   reass0->sess_index = value0.value;
1880                 }
1881               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1882             }
1883           else
1884             {
1885               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1886                 goto trace0;
1887               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1888                 {
1889                   if (nat_ip4_reass_add_fragment
1890                       (reass0, bi0, &fragments_to_drop))
1891                     {
1892                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1893                       nat_log_notice
1894                         ("maximum fragments per reassembly exceeded");
1895                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1896                       goto trace0;
1897                     }
1898                   cached0 = 1;
1899                   goto trace0;
1900                 }
1901               s0 = pool_elt_at_index (per_thread_data->sessions,
1902                                       reass0->sess_index);
1903             }
1904
1905           old_addr0 = ip0->src_address.as_u32;
1906           ip0->src_address = s0->out2in.addr;
1907           new_addr0 = ip0->src_address.as_u32;
1908           if (!is_output_feature)
1909             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1910
1911           sum0 = ip0->checksum;
1912           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1913                                  ip4_header_t,
1914                                  src_address /* changed member */ );
1915           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1916             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1917                                    s0->ext_host_addr.as_u32, ip4_header_t,
1918                                    dst_address);
1919           ip0->checksum = ip_csum_fold (sum0);
1920
1921           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1922             {
1923               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1924                 {
1925                   old_port0 = tcp0->src_port;
1926                   tcp0->src_port = s0->out2in.port;
1927                   new_port0 = tcp0->src_port;
1928
1929                   sum0 = tcp0->checksum;
1930                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1931                                          ip4_header_t,
1932                                          dst_address /* changed member */ );
1933                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1934                                          ip4_header_t /* cheat */ ,
1935                                          length /* changed member */ );
1936                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1937                     {
1938                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1939                                              s0->ext_host_addr.as_u32,
1940                                              ip4_header_t, dst_address);
1941                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1942                                              s0->ext_host_port, ip4_header_t,
1943                                              length);
1944                       tcp0->dst_port = s0->ext_host_port;
1945                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1946                     }
1947                   tcp0->checksum = ip_csum_fold (sum0);
1948                 }
1949               else
1950                 {
1951                   old_port0 = udp0->src_port;
1952                   udp0->src_port = s0->out2in.port;
1953                   udp0->checksum = 0;
1954                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1955                     {
1956                       udp0->dst_port = s0->ext_host_port;
1957                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1958                     }
1959                 }
1960             }
1961
1962           /* Hairpinning */
1963           if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_ICMP))
1964             nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1965                                      s0->ext_host_port, proto0, 1);
1966           else
1967             snat_icmp_hairpinning (sm, b0, ip0, icmp0, 1);
1968
1969           /* Accounting */
1970           nat44_session_update_counters (s0, now,
1971                                          vlib_buffer_length_in_chain (vm,
1972                                                                       b0));
1973           /* Per-user LRU list maintenance */
1974           nat44_session_update_lru (sm, s0, thread_index);
1975
1976         trace0:
1977           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1978                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1979             {
1980               nat44_reass_trace_t *t =
1981                 vlib_add_trace (vm, node, b0, sizeof (*t));
1982               t->cached = cached0;
1983               t->sw_if_index = sw_if_index0;
1984               t->next_index = next0;
1985             }
1986
1987           if (cached0)
1988             {
1989               n_left_to_next++;
1990               to_next--;
1991             }
1992           else
1993             {
1994               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1995
1996               /* verify speculative enqueue, maybe switch current next frame */
1997               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1998                                                to_next, n_left_to_next,
1999                                                bi0, next0);
2000             }
2001
2002           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2003             {
2004               from = vlib_frame_vector_args (frame);
2005               u32 len = vec_len (fragments_to_loopback);
2006               if (len <= VLIB_FRAME_SIZE)
2007                 {
2008                   clib_memcpy (from, fragments_to_loopback,
2009                                sizeof (u32) * len);
2010                   n_left_from = len;
2011                   vec_reset_length (fragments_to_loopback);
2012                 }
2013               else
2014                 {
2015                   clib_memcpy (from,
2016                                fragments_to_loopback + (len -
2017                                                         VLIB_FRAME_SIZE),
2018                                sizeof (u32) * VLIB_FRAME_SIZE);
2019                   n_left_from = VLIB_FRAME_SIZE;
2020                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2021                 }
2022             }
2023         }
2024
2025       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2026     }
2027
2028   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
2029                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
2030                                pkts_processed);
2031
2032   nat_send_all_to_node (vm, fragments_to_drop, node,
2033                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
2034                         NAT_IN2OUT_ED_NEXT_DROP);
2035
2036   vec_free (fragments_to_drop);
2037   vec_free (fragments_to_loopback);
2038   return frame->n_vectors;
2039 }
2040
2041 static uword
2042 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
2043                                vlib_node_runtime_t * node,
2044                                vlib_frame_t * frame)
2045 {
2046   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2047 }
2048
2049 /* *INDENT-OFF* */
2050 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2051   .function = nat44_ed_in2out_reass_node_fn,
2052   .name = "nat44-ed-in2out-reass",
2053   .vector_size = sizeof (u32),
2054   .format_trace = format_nat44_reass_trace,
2055   .type = VLIB_NODE_TYPE_INTERNAL,
2056   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2057   .error_strings = nat_in2out_ed_error_strings,
2058   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2059   .next_nodes = {
2060     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2061     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2062     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2063     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2064     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2065   },
2066 };
2067 /* *INDENT-ON* */
2068
2069 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2070                               nat44_ed_in2out_reass_node_fn);
2071
2072 static uword
2073 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2074                                       vlib_node_runtime_t * node,
2075                                       vlib_frame_t * frame)
2076 {
2077   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2078 }
2079
2080 /* *INDENT-OFF* */
2081 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2082   .function = nat44_ed_in2out_reass_output_node_fn,
2083   .name = "nat44-ed-in2out-reass-output",
2084   .vector_size = sizeof (u32),
2085   .format_trace = format_nat44_reass_trace,
2086   .type = VLIB_NODE_TYPE_INTERNAL,
2087   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2088   .error_strings = nat_in2out_ed_error_strings,
2089   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2090   .next_nodes = {
2091     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2092     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2093     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2094     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2095     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2096   },
2097 };
2098 /* *INDENT-ON* */
2099
2100 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2101                               nat44_ed_in2out_reass_output_node_fn);
2102
2103 /*
2104  * fd.io coding-style-patch-verification: ON
2105  *
2106  * Local Variables:
2107  * eval: (c-set-style "gnu")
2108  * End:
2109  */