src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #define foreach_nat_in2out_ed_error                       \
  33 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  34 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  35 _(OUT_OF_PORTS, "Out of ports")                         \
  36 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  37 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  38 _(DROP_FRAGMENT, "Drop fragment")                       \
  39 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  40 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  41
  42 typedef enum
  43 {
  44 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
  45   foreach_nat_in2out_ed_error
  46 #undef _
  47     NAT_IN2OUT_ED_N_ERROR,
  48 } nat_in2out_ed_error_t;
  49
  50 static char *nat_in2out_ed_error_strings[] = {
  51 #define _(sym,string) string,
  52   foreach_nat_in2out_ed_error
  53 #undef _
  54 };
  55
  56 typedef enum
  57 {
  58   NAT_IN2OUT_ED_NEXT_LOOKUP,
  59   NAT_IN2OUT_ED_NEXT_DROP,
  60   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
  61   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
  62   NAT_IN2OUT_ED_NEXT_REASS,
  63   NAT_IN2OUT_ED_N_NEXT,
  64 } nat_in2out_ed_next_t;
  65
  66 typedef struct
  67 {
  68   u32 sw_if_index;
  69   u32 next_index;
  70   u32 session_index;
  71   u32 is_slow_path;
  72 } nat_in2out_ed_trace_t;
  73
  74 vlib_node_registration_t nat44_ed_in2out_node;
  75 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
  76 vlib_node_registration_t nat44_ed_in2out_output_node;
  77 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
  78 vlib_node_registration_t nat44_ed_in2out_reass_node;
  79
  80 static u8 *
  81 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  82 {
  83   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  84   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  85   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  86   char *tag;
  87
  88   tag =
  89     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  90     "NAT44_IN2OUT_ED_FAST_PATH";
  91
  92   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  93               t->sw_if_index, t->next_index, t->session_index);
  94
  95   return s;
  96 }
  97
  98 static_always_inline int
  99 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 100 {
 101   icmp46_header_t *icmp0;
 102   nat_ed_ses_key_t key0;
 103   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 104   ip4_header_t *inner_ip0 = 0;
 105   void *l4_header = 0;
 106   icmp46_header_t *inner_icmp0;
 107
 108   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 109   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 110
 111   if (!icmp_is_error_message (icmp0))
 112     {
 113       key0.proto = IP_PROTOCOL_ICMP;
 114       key0.l_addr = ip0->src_address;
 115       key0.r_addr = ip0->dst_address;
 116       key0.l_port = echo0->identifier;
 117       key0.r_port = 0;
 118     }
 119   else
 120     {
 121       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 122       l4_header = ip4_next_header (inner_ip0);
 123       key0.proto = inner_ip0->protocol;
 124       key0.r_addr = inner_ip0->src_address;
 125       key0.l_addr = inner_ip0->dst_address;
 126       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 127         {
 128         case SNAT_PROTOCOL_ICMP:
 129           inner_icmp0 = (icmp46_header_t *) l4_header;
 130           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 131           key0.r_port = 0;
 132           key0.l_port = inner_echo0->identifier;
 133           break;
 134         case SNAT_PROTOCOL_UDP:
 135         case SNAT_PROTOCOL_TCP:
 136           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 137           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
 138           break;
 139         default:
 140           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
 141         }
 142     }
 143   *p_key0 = key0;
 144   return 0;
 145 }
 146
 147 int
 148 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 149 {
 150   snat_main_t *sm = &snat_main;
 151   nat44_is_idle_session_ctx_t *ctx = arg;
 152   snat_session_t *s;
 153   u64 sess_timeout_time;
 154   nat_ed_ses_key_t ed_key;
 155   clib_bihash_kv_16_8_t ed_kv;
 156   int i;
 157   snat_address_t *a;
 158   snat_session_key_t key;
 159   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 160                                                        ctx->thread_index);
 161
 162   s = pool_elt_at_index (tsm->sessions, kv->value);
 163   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 164   if (ctx->now >= sess_timeout_time)
 165     {
 166       if (is_fwd_bypass_session (s))
 167         goto delete;
 168
 169       ed_key.l_addr = s->out2in.addr;
 170       ed_key.r_addr = s->ext_host_addr;
 171       ed_key.fib_index = s->out2in.fib_index;
 172       if (snat_is_unk_proto_session (s))
 173         {
 174           ed_key.proto = s->in2out.port;
 175           ed_key.r_port = 0;
 176           ed_key.l_port = 0;
 177         }
 178       else
 179         {
 180           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 181           ed_key.l_port = s->out2in.port;
 182           ed_key.r_port = s->ext_host_port;
 183         }
 184       ed_kv.key[0] = ed_key.as_u64[0];
 185       ed_kv.key[1] = ed_key.as_u64[1];
 186       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 187         nat_log_warn ("out2in_ed key del failed");
 188
 189       if (snat_is_unk_proto_session (s))
 190         goto delete;
 191
 192       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 193                                            s->out2in.addr.as_u32,
 194                                            s->in2out.protocol,
 195                                            s->in2out.port,
 196                                            s->out2in.port,
 197                                            s->in2out.fib_index);
 198
 199       if (is_twice_nat_session (s))
 200         {
 201           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 202             {
 203               key.protocol = s->in2out.protocol;
 204               key.port = s->ext_host_nat_port;
 205               a = sm->twice_nat_addresses + i;
 206               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 207                 {
 208                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 209                                                       ctx->thread_index,
 210                                                       &key);
 211                   break;
 212                 }
 213             }
 214         }
 215
 216       if (snat_is_session_static (s))
 217         goto delete;
 218
 219       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 220                                           &s->out2in);
 221     delete:
 222       nat44_delete_session (sm, s, ctx->thread_index);
 223       return 1;
 224     }
 225
 226   return 0;
 227 }
 228
 229 static inline u32
 230 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 231                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 232                           u32 sw_if_index0, u32 rx_fib_index0,
 233                           vlib_node_runtime_t * node, u32 next0, f64 now,
 234                           u32 thread_index, snat_session_t ** p_s0)
 235 {
 236   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 237                        next0, thread_index, p_s0, 0);
 238   snat_session_t *s0 = *p_s0;
 239   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
 240     {
 241       /* Hairpinning */
 242       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 243         snat_icmp_hairpinning (sm, b0, ip0, icmp0, sm->endpoint_dependent);
 244       /* Accounting */
 245       nat44_session_update_counters (s0, now,
 246                                      vlib_buffer_length_in_chain
 247                                      (sm->vlib_main, b0));
 248     }
 249   return next0;
 250 }
 251
 252 static u32
 253 slow_path_ed (snat_main_t * sm,
 254               vlib_buffer_t * b,
 255               u32 rx_fib_index,
 256               clib_bihash_kv_16_8_t * kv,
 257               snat_session_t ** sessionp,
 258               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 259 {
 260   snat_session_t *s;
 261   snat_user_t *u;
 262   snat_session_key_t key0, key1;
 263   lb_nat_type_t lb = 0, is_sm = 0;
 264   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 265   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 266   u32 proto = ip_proto_to_snat_proto (key->proto);
 267   nat_outside_fib_t *outside_fib;
 268   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 269   fib_prefix_t pfx = {
 270     .fp_proto = FIB_PROTOCOL_IP4,
 271     .fp_len = 32,
 272     .fp_addr = {
 273                 .ip4.as_u32 = key->r_addr.as_u32,
 274                 },
 275   };
 276   nat44_is_idle_session_ctx_t ctx;
 277
 278   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 279     {
 280       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 281       nat_ipfix_logging_max_sessions (sm->max_translations);
 282       nat_log_notice ("maximum sessions exceeded");
 283       return NAT_IN2OUT_ED_NEXT_DROP;
 284     }
 285
 286   key0.addr = key->l_addr;
 287   key0.port = key->l_port;
 288   key1.protocol = key0.protocol = proto;
 289   key0.fib_index = rx_fib_index;
 290   key1.fib_index = sm->outside_fib_index;
 291   /* First try to match static mapping by local address and port */
 292   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0))
 293     {
 294       /* Try to create dynamic translation */
 295       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 296                                                thread_index, &key1,
 297                                                sm->port_per_thread,
 298                                                tsm->snat_thread_index))
 299         {
 300           nat_log_notice ("addresses exhausted");
 301           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 302           return NAT_IN2OUT_ED_NEXT_DROP;
 303         }
 304     }
 305   else
 306     is_sm = 1;
 307
 308   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 309   if (!u)
 310     {
 311       nat_log_warn ("create NAT user failed");
 312       if (!is_sm)
 313         snat_free_outside_address_and_port (sm->addresses,
 314                                             thread_index, &key1);
 315       return NAT_IN2OUT_ED_NEXT_DROP;
 316     }
 317
 318   s = nat_ed_session_alloc (sm, u, thread_index, now);
 319   if (!s)
 320     {
 321       nat44_delete_user_with_no_session (sm, u, thread_index);
 322       nat_log_warn ("create NAT session failed");
 323       if (!is_sm)
 324         snat_free_outside_address_and_port (sm->addresses,
 325                                             thread_index, &key1);
 326       return NAT_IN2OUT_ED_NEXT_DROP;
 327     }
 328
 329   user_session_increment (sm, u, is_sm);
 330   if (is_sm)
 331     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 332   if (lb)
 333     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 334   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 335   s->ext_host_addr = key->r_addr;
 336   s->ext_host_port = key->r_port;
 337   s->in2out = key0;
 338   s->out2in = key1;
 339   s->out2in.protocol = key0.protocol;
 340
 341   switch (vec_len (sm->outside_fibs))
 342     {
 343     case 0:
 344       s->out2in.fib_index = sm->outside_fib_index;
 345       break;
 346     case 1:
 347       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 348       break;
 349     default:
 350       /* *INDENT-OFF* */
 351       vec_foreach (outside_fib, sm->outside_fibs)
 352        {
 353           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 354           if (FIB_NODE_INDEX_INVALID != fei)
 355             {
 356               if (fib_entry_get_resolving_interface (fei) != ~0)
 357                 {
 358                   s->out2in.fib_index = outside_fib->fib_index;
 359                   break;
 360                 }
 361             }
 362         }
 363       /* *INDENT-ON* */
 364       break;
 365     }
 366
 367   /* Add to lookup tables */
 368   kv->value = s - tsm->sessions;
 369   ctx.now = now;
 370   ctx.thread_index = thread_index;
 371   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 372                                                nat44_i2o_ed_is_idle_session_cb,
 373                                                &ctx))
 374     nat_log_notice ("in2out-ed key add failed");
 375
 376   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 377               key1.port, key->r_port);
 378   kv->value = s - tsm->sessions;
 379   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 380                                                nat44_o2i_ed_is_idle_session_cb,
 381                                                &ctx))
 382     nat_log_notice ("out2in-ed key add failed");
 383
 384   *sessionp = s;
 385
 386   /* log NAT event */
 387   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 388                                        s->out2in.addr.as_u32,
 389                                        s->in2out.protocol,
 390                                        s->in2out.port,
 391                                        s->out2in.port, s->in2out.fib_index);
 392   return next;
 393 }
 394
 395 static_always_inline int
 396 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 397                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 398                         u32 rx_fib_index, u32 thread_index)
 399 {
 400   udp_header_t *udp = ip4_next_header (ip);
 401   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 402   clib_bihash_kv_16_8_t kv, value;
 403   snat_session_key_t key0, key1;
 404
 405   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 406               sm->outside_fib_index, udp->dst_port, udp->src_port);
 407
 408   /* NAT packet aimed at external address if */
 409   /* has active sessions */
 410   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 411     {
 412       key0.addr = ip->dst_address;
 413       key0.port = udp->dst_port;
 414       key0.protocol = proto;
 415       key0.fib_index = sm->outside_fib_index;
 416       /* or is static mappings */
 417       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0))
 418         return 0;
 419     }
 420   else
 421     return 0;
 422
 423   if (sm->forwarding_enabled)
 424     return 1;
 425
 426   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 427                                   rx_fib_index);
 428 }
 429
 430 static_always_inline int
 431 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 432                                       u32 thread_index, f64 now,
 433                                       vlib_main_t * vm, vlib_buffer_t * b)
 434 {
 435   nat_ed_ses_key_t key;
 436   clib_bihash_kv_16_8_t kv, value;
 437   udp_header_t *udp;
 438   snat_session_t *s = 0;
 439   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 440
 441   if (!sm->forwarding_enabled)
 442     return 0;
 443
 444   if (ip->protocol == IP_PROTOCOL_ICMP)
 445     {
 446       key.as_u64[0] = key.as_u64[1] = 0;
 447       if (icmp_get_ed_key (ip, &key))
 448         return 0;
 449       key.fib_index = 0;
 450       kv.key[0] = key.as_u64[0];
 451       kv.key[1] = key.as_u64[1];
 452     }
 453   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 454     {
 455       udp = ip4_next_header (ip);
 456       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 457                   udp->src_port, udp->dst_port);
 458     }
 459   else
 460     {
 461       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 462                   0);
 463     }
 464
 465   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 466     {
 467       s = pool_elt_at_index (tsm->sessions, value.value);
 468       if (is_fwd_bypass_session (s))
 469         {
 470           if (ip->protocol == IP_PROTOCOL_TCP)
 471             {
 472               tcp_header_t *tcp = ip4_next_header (ip);
 473               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 474                 return 1;
 475             }
 476           /* Accounting */
 477           nat44_session_update_counters (s, now,
 478                                          vlib_buffer_length_in_chain (vm, b));
 479           return 1;
 480         }
 481       else
 482         return 0;
 483     }
 484
 485   return 0;
 486 }
 487
 488 static_always_inline int
 489 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 490                                        u8 proto, u16 src_port, u16 dst_port,
 491                                        u32 thread_index, u32 rx_sw_if_index,
 492                                        u32 tx_sw_if_index)
 493 {
 494   clib_bihash_kv_16_8_t kv, value;
 495   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 496   snat_interface_t *i;
 497   snat_session_t *s;
 498   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 499   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 500
 501   /* src NAT check */
 502   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 503               src_port, dst_port);
 504   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 505     return 1;
 506
 507   /* dst NAT check */
 508   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 509               dst_port, src_port);
 510   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 511     {
 512       s = pool_elt_at_index (tsm->sessions, value.value);
 513       if (is_fwd_bypass_session (s))
 514         return 0;
 515
 516       /* hairpinning */
 517       /* *INDENT-OFF* */
 518       pool_foreach (i, sm->output_feature_interfaces,
 519       ({
 520         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 521            return 0;
 522       }));
 523       /* *INDENT-ON* */
 524       return 1;
 525     }
 526
 527   return 0;
 528 }
 529
 530 u32
 531 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 532                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 533                       u8 * p_proto, snat_session_key_t * p_value,
 534                       u8 * p_dont_translate, void *d, void *e)
 535 {
 536   icmp46_header_t *icmp;
 537   u32 sw_if_index;
 538   u32 rx_fib_index;
 539   nat_ed_ses_key_t key;
 540   snat_session_t *s = 0;
 541   u8 dont_translate = 0;
 542   clib_bihash_kv_16_8_t kv, value;
 543   u32 next = ~0;
 544   int err;
 545   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 546
 547   icmp = (icmp46_header_t *) ip4_next_header (ip);
 548   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 549   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 550
 551   key.as_u64[0] = key.as_u64[1] = 0;
 552   err = icmp_get_ed_key (ip, &key);
 553   if (err != 0)
 554     {
 555       b->error = node->errors[err];
 556       next = NAT_IN2OUT_ED_NEXT_DROP;
 557       goto out;
 558     }
 559   key.fib_index = rx_fib_index;
 560
 561   kv.key[0] = key.as_u64[0];
 562   kv.key[1] = key.as_u64[1];
 563
 564   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 565     {
 566       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 567         {
 568           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 569                                                                     key.proto,
 570                                                                     key.
 571                                                                     l_port,
 572                                                                     key.
 573                                                                     r_port,
 574                                                                     thread_index,
 575                                                                     sw_if_index,
 576                                                                     vnet_buffer
 577                                                                     (b)->
 578                                                                     sw_if_index
 579                                                                     [VLIB_TX])))
 580             {
 581               dont_translate = 1;
 582               goto out;
 583             }
 584         }
 585       else
 586         {
 587           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 588                                                      ip, SNAT_PROTOCOL_ICMP,
 589                                                      rx_fib_index,
 590                                                      thread_index)))
 591             {
 592               dont_translate = 1;
 593               goto out;
 594             }
 595         }
 596
 597       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 598         {
 599           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 600           next = NAT_IN2OUT_ED_NEXT_DROP;
 601           goto out;
 602         }
 603
 604       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 605                            thread_index, vlib_time_now (sm->vlib_main));
 606
 607       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
 608         goto out;
 609     }
 610   else
 611     {
 612       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 613                          icmp->type != ICMP4_echo_reply &&
 614                          !icmp_is_error_message (icmp)))
 615         {
 616           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 617           next = NAT_IN2OUT_ED_NEXT_DROP;
 618           goto out;
 619         }
 620
 621       s = pool_elt_at_index (tsm->sessions, value.value);
 622     }
 623
 624   *p_proto = ip_proto_to_snat_proto (key.proto);
 625 out:
 626   if (s)
 627     *p_value = s->out2in;
 628   *p_dont_translate = dont_translate;
 629   if (d)
 630     *(snat_session_t **) d = s;
 631   return next;
 632 }
 633
 634 static snat_session_t *
 635 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 636                                vlib_buffer_t * b,
 637                                ip4_header_t * ip,
 638                                u32 rx_fib_index,
 639                                u32 thread_index,
 640                                f64 now,
 641                                vlib_main_t * vm, vlib_node_runtime_t * node)
 642 {
 643   clib_bihash_kv_8_8_t kv, value;
 644   clib_bihash_kv_16_8_t s_kv, s_value;
 645   snat_static_mapping_t *m;
 646   u32 old_addr, new_addr = 0;
 647   ip_csum_t sum;
 648   snat_user_t *u;
 649   dlist_elt_t *head, *elt;
 650   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 651   u32 elt_index, head_index, ses_index;
 652   snat_session_t *s;
 653   u32 outside_fib_index = sm->outside_fib_index;
 654   int i;
 655   u8 is_sm = 0;
 656   nat_outside_fib_t *outside_fib;
 657   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 658   fib_prefix_t pfx = {
 659     .fp_proto = FIB_PROTOCOL_IP4,
 660     .fp_len = 32,
 661     .fp_addr = {
 662                 .ip4.as_u32 = ip->dst_address.as_u32,
 663                 },
 664   };
 665
 666   switch (vec_len (sm->outside_fibs))
 667     {
 668     case 0:
 669       outside_fib_index = sm->outside_fib_index;
 670       break;
 671     case 1:
 672       outside_fib_index = sm->outside_fibs[0].fib_index;
 673       break;
 674     default:
 675       /* *INDENT-OFF* */
 676       vec_foreach (outside_fib, sm->outside_fibs)
 677         {
 678           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 679           if (FIB_NODE_INDEX_INVALID != fei)
 680             {
 681               if (fib_entry_get_resolving_interface (fei) != ~0)
 682                 {
 683                   outside_fib_index = outside_fib->fib_index;
 684                   break;
 685                 }
 686             }
 687         }
 688       /* *INDENT-ON* */
 689       break;
 690     }
 691   old_addr = ip->src_address.as_u32;
 692
 693   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 694               rx_fib_index, 0, 0);
 695
 696   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 697     {
 698       s = pool_elt_at_index (tsm->sessions, s_value.value);
 699       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 700     }
 701   else
 702     {
 703       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 704         {
 705           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 706           nat_ipfix_logging_max_sessions (sm->max_translations);
 707           nat_log_notice ("maximum sessions exceeded");
 708           return 0;
 709         }
 710
 711       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 712                                   thread_index);
 713       if (!u)
 714         {
 715           nat_log_warn ("create NAT user failed");
 716           return 0;
 717         }
 718
 719       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 720
 721       /* Try to find static mapping first */
 722       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 723         {
 724           m = pool_elt_at_index (sm->static_mappings, value.value);
 725           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 726           is_sm = 1;
 727           goto create_ses;
 728         }
 729       /* Fallback to 3-tuple key */
 730       else
 731         {
 732           /* Choose same out address as for TCP/UDP session to same destination */
 733           head_index = u->sessions_per_user_list_head_index;
 734           head = pool_elt_at_index (tsm->list_pool, head_index);
 735           elt_index = head->next;
 736           if (PREDICT_FALSE (elt_index == ~0))
 737             ses_index = ~0;
 738           else
 739             {
 740               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 741               ses_index = elt->value;
 742             }
 743
 744           while (ses_index != ~0)
 745             {
 746               s = pool_elt_at_index (tsm->sessions, ses_index);
 747               elt_index = elt->next;
 748               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 749               ses_index = elt->value;
 750
 751               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 752                 {
 753                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 754
 755                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 756                               ip->protocol, outside_fib_index, 0, 0);
 757                   if (clib_bihash_search_16_8
 758                       (&tsm->out2in_ed, &s_kv, &s_value))
 759                     goto create_ses;
 760
 761                   break;
 762                 }
 763             }
 764
 765           for (i = 0; i < vec_len (sm->addresses); i++)
 766             {
 767               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 768                           ip->protocol, outside_fib_index, 0, 0);
 769               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 770                 {
 771                   new_addr = ip->src_address.as_u32 =
 772                     sm->addresses[i].addr.as_u32;
 773                   goto create_ses;
 774                 }
 775             }
 776           return 0;
 777         }
 778
 779     create_ses:
 780       s = nat_ed_session_alloc (sm, u, thread_index, now);
 781       if (!s)
 782         {
 783           nat44_delete_user_with_no_session (sm, u, thread_index);
 784           nat_log_warn ("create NAT session failed");
 785           return 0;
 786         }
 787
 788       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 789       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 790       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 791       s->out2in.addr.as_u32 = new_addr;
 792       s->out2in.fib_index = outside_fib_index;
 793       s->in2out.addr.as_u32 = old_addr;
 794       s->in2out.fib_index = rx_fib_index;
 795       s->in2out.port = s->out2in.port = ip->protocol;
 796       if (is_sm)
 797         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 798       user_session_increment (sm, u, is_sm);
 799
 800       /* Add to lookup tables */
 801       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 802                   rx_fib_index, 0, 0);
 803       s_kv.value = s - tsm->sessions;
 804       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 805         nat_log_notice ("in2out key add failed");
 806
 807       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 808                   outside_fib_index, 0, 0);
 809       s_kv.value = s - tsm->sessions;
 810       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 811         nat_log_notice ("out2in key add failed");
 812     }
 813
 814   /* Update IP checksum */
 815   sum = ip->checksum;
 816   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 817   ip->checksum = ip_csum_fold (sum);
 818
 819   /* Accounting */
 820   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 821
 822   /* Hairpinning */
 823   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 824     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 825
 826   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 827     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 828
 829   return s;
 830 }
 831
 832 static inline uword
 833 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 834                                 vlib_node_runtime_t * node,
 835                                 vlib_frame_t * frame, int is_slow_path,
 836                                 int is_output_feature)
 837 {
 838   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 839   nat_in2out_ed_next_t next_index;
 840   snat_main_t *sm = &snat_main;
 841   f64 now = vlib_time_now (vm);
 842   u32 thread_index = vm->thread_index;
 843   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 844
 845   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
 846     nat44_ed_in2out_node.index;
 847
 848   from = vlib_frame_vector_args (frame);
 849   n_left_from = frame->n_vectors;
 850   next_index = node->cached_next_index;
 851
 852   while (n_left_from > 0)
 853     {
 854       u32 n_left_to_next;
 855
 856       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 857
 858       while (n_left_from >= 4 && n_left_to_next >= 2)
 859         {
 860           u32 bi0, bi1;
 861           vlib_buffer_t *b0, *b1;
 862           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 863             new_addr0, old_addr0;
 864           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 865             new_addr1, old_addr1;
 866           u16 old_port0, new_port0, old_port1, new_port1;
 867           ip4_header_t *ip0, *ip1;
 868           udp_header_t *udp0, *udp1;
 869           tcp_header_t *tcp0, *tcp1;
 870           icmp46_header_t *icmp0, *icmp1;
 871           snat_session_t *s0 = 0, *s1 = 0;
 872           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 873           ip_csum_t sum0, sum1;
 874
 875           /* Prefetch next iteration. */
 876           {
 877             vlib_buffer_t *p2, *p3;
 878
 879             p2 = vlib_get_buffer (vm, from[2]);
 880             p3 = vlib_get_buffer (vm, from[3]);
 881
 882             vlib_prefetch_buffer_header (p2, LOAD);
 883             vlib_prefetch_buffer_header (p3, LOAD);
 884
 885             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 886             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 887           }
 888
 889           /* speculatively enqueue b0 and b1 to the current next frame */
 890           to_next[0] = bi0 = from[0];
 891           to_next[1] = bi1 = from[1];
 892           from += 2;
 893           to_next += 2;
 894           n_left_from -= 2;
 895           n_left_to_next -= 2;
 896
 897           b0 = vlib_get_buffer (vm, bi0);
 898           b1 = vlib_get_buffer (vm, bi1);
 899
 900           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
 901
 902           if (is_output_feature)
 903             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 904
 905           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 906                                   iph_offset0);
 907
 908           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 909           rx_fib_index0 =
 910             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 911                                                  sw_if_index0);
 912
 913           if (PREDICT_FALSE (ip0->ttl == 1))
 914             {
 915               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 916               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 917                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 918                                            0);
 919               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
 920               goto trace00;
 921             }
 922
 923           udp0 = ip4_next_header (ip0);
 924           tcp0 = (tcp_header_t *) udp0;
 925           icmp0 = (icmp46_header_t *) udp0;
 926           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 927
 928           if (is_slow_path)
 929             {
 930               if (PREDICT_FALSE (proto0 == ~0))
 931                 {
 932                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
 933                                                       rx_fib_index0,
 934                                                       thread_index, now, vm,
 935                                                       node);
 936                   if (!s0)
 937                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
 938                   goto trace00;
 939                 }
 940
 941               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 942                 {
 943                   next0 = icmp_in2out_ed_slow_path
 944                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 945                      next0, now, thread_index, &s0);
 946                   goto trace00;
 947                 }
 948             }
 949           else
 950             {
 951               if (PREDICT_FALSE (proto0 == ~0))
 952                 {
 953                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 954                   goto trace00;
 955                 }
 956
 957               if (ip4_is_fragment (ip0))
 958                 {
 959                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
 960                   goto trace00;
 961                 }
 962
 963               if (is_output_feature)
 964                 {
 965                   if (PREDICT_FALSE
 966                       (nat_not_translate_output_feature_fwd
 967                        (sm, ip0, thread_index, now, vm, b0)))
 968                     goto trace00;
 969                 }
 970
 971               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 972                 {
 973                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 974                   goto trace00;
 975                 }
 976             }
 977
 978           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
 979                       ip0->protocol, rx_fib_index0, udp0->src_port,
 980                       udp0->dst_port);
 981
 982           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
 983             {
 984               if (is_slow_path)
 985                 {
 986                   if (is_output_feature)
 987                     {
 988                       if (PREDICT_FALSE
 989                           (nat44_ed_not_translate_output_feature
 990                            (sm, ip0, ip0->protocol, udp0->src_port,
 991                             udp0->dst_port, thread_index, sw_if_index0,
 992                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
 993                         goto trace00;
 994                     }
 995                   else
 996                     {
 997                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
 998                                                                  sw_if_index0,
 999                                                                  ip0, proto0,
1000                                                                  rx_fib_index0,
1001                                                                  thread_index)))
1002                         goto trace00;
1003                     }
1004
1005                   next0 =
1006                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1007                                   next0, thread_index, now);
1008
1009                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1010                     goto trace00;
1011                 }
1012               else
1013                 {
1014                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1015                   goto trace00;
1016                 }
1017             }
1018           else
1019             {
1020               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1021             }
1022
1023           b0->flags |= VNET_BUFFER_F_IS_NATED;
1024
1025           if (!is_output_feature)
1026             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1027
1028           old_addr0 = ip0->src_address.as_u32;
1029           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1030           sum0 = ip0->checksum;
1031           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1032                                  src_address);
1033           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1034             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1035                                    s0->ext_host_addr.as_u32, ip4_header_t,
1036                                    dst_address);
1037           ip0->checksum = ip_csum_fold (sum0);
1038
1039           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1040             {
1041               old_port0 = tcp0->src_port;
1042               new_port0 = tcp0->src_port = s0->out2in.port;
1043
1044               sum0 = tcp0->checksum;
1045               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1046                                      dst_address);
1047               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1048                                      length);
1049               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1050                 {
1051                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1052                                          s0->ext_host_addr.as_u32,
1053                                          ip4_header_t, dst_address);
1054                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1055                                          s0->ext_host_port, ip4_header_t,
1056                                          length);
1057                   tcp0->dst_port = s0->ext_host_port;
1058                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1059                 }
1060               mss_clamping (sm, tcp0, &sum0);
1061               tcp0->checksum = ip_csum_fold (sum0);
1062               if (nat44_set_tcp_session_state_i2o
1063                   (sm, s0, tcp0, thread_index))
1064                 goto trace00;
1065             }
1066           else
1067             {
1068               udp0->src_port = s0->out2in.port;
1069               udp0->checksum = 0;
1070               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1071                 {
1072                   udp0->dst_port = s0->ext_host_port;
1073                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1074                 }
1075             }
1076
1077           /* Accounting */
1078           nat44_session_update_counters (s0, now,
1079                                          vlib_buffer_length_in_chain (vm,
1080                                                                       b0));
1081
1082         trace00:
1083           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1084                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1085             {
1086               nat_in2out_ed_trace_t *t =
1087                 vlib_add_trace (vm, node, b0, sizeof (*t));
1088               t->is_slow_path = is_slow_path;
1089               t->sw_if_index = sw_if_index0;
1090               t->next_index = next0;
1091               t->session_index = ~0;
1092               if (s0)
1093                 t->session_index = s0 - tsm->sessions;
1094             }
1095
1096           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1097
1098
1099           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1100
1101           if (is_output_feature)
1102             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1103
1104           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1105                                   iph_offset1);
1106
1107           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1108           rx_fib_index1 =
1109             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1110                                                  sw_if_index1);
1111
1112           if (PREDICT_FALSE (ip1->ttl == 1))
1113             {
1114               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1115               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1116                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1117                                            0);
1118               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1119               goto trace01;
1120             }
1121
1122           udp1 = ip4_next_header (ip1);
1123           tcp1 = (tcp_header_t *) udp1;
1124           icmp1 = (icmp46_header_t *) udp1;
1125           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1126
1127           if (is_slow_path)
1128             {
1129               if (PREDICT_FALSE (proto1 == ~0))
1130                 {
1131                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1132                                                       rx_fib_index1,
1133                                                       thread_index, now, vm,
1134                                                       node);
1135                   if (!s1)
1136                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1137                   goto trace01;
1138                 }
1139
1140               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1141                 {
1142                   next1 = icmp_in2out_ed_slow_path
1143                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1144                      next1, now, thread_index, &s1);
1145                   goto trace01;
1146                 }
1147             }
1148           else
1149             {
1150               if (PREDICT_FALSE (proto1 == ~0))
1151                 {
1152                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1153                   goto trace01;
1154                 }
1155
1156               if (ip4_is_fragment (ip1))
1157                 {
1158                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1159                   goto trace01;
1160                 }
1161
1162               if (is_output_feature)
1163                 {
1164                   if (PREDICT_FALSE
1165                       (nat_not_translate_output_feature_fwd
1166                        (sm, ip1, thread_index, now, vm, b1)))
1167                     goto trace01;
1168                 }
1169
1170               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1171                 {
1172                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1173                   goto trace01;
1174                 }
1175             }
1176
1177           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1178                       ip1->protocol, rx_fib_index1, udp1->src_port,
1179                       udp1->dst_port);
1180
1181           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1182             {
1183               if (is_slow_path)
1184                 {
1185                   if (is_output_feature)
1186                     {
1187                       if (PREDICT_FALSE
1188                           (nat44_ed_not_translate_output_feature
1189                            (sm, ip1, ip1->protocol, udp1->src_port,
1190                             udp1->dst_port, thread_index, sw_if_index1,
1191                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1192                         goto trace01;
1193                     }
1194                   else
1195                     {
1196                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1197                                                                  sw_if_index1,
1198                                                                  ip1, proto1,
1199                                                                  rx_fib_index1,
1200                                                                  thread_index)))
1201                         goto trace01;
1202                     }
1203
1204                   next1 =
1205                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1206                                   next1, thread_index, now);
1207
1208                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1209                     goto trace01;
1210                 }
1211               else
1212                 {
1213                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1214                   goto trace01;
1215                 }
1216             }
1217           else
1218             {
1219               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1220             }
1221
1222           b1->flags |= VNET_BUFFER_F_IS_NATED;
1223
1224           if (!is_output_feature)
1225             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1226
1227           old_addr1 = ip1->src_address.as_u32;
1228           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1229           sum1 = ip1->checksum;
1230           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1231                                  src_address);
1232           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1233             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1234                                    s1->ext_host_addr.as_u32, ip4_header_t,
1235                                    dst_address);
1236           ip1->checksum = ip_csum_fold (sum1);
1237
1238           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1239             {
1240               old_port1 = tcp1->src_port;
1241               new_port1 = tcp1->src_port = s1->out2in.port;
1242
1243               sum1 = tcp1->checksum;
1244               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1245                                      dst_address);
1246               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1247                                      length);
1248               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1249                 {
1250                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1251                                          s1->ext_host_addr.as_u32,
1252                                          ip4_header_t, dst_address);
1253                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1254                                          s1->ext_host_port, ip4_header_t,
1255                                          length);
1256                   tcp1->dst_port = s1->ext_host_port;
1257                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1258                 }
1259               tcp1->checksum = ip_csum_fold (sum1);
1260               mss_clamping (sm, tcp1, &sum1);
1261               if (nat44_set_tcp_session_state_i2o
1262                   (sm, s1, tcp1, thread_index))
1263                 goto trace01;
1264             }
1265           else
1266             {
1267               udp1->src_port = s1->out2in.port;
1268               udp1->checksum = 0;
1269               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1270                 {
1271                   udp1->dst_port = s1->ext_host_port;
1272                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1273                 }
1274             }
1275
1276           /* Accounting */
1277           nat44_session_update_counters (s1, now,
1278                                          vlib_buffer_length_in_chain (vm,
1279                                                                       b1));
1280
1281         trace01:
1282           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1283                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1284             {
1285               nat_in2out_ed_trace_t *t =
1286                 vlib_add_trace (vm, node, b1, sizeof (*t));
1287               t->is_slow_path = is_slow_path;
1288               t->sw_if_index = sw_if_index1;
1289               t->next_index = next1;
1290               t->session_index = ~0;
1291               if (s1)
1292                 t->session_index = s1 - tsm->sessions;
1293             }
1294
1295           pkts_processed += next1 != NAT_IN2OUT_ED_NEXT_DROP;
1296
1297           /* verify speculative enqueues, maybe switch current next frame */
1298           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1299                                            to_next, n_left_to_next,
1300                                            bi0, bi1, next0, next1);
1301         }
1302
1303       while (n_left_from > 0 && n_left_to_next > 0)
1304         {
1305           u32 bi0;
1306           vlib_buffer_t *b0;
1307           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1308             new_addr0, old_addr0;
1309           u16 old_port0, new_port0;
1310           ip4_header_t *ip0;
1311           udp_header_t *udp0;
1312           tcp_header_t *tcp0;
1313           icmp46_header_t *icmp0;
1314           snat_session_t *s0 = 0;
1315           clib_bihash_kv_16_8_t kv0, value0;
1316           ip_csum_t sum0;
1317
1318           /* speculatively enqueue b0 to the current next frame */
1319           bi0 = from[0];
1320           to_next[0] = bi0;
1321           from += 1;
1322           to_next += 1;
1323           n_left_from -= 1;
1324           n_left_to_next -= 1;
1325
1326           b0 = vlib_get_buffer (vm, bi0);
1327           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1328
1329           if (is_output_feature)
1330             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1331
1332           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1333                                   iph_offset0);
1334
1335           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1336           rx_fib_index0 =
1337             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1338                                                  sw_if_index0);
1339
1340           if (PREDICT_FALSE (ip0->ttl == 1))
1341             {
1342               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1343               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1344                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1345                                            0);
1346               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1347               goto trace0;
1348             }
1349
1350           udp0 = ip4_next_header (ip0);
1351           tcp0 = (tcp_header_t *) udp0;
1352           icmp0 = (icmp46_header_t *) udp0;
1353           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1354
1355           if (is_slow_path)
1356             {
1357               if (PREDICT_FALSE (proto0 == ~0))
1358                 {
1359                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1360                                                       rx_fib_index0,
1361                                                       thread_index, now, vm,
1362                                                       node);
1363                   if (!s0)
1364                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1365                   goto trace0;
1366                 }
1367
1368               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1369                 {
1370                   next0 = icmp_in2out_ed_slow_path
1371                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1372                      next0, now, thread_index, &s0);
1373                   goto trace0;
1374                 }
1375             }
1376           else
1377             {
1378               if (PREDICT_FALSE (proto0 == ~0))
1379                 {
1380                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1381                   goto trace0;
1382                 }
1383
1384               if (ip4_is_fragment (ip0))
1385                 {
1386                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1387                   goto trace0;
1388                 }
1389
1390               if (is_output_feature)
1391                 {
1392                   if (PREDICT_FALSE
1393                       (nat_not_translate_output_feature_fwd
1394                        (sm, ip0, thread_index, now, vm, b0)))
1395                     goto trace0;
1396                 }
1397
1398               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1399                 {
1400                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1401                   goto trace0;
1402                 }
1403             }
1404
1405           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1406                       ip0->protocol, rx_fib_index0, udp0->src_port,
1407                       udp0->dst_port);
1408
1409           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1410             {
1411               if (is_slow_path)
1412                 {
1413                   if (is_output_feature)
1414                     {
1415                       if (PREDICT_FALSE
1416                           (nat44_ed_not_translate_output_feature
1417                            (sm, ip0, ip0->protocol, udp0->src_port,
1418                             udp0->dst_port, thread_index, sw_if_index0,
1419                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1420                         goto trace0;
1421                     }
1422                   else
1423                     {
1424                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1425                                                                  sw_if_index0,
1426                                                                  ip0, proto0,
1427                                                                  rx_fib_index0,
1428                                                                  thread_index)))
1429                         goto trace0;
1430                     }
1431
1432                   next0 =
1433                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1434                                   next0, thread_index, now);
1435
1436                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1437                     goto trace0;
1438                 }
1439               else
1440                 {
1441                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1442                   goto trace0;
1443                 }
1444             }
1445           else
1446             {
1447               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1448             }
1449
1450           b0->flags |= VNET_BUFFER_F_IS_NATED;
1451
1452           if (!is_output_feature)
1453             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1454
1455           old_addr0 = ip0->src_address.as_u32;
1456           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1457           sum0 = ip0->checksum;
1458           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1459                                  src_address);
1460           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1461             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1462                                    s0->ext_host_addr.as_u32, ip4_header_t,
1463                                    dst_address);
1464           ip0->checksum = ip_csum_fold (sum0);
1465
1466           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1467             {
1468               old_port0 = tcp0->src_port;
1469               new_port0 = tcp0->src_port = s0->out2in.port;
1470
1471               sum0 = tcp0->checksum;
1472               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1473                                      dst_address);
1474               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1475                                      length);
1476               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1477                 {
1478                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1479                                          s0->ext_host_addr.as_u32,
1480                                          ip4_header_t, dst_address);
1481                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1482                                          s0->ext_host_port, ip4_header_t,
1483                                          length);
1484                   tcp0->dst_port = s0->ext_host_port;
1485                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1486                 }
1487               mss_clamping (sm, tcp0, &sum0);
1488               tcp0->checksum = ip_csum_fold (sum0);
1489               if (nat44_set_tcp_session_state_i2o
1490                   (sm, s0, tcp0, thread_index))
1491                 goto trace0;
1492             }
1493           else
1494             {
1495               udp0->src_port = s0->out2in.port;
1496               udp0->checksum = 0;
1497               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1498                 {
1499                   udp0->dst_port = s0->ext_host_port;
1500                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1501                 }
1502             }
1503
1504           /* Accounting */
1505           nat44_session_update_counters (s0, now,
1506                                          vlib_buffer_length_in_chain (vm,
1507                                                                       b0));
1508
1509         trace0:
1510           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1511                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1512             {
1513               nat_in2out_ed_trace_t *t =
1514                 vlib_add_trace (vm, node, b0, sizeof (*t));
1515               t->is_slow_path = is_slow_path;
1516               t->sw_if_index = sw_if_index0;
1517               t->next_index = next0;
1518               t->session_index = ~0;
1519               if (s0)
1520                 t->session_index = s0 - tsm->sessions;
1521             }
1522
1523           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1524
1525           /* verify speculative enqueue, maybe switch current next frame */
1526           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1527                                            to_next, n_left_to_next,
1528                                            bi0, next0);
1529         }
1530
1531       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1532     }
1533
1534   vlib_node_increment_counter (vm, stats_node_index,
1535                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1536                                pkts_processed);
1537   return frame->n_vectors;
1538 }
1539
1540 static uword
1541 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1542                               vlib_node_runtime_t * node,
1543                               vlib_frame_t * frame)
1544 {
1545   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1546 }
1547
1548 /* *INDENT-OFF* */
1549 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1550   .function = nat44_ed_in2out_fast_path_fn,
1551   .name = "nat44-ed-in2out",
1552   .vector_size = sizeof (u32),
1553   .format_trace = format_nat_in2out_ed_trace,
1554   .type = VLIB_NODE_TYPE_INTERNAL,
1555   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1556   .error_strings = nat_in2out_ed_error_strings,
1557   .runtime_data_bytes = sizeof (snat_runtime_t),
1558   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1559   .next_nodes = {
1560     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1561     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1562     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1563     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1564     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1565   },
1566 };
1567 /* *INDENT-ON* */
1568
1569 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1570                               nat44_ed_in2out_fast_path_fn);
1571
1572 static uword
1573 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1574                                      vlib_node_runtime_t * node,
1575                                      vlib_frame_t * frame)
1576 {
1577   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1578 }
1579
1580 /* *INDENT-OFF* */
1581 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1582   .function = nat44_ed_in2out_output_fast_path_fn,
1583   .name = "nat44-ed-in2out-output",
1584   .vector_size = sizeof (u32),
1585   .format_trace = format_nat_in2out_ed_trace,
1586   .type = VLIB_NODE_TYPE_INTERNAL,
1587   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1588   .error_strings = nat_in2out_ed_error_strings,
1589   .runtime_data_bytes = sizeof (snat_runtime_t),
1590   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1591   .next_nodes = {
1592     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1593     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1594     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1595     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1596     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1597   },
1598 };
1599 /* *INDENT-ON* */
1600
1601 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1602                               nat44_ed_in2out_output_fast_path_fn);
1603
1604 static uword
1605 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1606                               vlib_node_runtime_t * node,
1607                               vlib_frame_t * frame)
1608 {
1609   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1610 }
1611
1612 /* *INDENT-OFF* */
1613 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1614   .function = nat44_ed_in2out_slow_path_fn,
1615   .name = "nat44-ed-in2out-slowpath",
1616   .vector_size = sizeof (u32),
1617   .format_trace = format_nat_in2out_ed_trace,
1618   .type = VLIB_NODE_TYPE_INTERNAL,
1619   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1620   .error_strings = nat_in2out_ed_error_strings,
1621   .runtime_data_bytes = sizeof (snat_runtime_t),
1622   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1623   .next_nodes = {
1624     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1625     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1626     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1627     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1628     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1629   },
1630 };
1631 /* *INDENT-ON* */
1632
1633 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1634                               nat44_ed_in2out_slow_path_fn);
1635
1636 static uword
1637 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1638                                      vlib_node_runtime_t * node,
1639                                      vlib_frame_t * frame)
1640 {
1641   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1642 }
1643
1644 /* *INDENT-OFF* */
1645 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1646   .function = nat44_ed_in2out_output_slow_path_fn,
1647   .name = "nat44-ed-in2out-output-slowpath",
1648   .vector_size = sizeof (u32),
1649   .format_trace = format_nat_in2out_ed_trace,
1650   .type = VLIB_NODE_TYPE_INTERNAL,
1651   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1652   .error_strings = nat_in2out_ed_error_strings,
1653   .runtime_data_bytes = sizeof (snat_runtime_t),
1654   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1655   .next_nodes = {
1656     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1657     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1658     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1659     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1660     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1661   },
1662 };
1663 /* *INDENT-ON* */
1664
1665 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1666                               nat44_ed_in2out_output_slow_path_fn);
1667
1668 static inline uword
1669 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1670                                       vlib_node_runtime_t * node,
1671                                       vlib_frame_t * frame,
1672                                       int is_output_feature)
1673 {
1674   u32 n_left_from, *from, *to_next;
1675   nat_in2out_ed_next_t next_index;
1676   u32 pkts_processed = 0;
1677   snat_main_t *sm = &snat_main;
1678   f64 now = vlib_time_now (vm);
1679   u32 thread_index = vm->thread_index;
1680   snat_main_per_thread_data_t *per_thread_data =
1681     &sm->per_thread_data[thread_index];
1682   u32 *fragments_to_drop = 0;
1683   u32 *fragments_to_loopback = 0;
1684
1685   from = vlib_frame_vector_args (frame);
1686   n_left_from = frame->n_vectors;
1687   next_index = node->cached_next_index;
1688
1689   while (n_left_from > 0)
1690     {
1691       u32 n_left_to_next;
1692
1693       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1694
1695       while (n_left_from > 0 && n_left_to_next > 0)
1696         {
1697           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1698           u32 iph_offset0 = 0;
1699           vlib_buffer_t *b0;
1700           u32 next0;
1701           u8 cached0 = 0;
1702           ip4_header_t *ip0 = 0;
1703           nat_reass_ip4_t *reass0;
1704           udp_header_t *udp0;
1705           tcp_header_t *tcp0;
1706           icmp46_header_t *icmp0;
1707           clib_bihash_kv_16_8_t kv0, value0;
1708           snat_session_t *s0 = 0;
1709           u16 old_port0, new_port0;
1710           ip_csum_t sum0;
1711
1712           /* speculatively enqueue b0 to the current next frame */
1713           bi0 = from[0];
1714           to_next[0] = bi0;
1715           from += 1;
1716           to_next += 1;
1717           n_left_from -= 1;
1718           n_left_to_next -= 1;
1719
1720           b0 = vlib_get_buffer (vm, bi0);
1721
1722           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1723
1724           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1725           rx_fib_index0 =
1726             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1727                                                  sw_if_index0);
1728
1729           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1730             {
1731               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1732               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1733               goto trace0;
1734             }
1735
1736           if (is_output_feature)
1737             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1738
1739           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1740                                   iph_offset0);
1741
1742           udp0 = ip4_next_header (ip0);
1743           tcp0 = (tcp_header_t *) udp0;
1744           icmp0 = (icmp46_header_t *) udp0;
1745           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1746
1747           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1748                                                  ip0->dst_address,
1749                                                  ip0->fragment_id,
1750                                                  ip0->protocol,
1751                                                  1, &fragments_to_drop);
1752
1753           if (PREDICT_FALSE (!reass0))
1754             {
1755               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1756               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1757               nat_log_notice ("maximum reassemblies exceeded");
1758               goto trace0;
1759             }
1760
1761           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1762             {
1763               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1764                 {
1765                   if (is_output_feature)
1766                     {
1767                       if (PREDICT_FALSE
1768                           (nat_not_translate_output_feature_fwd
1769                            (sm, ip0, thread_index, now, vm, b0)))
1770                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1771                       goto trace0;
1772                     }
1773
1774                   next0 = icmp_in2out_ed_slow_path
1775                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1776                      next0, now, thread_index, &s0);
1777
1778                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1779                     {
1780                       if (s0)
1781                         reass0->sess_index = s0 - per_thread_data->sessions;
1782                       else
1783                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1784                       nat_ip4_reass_get_frags (reass0,
1785                                                &fragments_to_loopback);
1786                     }
1787
1788                   goto trace0;
1789                 }
1790
1791               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1792                           ip0->protocol, rx_fib_index0, udp0->src_port,
1793                           udp0->dst_port);
1794
1795               if (clib_bihash_search_16_8
1796                   (&per_thread_data->in2out_ed, &kv0, &value0))
1797                 {
1798                   if (is_output_feature)
1799                     {
1800                       if (PREDICT_FALSE
1801                           (nat44_ed_not_translate_output_feature
1802                            (sm, ip0, ip0->protocol, udp0->src_port,
1803                             udp0->dst_port, thread_index, sw_if_index0,
1804                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1805                         {
1806                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1807                           nat_ip4_reass_get_frags (reass0,
1808                                                    &fragments_to_loopback);
1809                           goto trace0;
1810                         }
1811                     }
1812                   else
1813                     {
1814                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1815                                                                  sw_if_index0,
1816                                                                  ip0, proto0,
1817                                                                  rx_fib_index0,
1818                                                                  thread_index)))
1819                         {
1820                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1821                           nat_ip4_reass_get_frags (reass0,
1822                                                    &fragments_to_loopback);
1823                           goto trace0;
1824                         }
1825                     }
1826
1827                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1828                                         &s0, node, next0, thread_index, now);
1829
1830                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1831                     goto trace0;
1832
1833                   reass0->sess_index = s0 - per_thread_data->sessions;
1834                 }
1835               else
1836                 {
1837                   s0 = pool_elt_at_index (per_thread_data->sessions,
1838                                           value0.value);
1839                   reass0->sess_index = value0.value;
1840                 }
1841               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1842             }
1843           else
1844             {
1845               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1846                 goto trace0;
1847               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1848                 {
1849                   if (nat_ip4_reass_add_fragment
1850                       (reass0, bi0, &fragments_to_drop))
1851                     {
1852                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1853                       nat_log_notice
1854                         ("maximum fragments per reassembly exceeded");
1855                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1856                       goto trace0;
1857                     }
1858                   cached0 = 1;
1859                   goto trace0;
1860                 }
1861               s0 = pool_elt_at_index (per_thread_data->sessions,
1862                                       reass0->sess_index);
1863             }
1864
1865           old_addr0 = ip0->src_address.as_u32;
1866           ip0->src_address = s0->out2in.addr;
1867           new_addr0 = ip0->src_address.as_u32;
1868           if (!is_output_feature)
1869             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1870
1871           sum0 = ip0->checksum;
1872           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1873                                  ip4_header_t,
1874                                  src_address /* changed member */ );
1875           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1876             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1877                                    s0->ext_host_addr.as_u32, ip4_header_t,
1878                                    dst_address);
1879           ip0->checksum = ip_csum_fold (sum0);
1880
1881           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1882             {
1883               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1884                 {
1885                   old_port0 = tcp0->src_port;
1886                   tcp0->src_port = s0->out2in.port;
1887                   new_port0 = tcp0->src_port;
1888
1889                   sum0 = tcp0->checksum;
1890                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1891                                          ip4_header_t,
1892                                          dst_address /* changed member */ );
1893                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1894                                          ip4_header_t /* cheat */ ,
1895                                          length /* changed member */ );
1896                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1897                     {
1898                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1899                                              s0->ext_host_addr.as_u32,
1900                                              ip4_header_t, dst_address);
1901                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1902                                              s0->ext_host_port, ip4_header_t,
1903                                              length);
1904                       tcp0->dst_port = s0->ext_host_port;
1905                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1906                     }
1907                   tcp0->checksum = ip_csum_fold (sum0);
1908                 }
1909               else
1910                 {
1911                   old_port0 = udp0->src_port;
1912                   udp0->src_port = s0->out2in.port;
1913                   udp0->checksum = 0;
1914                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1915                     {
1916                       udp0->dst_port = s0->ext_host_port;
1917                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1918                     }
1919                 }
1920             }
1921
1922           /* Hairpinning */
1923           if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_ICMP))
1924             nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1925                                      s0->ext_host_port, proto0, 1);
1926           else
1927             snat_icmp_hairpinning (sm, b0, ip0, icmp0, 1);
1928
1929           /* Accounting */
1930           nat44_session_update_counters (s0, now,
1931                                          vlib_buffer_length_in_chain (vm,
1932                                                                       b0));
1933
1934         trace0:
1935           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1936                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1937             {
1938               nat44_reass_trace_t *t =
1939                 vlib_add_trace (vm, node, b0, sizeof (*t));
1940               t->cached = cached0;
1941               t->sw_if_index = sw_if_index0;
1942               t->next_index = next0;
1943             }
1944
1945           if (cached0)
1946             {
1947               n_left_to_next++;
1948               to_next--;
1949             }
1950           else
1951             {
1952               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1953
1954               /* verify speculative enqueue, maybe switch current next frame */
1955               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1956                                                to_next, n_left_to_next,
1957                                                bi0, next0);
1958             }
1959
1960           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1961             {
1962               from = vlib_frame_vector_args (frame);
1963               u32 len = vec_len (fragments_to_loopback);
1964               if (len <= VLIB_FRAME_SIZE)
1965                 {
1966                   clib_memcpy (from, fragments_to_loopback,
1967                                sizeof (u32) * len);
1968                   n_left_from = len;
1969                   vec_reset_length (fragments_to_loopback);
1970                 }
1971               else
1972                 {
1973                   clib_memcpy (from,
1974                                fragments_to_loopback + (len -
1975                                                         VLIB_FRAME_SIZE),
1976                                sizeof (u32) * VLIB_FRAME_SIZE);
1977                   n_left_from = VLIB_FRAME_SIZE;
1978                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1979                 }
1980             }
1981         }
1982
1983       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1984     }
1985
1986   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
1987                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1988                                pkts_processed);
1989
1990   nat_send_all_to_node (vm, fragments_to_drop, node,
1991                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
1992                         NAT_IN2OUT_ED_NEXT_DROP);
1993
1994   vec_free (fragments_to_drop);
1995   vec_free (fragments_to_loopback);
1996   return frame->n_vectors;
1997 }
1998
1999 static uword
2000 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
2001                                vlib_node_runtime_t * node,
2002                                vlib_frame_t * frame)
2003 {
2004   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2005 }
2006
2007 /* *INDENT-OFF* */
2008 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2009   .function = nat44_ed_in2out_reass_node_fn,
2010   .name = "nat44-ed-in2out-reass",
2011   .vector_size = sizeof (u32),
2012   .format_trace = format_nat44_reass_trace,
2013   .type = VLIB_NODE_TYPE_INTERNAL,
2014   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2015   .error_strings = nat_in2out_ed_error_strings,
2016   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2017   .next_nodes = {
2018     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2019     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2020     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2021     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2022     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2023   },
2024 };
2025 /* *INDENT-ON* */
2026
2027 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2028                               nat44_ed_in2out_reass_node_fn);
2029
2030 static uword
2031 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2032                                       vlib_node_runtime_t * node,
2033                                       vlib_frame_t * frame)
2034 {
2035   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2036 }
2037
2038 /* *INDENT-OFF* */
2039 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2040   .function = nat44_ed_in2out_reass_output_node_fn,
2041   .name = "nat44-ed-in2out-reass-output",
2042   .vector_size = sizeof (u32),
2043   .format_trace = format_nat44_reass_trace,
2044   .type = VLIB_NODE_TYPE_INTERNAL,
2045   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2046   .error_strings = nat_in2out_ed_error_strings,
2047   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2048   .next_nodes = {
2049     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2050     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2051     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2052     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2053     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2054   },
2055 };
2056 /* *INDENT-ON* */
2057
2058 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2059                               nat44_ed_in2out_reass_output_node_fn);
2060
2061 /*
2062  * fd.io coding-style-patch-verification: ON
2063  *
2064  * Local Variables:
2065  * eval: (c-set-style "gnu")
2066  * End:
2067  */