src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #define foreach_nat_in2out_ed_error                       \
  33 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  34 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  35 _(OUT_OF_PORTS, "Out of ports")                         \
  36 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  37 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  38 _(DROP_FRAGMENT, "Drop fragment")                       \
  39 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  40 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
  41 _(NON_SYN, "non-SYN packet try to create session")
  42
  43 typedef enum
  44 {
  45 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
  46   foreach_nat_in2out_ed_error
  47 #undef _
  48     NAT_IN2OUT_ED_N_ERROR,
  49 } nat_in2out_ed_error_t;
  50
  51 static char *nat_in2out_ed_error_strings[] = {
  52 #define _(sym,string) string,
  53   foreach_nat_in2out_ed_error
  54 #undef _
  55 };
  56
  57 typedef enum
  58 {
  59   NAT_IN2OUT_ED_NEXT_LOOKUP,
  60   NAT_IN2OUT_ED_NEXT_DROP,
  61   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
  62   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
  63   NAT_IN2OUT_ED_NEXT_REASS,
  64   NAT_IN2OUT_ED_N_NEXT,
  65 } nat_in2out_ed_next_t;
  66
  67 typedef struct
  68 {
  69   u32 sw_if_index;
  70   u32 next_index;
  71   u32 session_index;
  72   u32 is_slow_path;
  73 } nat_in2out_ed_trace_t;
  74
  75 vlib_node_registration_t nat44_ed_in2out_node;
  76 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
  77 vlib_node_registration_t nat44_ed_in2out_output_node;
  78 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
  79 vlib_node_registration_t nat44_ed_in2out_reass_node;
  80
  81 static u8 *
  82 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  83 {
  84   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  85   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  86   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  87   char *tag;
  88
  89   tag =
  90     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  91     "NAT44_IN2OUT_ED_FAST_PATH";
  92
  93   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  94               t->sw_if_index, t->next_index, t->session_index);
  95
  96   return s;
  97 }
  98
  99 static_always_inline int
 100 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 101 {
 102   icmp46_header_t *icmp0;
 103   nat_ed_ses_key_t key0;
 104   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 105   ip4_header_t *inner_ip0 = 0;
 106   void *l4_header = 0;
 107   icmp46_header_t *inner_icmp0;
 108
 109   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 110   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 111
 112   if (!icmp_is_error_message (icmp0))
 113     {
 114       key0.proto = IP_PROTOCOL_ICMP;
 115       key0.l_addr = ip0->src_address;
 116       key0.r_addr = ip0->dst_address;
 117       key0.l_port = echo0->identifier;
 118       key0.r_port = 0;
 119     }
 120   else
 121     {
 122       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 123       l4_header = ip4_next_header (inner_ip0);
 124       key0.proto = inner_ip0->protocol;
 125       key0.r_addr = inner_ip0->src_address;
 126       key0.l_addr = inner_ip0->dst_address;
 127       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 128         {
 129         case SNAT_PROTOCOL_ICMP:
 130           inner_icmp0 = (icmp46_header_t *) l4_header;
 131           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 132           key0.r_port = 0;
 133           key0.l_port = inner_echo0->identifier;
 134           break;
 135         case SNAT_PROTOCOL_UDP:
 136         case SNAT_PROTOCOL_TCP:
 137           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 138           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
 139           break;
 140         default:
 141           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
 142         }
 143     }
 144   *p_key0 = key0;
 145   return 0;
 146 }
 147
 148 int
 149 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 150 {
 151   snat_main_t *sm = &snat_main;
 152   nat44_is_idle_session_ctx_t *ctx = arg;
 153   snat_session_t *s;
 154   u64 sess_timeout_time;
 155   nat_ed_ses_key_t ed_key;
 156   clib_bihash_kv_16_8_t ed_kv;
 157   int i;
 158   snat_address_t *a;
 159   snat_session_key_t key;
 160   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 161                                                        ctx->thread_index);
 162
 163   s = pool_elt_at_index (tsm->sessions, kv->value);
 164   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 165   if (ctx->now >= sess_timeout_time)
 166     {
 167       if (is_fwd_bypass_session (s))
 168         goto delete;
 169
 170       ed_key.l_addr = s->out2in.addr;
 171       ed_key.r_addr = s->ext_host_addr;
 172       ed_key.fib_index = s->out2in.fib_index;
 173       if (snat_is_unk_proto_session (s))
 174         {
 175           ed_key.proto = s->in2out.port;
 176           ed_key.r_port = 0;
 177           ed_key.l_port = 0;
 178         }
 179       else
 180         {
 181           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 182           ed_key.l_port = s->out2in.port;
 183           ed_key.r_port = s->ext_host_port;
 184         }
 185       ed_kv.key[0] = ed_key.as_u64[0];
 186       ed_kv.key[1] = ed_key.as_u64[1];
 187       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 188         nat_log_warn ("out2in_ed key del failed");
 189
 190       if (snat_is_unk_proto_session (s))
 191         goto delete;
 192
 193       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 194                                            s->out2in.addr.as_u32,
 195                                            s->in2out.protocol,
 196                                            s->in2out.port,
 197                                            s->out2in.port,
 198                                            s->in2out.fib_index);
 199
 200       if (is_twice_nat_session (s))
 201         {
 202           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 203             {
 204               key.protocol = s->in2out.protocol;
 205               key.port = s->ext_host_nat_port;
 206               a = sm->twice_nat_addresses + i;
 207               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 208                 {
 209                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 210                                                       ctx->thread_index,
 211                                                       &key);
 212                   break;
 213                 }
 214             }
 215         }
 216
 217       if (snat_is_session_static (s))
 218         goto delete;
 219
 220       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 221                                           &s->out2in);
 222     delete:
 223       nat44_delete_session (sm, s, ctx->thread_index);
 224       return 1;
 225     }
 226
 227   return 0;
 228 }
 229
 230 static inline u32
 231 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 232                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 233                           u32 sw_if_index0, u32 rx_fib_index0,
 234                           vlib_node_runtime_t * node, u32 next0, f64 now,
 235                           u32 thread_index, snat_session_t ** p_s0)
 236 {
 237   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 238                        next0, thread_index, p_s0, 0);
 239   snat_session_t *s0 = *p_s0;
 240   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
 241     {
 242       /* Accounting */
 243       nat44_session_update_counters (s0, now,
 244                                      vlib_buffer_length_in_chain
 245                                      (sm->vlib_main, b0));
 246       /* Per-user LRU list maintenance */
 247       nat44_session_update_lru (sm, s0, thread_index);
 248     }
 249   return next0;
 250 }
 251
 252 static u32
 253 slow_path_ed (snat_main_t * sm,
 254               vlib_buffer_t * b,
 255               u32 rx_fib_index,
 256               clib_bihash_kv_16_8_t * kv,
 257               snat_session_t ** sessionp,
 258               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 259 {
 260   snat_session_t *s = 0;
 261   snat_user_t *u;
 262   snat_session_key_t key0, key1;
 263   lb_nat_type_t lb = 0, is_sm = 0;
 264   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 265   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 266   u32 proto = ip_proto_to_snat_proto (key->proto);
 267   nat_outside_fib_t *outside_fib;
 268   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 269   u8 identity_nat;
 270   fib_prefix_t pfx = {
 271     .fp_proto = FIB_PROTOCOL_IP4,
 272     .fp_len = 32,
 273     .fp_addr = {
 274                 .ip4.as_u32 = key->r_addr.as_u32,
 275                 },
 276   };
 277   nat44_is_idle_session_ctx_t ctx;
 278
 279   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 280     {
 281       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 282       nat_ipfix_logging_max_sessions (sm->max_translations);
 283       nat_log_notice ("maximum sessions exceeded");
 284       return NAT_IN2OUT_ED_NEXT_DROP;
 285     }
 286
 287   key0.addr = key->l_addr;
 288   key0.port = key->l_port;
 289   key1.protocol = key0.protocol = proto;
 290   key0.fib_index = rx_fib_index;
 291   key1.fib_index = sm->outside_fib_index;
 292   /* First try to match static mapping by local address and port */
 293   if (snat_static_mapping_match
 294       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 295     {
 296       /* Try to create dynamic translation */
 297       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 298                                                thread_index, &key1,
 299                                                sm->port_per_thread,
 300                                                tsm->snat_thread_index))
 301         {
 302           nat_log_notice ("addresses exhausted");
 303           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 304           return NAT_IN2OUT_ED_NEXT_DROP;
 305         }
 306     }
 307   else
 308     {
 309       if (PREDICT_FALSE (identity_nat))
 310         {
 311           *sessionp = s;
 312           return next;
 313         }
 314
 315       is_sm = 1;
 316     }
 317
 318   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 319   if (!u)
 320     {
 321       nat_log_warn ("create NAT user failed");
 322       if (!is_sm)
 323         snat_free_outside_address_and_port (sm->addresses,
 324                                             thread_index, &key1);
 325       return NAT_IN2OUT_ED_NEXT_DROP;
 326     }
 327
 328   s = nat_ed_session_alloc (sm, u, thread_index, now);
 329   if (!s)
 330     {
 331       nat44_delete_user_with_no_session (sm, u, thread_index);
 332       nat_log_warn ("create NAT session failed");
 333       if (!is_sm)
 334         snat_free_outside_address_and_port (sm->addresses,
 335                                             thread_index, &key1);
 336       return NAT_IN2OUT_ED_NEXT_DROP;
 337     }
 338
 339   user_session_increment (sm, u, is_sm);
 340   if (is_sm)
 341     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 342   if (lb)
 343     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 344   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 345   s->ext_host_addr = key->r_addr;
 346   s->ext_host_port = key->r_port;
 347   s->in2out = key0;
 348   s->out2in = key1;
 349   s->out2in.protocol = key0.protocol;
 350
 351   switch (vec_len (sm->outside_fibs))
 352     {
 353     case 0:
 354       s->out2in.fib_index = sm->outside_fib_index;
 355       break;
 356     case 1:
 357       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 358       break;
 359     default:
 360       /* *INDENT-OFF* */
 361       vec_foreach (outside_fib, sm->outside_fibs)
 362        {
 363           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 364           if (FIB_NODE_INDEX_INVALID != fei)
 365             {
 366               if (fib_entry_get_resolving_interface (fei) != ~0)
 367                 {
 368                   s->out2in.fib_index = outside_fib->fib_index;
 369                   break;
 370                 }
 371             }
 372         }
 373       /* *INDENT-ON* */
 374       break;
 375     }
 376
 377   /* Add to lookup tables */
 378   kv->value = s - tsm->sessions;
 379   ctx.now = now;
 380   ctx.thread_index = thread_index;
 381   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 382                                                nat44_i2o_ed_is_idle_session_cb,
 383                                                &ctx))
 384     nat_log_notice ("in2out-ed key add failed");
 385
 386   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 387               key1.port, key->r_port);
 388   kv->value = s - tsm->sessions;
 389   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 390                                                nat44_o2i_ed_is_idle_session_cb,
 391                                                &ctx))
 392     nat_log_notice ("out2in-ed key add failed");
 393
 394   *sessionp = s;
 395
 396   /* log NAT event */
 397   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 398                                        s->out2in.addr.as_u32,
 399                                        s->in2out.protocol,
 400                                        s->in2out.port,
 401                                        s->out2in.port, s->in2out.fib_index);
 402   return next;
 403 }
 404
 405 static_always_inline int
 406 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 407                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 408                         u32 rx_fib_index, u32 thread_index)
 409 {
 410   udp_header_t *udp = ip4_next_header (ip);
 411   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 412   clib_bihash_kv_16_8_t kv, value;
 413   snat_session_key_t key0, key1;
 414
 415   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 416               sm->outside_fib_index, udp->dst_port, udp->src_port);
 417
 418   /* NAT packet aimed at external address if */
 419   /* has active sessions */
 420   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 421     {
 422       key0.addr = ip->dst_address;
 423       key0.port = udp->dst_port;
 424       key0.protocol = proto;
 425       key0.fib_index = sm->outside_fib_index;
 426       /* or is static mappings */
 427       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 428         return 0;
 429     }
 430   else
 431     return 0;
 432
 433   if (sm->forwarding_enabled)
 434     return 1;
 435
 436   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 437                                   rx_fib_index);
 438 }
 439
 440 static_always_inline int
 441 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 442                                       u32 thread_index, f64 now,
 443                                       vlib_main_t * vm, vlib_buffer_t * b)
 444 {
 445   nat_ed_ses_key_t key;
 446   clib_bihash_kv_16_8_t kv, value;
 447   udp_header_t *udp;
 448   snat_session_t *s = 0;
 449   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 450
 451   if (!sm->forwarding_enabled)
 452     return 0;
 453
 454   if (ip->protocol == IP_PROTOCOL_ICMP)
 455     {
 456       key.as_u64[0] = key.as_u64[1] = 0;
 457       if (icmp_get_ed_key (ip, &key))
 458         return 0;
 459       key.fib_index = 0;
 460       kv.key[0] = key.as_u64[0];
 461       kv.key[1] = key.as_u64[1];
 462     }
 463   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 464     {
 465       udp = ip4_next_header (ip);
 466       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 467                   udp->src_port, udp->dst_port);
 468     }
 469   else
 470     {
 471       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 472                   0);
 473     }
 474
 475   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 476     {
 477       s = pool_elt_at_index (tsm->sessions, value.value);
 478       if (is_fwd_bypass_session (s))
 479         {
 480           if (ip->protocol == IP_PROTOCOL_TCP)
 481             {
 482               tcp_header_t *tcp = ip4_next_header (ip);
 483               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 484                 return 1;
 485             }
 486           /* Accounting */
 487           nat44_session_update_counters (s, now,
 488                                          vlib_buffer_length_in_chain (vm, b));
 489           /* Per-user LRU list maintenance */
 490           nat44_session_update_lru (sm, s, thread_index);
 491           return 1;
 492         }
 493       else
 494         return 0;
 495     }
 496
 497   return 0;
 498 }
 499
 500 static_always_inline int
 501 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 502                                        u8 proto, u16 src_port, u16 dst_port,
 503                                        u32 thread_index, u32 rx_sw_if_index,
 504                                        u32 tx_sw_if_index)
 505 {
 506   clib_bihash_kv_16_8_t kv, value;
 507   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 508   snat_interface_t *i;
 509   snat_session_t *s;
 510   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 511   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 512
 513   /* src NAT check */
 514   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 515               src_port, dst_port);
 516   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 517     {
 518       s = pool_elt_at_index (tsm->sessions, value.value);
 519       if (nat44_is_ses_closed (s))
 520         {
 521           nat_log_debug ("TCP close connection %U", format_snat_session,
 522                          &sm->per_thread_data[thread_index], s);
 523           nat_free_session_data (sm, s, thread_index);
 524           nat44_delete_session (sm, s, thread_index);
 525         }
 526       else
 527         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 528       return 1;
 529     }
 530
 531   /* dst NAT check */
 532   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 533               dst_port, src_port);
 534   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 535     {
 536       s = pool_elt_at_index (tsm->sessions, value.value);
 537       if (is_fwd_bypass_session (s))
 538         return 0;
 539
 540       /* hairpinning */
 541       /* *INDENT-OFF* */
 542       pool_foreach (i, sm->output_feature_interfaces,
 543       ({
 544         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 545            return 0;
 546       }));
 547       /* *INDENT-ON* */
 548       return 1;
 549     }
 550
 551   return 0;
 552 }
 553
 554 u32
 555 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 556                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 557                       u8 * p_proto, snat_session_key_t * p_value,
 558                       u8 * p_dont_translate, void *d, void *e)
 559 {
 560   icmp46_header_t *icmp;
 561   u32 sw_if_index;
 562   u32 rx_fib_index;
 563   nat_ed_ses_key_t key;
 564   snat_session_t *s = 0;
 565   u8 dont_translate = 0;
 566   clib_bihash_kv_16_8_t kv, value;
 567   u32 next = ~0;
 568   int err;
 569   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 570
 571   icmp = (icmp46_header_t *) ip4_next_header (ip);
 572   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 573   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 574
 575   key.as_u64[0] = key.as_u64[1] = 0;
 576   err = icmp_get_ed_key (ip, &key);
 577   if (err != 0)
 578     {
 579       b->error = node->errors[err];
 580       next = NAT_IN2OUT_ED_NEXT_DROP;
 581       goto out;
 582     }
 583   key.fib_index = rx_fib_index;
 584
 585   kv.key[0] = key.as_u64[0];
 586   kv.key[1] = key.as_u64[1];
 587
 588   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 589     {
 590       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 591         {
 592           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 593                                                                     key.proto,
 594                                                                     key.
 595                                                                     l_port,
 596                                                                     key.
 597                                                                     r_port,
 598                                                                     thread_index,
 599                                                                     sw_if_index,
 600                                                                     vnet_buffer
 601                                                                     (b)->
 602                                                                     sw_if_index
 603                                                                     [VLIB_TX])))
 604             {
 605               dont_translate = 1;
 606               goto out;
 607             }
 608         }
 609       else
 610         {
 611           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 612                                                      ip, SNAT_PROTOCOL_ICMP,
 613                                                      rx_fib_index,
 614                                                      thread_index)))
 615             {
 616               dont_translate = 1;
 617               goto out;
 618             }
 619         }
 620
 621       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 622         {
 623           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 624           next = NAT_IN2OUT_ED_NEXT_DROP;
 625           goto out;
 626         }
 627
 628       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 629                            thread_index, vlib_time_now (sm->vlib_main));
 630
 631       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
 632         goto out;
 633
 634       if (!s)
 635         {
 636           dont_translate = 1;
 637           goto out;
 638         }
 639     }
 640   else
 641     {
 642       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 643                          icmp->type != ICMP4_echo_reply &&
 644                          !icmp_is_error_message (icmp)))
 645         {
 646           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 647           next = NAT_IN2OUT_ED_NEXT_DROP;
 648           goto out;
 649         }
 650
 651       s = pool_elt_at_index (tsm->sessions, value.value);
 652     }
 653
 654   *p_proto = ip_proto_to_snat_proto (key.proto);
 655 out:
 656   if (s)
 657     *p_value = s->out2in;
 658   *p_dont_translate = dont_translate;
 659   if (d)
 660     *(snat_session_t **) d = s;
 661   return next;
 662 }
 663
 664 static snat_session_t *
 665 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 666                                vlib_buffer_t * b,
 667                                ip4_header_t * ip,
 668                                u32 rx_fib_index,
 669                                u32 thread_index,
 670                                f64 now,
 671                                vlib_main_t * vm, vlib_node_runtime_t * node)
 672 {
 673   clib_bihash_kv_8_8_t kv, value;
 674   clib_bihash_kv_16_8_t s_kv, s_value;
 675   snat_static_mapping_t *m;
 676   u32 old_addr, new_addr = 0;
 677   ip_csum_t sum;
 678   snat_user_t *u;
 679   dlist_elt_t *head, *elt;
 680   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 681   u32 elt_index, head_index, ses_index;
 682   snat_session_t *s;
 683   u32 outside_fib_index = sm->outside_fib_index;
 684   int i;
 685   u8 is_sm = 0;
 686   nat_outside_fib_t *outside_fib;
 687   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 688   fib_prefix_t pfx = {
 689     .fp_proto = FIB_PROTOCOL_IP4,
 690     .fp_len = 32,
 691     .fp_addr = {
 692                 .ip4.as_u32 = ip->dst_address.as_u32,
 693                 },
 694   };
 695
 696   switch (vec_len (sm->outside_fibs))
 697     {
 698     case 0:
 699       outside_fib_index = sm->outside_fib_index;
 700       break;
 701     case 1:
 702       outside_fib_index = sm->outside_fibs[0].fib_index;
 703       break;
 704     default:
 705       /* *INDENT-OFF* */
 706       vec_foreach (outside_fib, sm->outside_fibs)
 707         {
 708           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 709           if (FIB_NODE_INDEX_INVALID != fei)
 710             {
 711               if (fib_entry_get_resolving_interface (fei) != ~0)
 712                 {
 713                   outside_fib_index = outside_fib->fib_index;
 714                   break;
 715                 }
 716             }
 717         }
 718       /* *INDENT-ON* */
 719       break;
 720     }
 721   old_addr = ip->src_address.as_u32;
 722
 723   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 724               rx_fib_index, 0, 0);
 725
 726   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 727     {
 728       s = pool_elt_at_index (tsm->sessions, s_value.value);
 729       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 730     }
 731   else
 732     {
 733       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 734         {
 735           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 736           nat_ipfix_logging_max_sessions (sm->max_translations);
 737           nat_log_notice ("maximum sessions exceeded");
 738           return 0;
 739         }
 740
 741       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 742                                   thread_index);
 743       if (!u)
 744         {
 745           nat_log_warn ("create NAT user failed");
 746           return 0;
 747         }
 748
 749       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 750
 751       /* Try to find static mapping first */
 752       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 753         {
 754           m = pool_elt_at_index (sm->static_mappings, value.value);
 755           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 756           is_sm = 1;
 757           goto create_ses;
 758         }
 759       /* Fallback to 3-tuple key */
 760       else
 761         {
 762           /* Choose same out address as for TCP/UDP session to same destination */
 763           head_index = u->sessions_per_user_list_head_index;
 764           head = pool_elt_at_index (tsm->list_pool, head_index);
 765           elt_index = head->next;
 766           if (PREDICT_FALSE (elt_index == ~0))
 767             ses_index = ~0;
 768           else
 769             {
 770               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 771               ses_index = elt->value;
 772             }
 773
 774           while (ses_index != ~0)
 775             {
 776               s = pool_elt_at_index (tsm->sessions, ses_index);
 777               elt_index = elt->next;
 778               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 779               ses_index = elt->value;
 780
 781               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 782                 {
 783                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 784
 785                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 786                               ip->protocol, outside_fib_index, 0, 0);
 787                   if (clib_bihash_search_16_8
 788                       (&tsm->out2in_ed, &s_kv, &s_value))
 789                     goto create_ses;
 790
 791                   break;
 792                 }
 793             }
 794
 795           for (i = 0; i < vec_len (sm->addresses); i++)
 796             {
 797               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 798                           ip->protocol, outside_fib_index, 0, 0);
 799               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 800                 {
 801                   new_addr = ip->src_address.as_u32 =
 802                     sm->addresses[i].addr.as_u32;
 803                   goto create_ses;
 804                 }
 805             }
 806           return 0;
 807         }
 808
 809     create_ses:
 810       s = nat_ed_session_alloc (sm, u, thread_index, now);
 811       if (!s)
 812         {
 813           nat44_delete_user_with_no_session (sm, u, thread_index);
 814           nat_log_warn ("create NAT session failed");
 815           return 0;
 816         }
 817
 818       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 819       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 820       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 821       s->out2in.addr.as_u32 = new_addr;
 822       s->out2in.fib_index = outside_fib_index;
 823       s->in2out.addr.as_u32 = old_addr;
 824       s->in2out.fib_index = rx_fib_index;
 825       s->in2out.port = s->out2in.port = ip->protocol;
 826       if (is_sm)
 827         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 828       user_session_increment (sm, u, is_sm);
 829
 830       /* Add to lookup tables */
 831       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 832                   rx_fib_index, 0, 0);
 833       s_kv.value = s - tsm->sessions;
 834       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 835         nat_log_notice ("in2out key add failed");
 836
 837       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 838                   outside_fib_index, 0, 0);
 839       s_kv.value = s - tsm->sessions;
 840       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 841         nat_log_notice ("out2in key add failed");
 842     }
 843
 844   /* Update IP checksum */
 845   sum = ip->checksum;
 846   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 847   ip->checksum = ip_csum_fold (sum);
 848
 849   /* Accounting */
 850   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 851   /* Per-user LRU list maintenance */
 852   nat44_session_update_lru (sm, s, thread_index);
 853
 854   /* Hairpinning */
 855   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 856     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 857
 858   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 859     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 860
 861   return s;
 862 }
 863
 864 static inline uword
 865 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 866                                 vlib_node_runtime_t * node,
 867                                 vlib_frame_t * frame, int is_slow_path,
 868                                 int is_output_feature)
 869 {
 870   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 871   nat_in2out_ed_next_t next_index;
 872   snat_main_t *sm = &snat_main;
 873   f64 now = vlib_time_now (vm);
 874   u32 thread_index = vm->thread_index;
 875   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 876
 877   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
 878     nat44_ed_in2out_node.index;
 879
 880   from = vlib_frame_vector_args (frame);
 881   n_left_from = frame->n_vectors;
 882   next_index = node->cached_next_index;
 883
 884   while (n_left_from > 0)
 885     {
 886       u32 n_left_to_next;
 887
 888       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 889
 890       while (n_left_from >= 4 && n_left_to_next >= 2)
 891         {
 892           u32 bi0, bi1;
 893           vlib_buffer_t *b0, *b1;
 894           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 895             new_addr0, old_addr0;
 896           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 897             new_addr1, old_addr1;
 898           u16 old_port0, new_port0, old_port1, new_port1;
 899           ip4_header_t *ip0, *ip1;
 900           udp_header_t *udp0, *udp1;
 901           tcp_header_t *tcp0, *tcp1;
 902           icmp46_header_t *icmp0, *icmp1;
 903           snat_session_t *s0 = 0, *s1 = 0;
 904           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 905           ip_csum_t sum0, sum1;
 906
 907           /* Prefetch next iteration. */
 908           {
 909             vlib_buffer_t *p2, *p3;
 910
 911             p2 = vlib_get_buffer (vm, from[2]);
 912             p3 = vlib_get_buffer (vm, from[3]);
 913
 914             vlib_prefetch_buffer_header (p2, LOAD);
 915             vlib_prefetch_buffer_header (p3, LOAD);
 916
 917             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 918             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 919           }
 920
 921           /* speculatively enqueue b0 and b1 to the current next frame */
 922           to_next[0] = bi0 = from[0];
 923           to_next[1] = bi1 = from[1];
 924           from += 2;
 925           to_next += 2;
 926           n_left_from -= 2;
 927           n_left_to_next -= 2;
 928
 929           b0 = vlib_get_buffer (vm, bi0);
 930           b1 = vlib_get_buffer (vm, bi1);
 931
 932           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
 933
 934           if (is_output_feature)
 935             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 936
 937           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 938                                   iph_offset0);
 939
 940           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 941           rx_fib_index0 =
 942             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 943                                                  sw_if_index0);
 944
 945           if (PREDICT_FALSE (ip0->ttl == 1))
 946             {
 947               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 948               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 949                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 950                                            0);
 951               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
 952               goto trace00;
 953             }
 954
 955           udp0 = ip4_next_header (ip0);
 956           tcp0 = (tcp_header_t *) udp0;
 957           icmp0 = (icmp46_header_t *) udp0;
 958           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 959
 960           if (is_slow_path)
 961             {
 962               if (PREDICT_FALSE (proto0 == ~0))
 963                 {
 964                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
 965                                                       rx_fib_index0,
 966                                                       thread_index, now, vm,
 967                                                       node);
 968                   if (!s0)
 969                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
 970                   goto trace00;
 971                 }
 972
 973               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 974                 {
 975                   next0 = icmp_in2out_ed_slow_path
 976                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 977                      next0, now, thread_index, &s0);
 978                   goto trace00;
 979                 }
 980             }
 981           else
 982             {
 983               if (PREDICT_FALSE (proto0 == ~0))
 984                 {
 985                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
 986                   goto trace00;
 987                 }
 988
 989               if (ip4_is_fragment (ip0))
 990                 {
 991                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
 992                   goto trace00;
 993                 }
 994
 995               if (is_output_feature)
 996                 {
 997                   if (PREDICT_FALSE
 998                       (nat_not_translate_output_feature_fwd
 999                        (sm, ip0, thread_index, now, vm, b0)))
1000                     goto trace00;
1001                 }
1002
1003               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1004                 {
1005                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1006                   goto trace00;
1007                 }
1008             }
1009
1010           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1011                       ip0->protocol, rx_fib_index0, udp0->src_port,
1012                       udp0->dst_port);
1013
1014           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1015             {
1016               if (is_slow_path)
1017                 {
1018                   if (is_output_feature)
1019                     {
1020                       if (PREDICT_FALSE
1021                           (nat44_ed_not_translate_output_feature
1022                            (sm, ip0, ip0->protocol, udp0->src_port,
1023                             udp0->dst_port, thread_index, sw_if_index0,
1024                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1025                         goto trace00;
1026                     }
1027                   else
1028                     {
1029                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1030                                                                  sw_if_index0,
1031                                                                  ip0, proto0,
1032                                                                  rx_fib_index0,
1033                                                                  thread_index)))
1034                         goto trace00;
1035                     }
1036
1037                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1038                     {
1039                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
1040                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1041                       goto trace00;
1042                     }
1043
1044                   next0 =
1045                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1046                                   next0, thread_index, now);
1047
1048                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1049                     goto trace00;
1050
1051                   if (PREDICT_FALSE (!s0))
1052                     goto trace00;
1053                 }
1054               else
1055                 {
1056                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1057                   goto trace00;
1058                 }
1059             }
1060           else
1061             {
1062               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1063             }
1064
1065           b0->flags |= VNET_BUFFER_F_IS_NATED;
1066
1067           if (!is_output_feature)
1068             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1069
1070           old_addr0 = ip0->src_address.as_u32;
1071           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1072           sum0 = ip0->checksum;
1073           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1074                                  src_address);
1075           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1076             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1077                                    s0->ext_host_addr.as_u32, ip4_header_t,
1078                                    dst_address);
1079           ip0->checksum = ip_csum_fold (sum0);
1080
1081           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1082             {
1083               old_port0 = tcp0->src_port;
1084               new_port0 = tcp0->src_port = s0->out2in.port;
1085
1086               sum0 = tcp0->checksum;
1087               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1088                                      dst_address);
1089               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1090                                      length);
1091               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1092                 {
1093                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1094                                          s0->ext_host_addr.as_u32,
1095                                          ip4_header_t, dst_address);
1096                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1097                                          s0->ext_host_port, ip4_header_t,
1098                                          length);
1099                   tcp0->dst_port = s0->ext_host_port;
1100                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1101                 }
1102               mss_clamping (sm, tcp0, &sum0);
1103               tcp0->checksum = ip_csum_fold (sum0);
1104               if (nat44_set_tcp_session_state_i2o
1105                   (sm, s0, tcp0, thread_index))
1106                 goto trace00;
1107             }
1108           else
1109             {
1110               udp0->src_port = s0->out2in.port;
1111               udp0->checksum = 0;
1112               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1113                 {
1114                   udp0->dst_port = s0->ext_host_port;
1115                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1116                 }
1117             }
1118
1119           /* Accounting */
1120           nat44_session_update_counters (s0, now,
1121                                          vlib_buffer_length_in_chain (vm,
1122                                                                       b0));
1123           /* Per-user LRU list maintenance */
1124           nat44_session_update_lru (sm, s0, thread_index);
1125
1126         trace00:
1127           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1128                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1129             {
1130               nat_in2out_ed_trace_t *t =
1131                 vlib_add_trace (vm, node, b0, sizeof (*t));
1132               t->is_slow_path = is_slow_path;
1133               t->sw_if_index = sw_if_index0;
1134               t->next_index = next0;
1135               t->session_index = ~0;
1136               if (s0)
1137                 t->session_index = s0 - tsm->sessions;
1138             }
1139
1140           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1141
1142
1143           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1144
1145           if (is_output_feature)
1146             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1147
1148           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1149                                   iph_offset1);
1150
1151           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1152           rx_fib_index1 =
1153             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1154                                                  sw_if_index1);
1155
1156           if (PREDICT_FALSE (ip1->ttl == 1))
1157             {
1158               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1159               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1160                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1161                                            0);
1162               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1163               goto trace01;
1164             }
1165
1166           udp1 = ip4_next_header (ip1);
1167           tcp1 = (tcp_header_t *) udp1;
1168           icmp1 = (icmp46_header_t *) udp1;
1169           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1170
1171           if (is_slow_path)
1172             {
1173               if (PREDICT_FALSE (proto1 == ~0))
1174                 {
1175                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1176                                                       rx_fib_index1,
1177                                                       thread_index, now, vm,
1178                                                       node);
1179                   if (!s1)
1180                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1181                   goto trace01;
1182                 }
1183
1184               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1185                 {
1186                   next1 = icmp_in2out_ed_slow_path
1187                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1188                      next1, now, thread_index, &s1);
1189                   goto trace01;
1190                 }
1191             }
1192           else
1193             {
1194               if (PREDICT_FALSE (proto1 == ~0))
1195                 {
1196                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1197                   goto trace01;
1198                 }
1199
1200               if (ip4_is_fragment (ip1))
1201                 {
1202                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1203                   goto trace01;
1204                 }
1205
1206               if (is_output_feature)
1207                 {
1208                   if (PREDICT_FALSE
1209                       (nat_not_translate_output_feature_fwd
1210                        (sm, ip1, thread_index, now, vm, b1)))
1211                     goto trace01;
1212                 }
1213
1214               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1215                 {
1216                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1217                   goto trace01;
1218                 }
1219             }
1220
1221           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1222                       ip1->protocol, rx_fib_index1, udp1->src_port,
1223                       udp1->dst_port);
1224
1225           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1226             {
1227               if (is_slow_path)
1228                 {
1229                   if (is_output_feature)
1230                     {
1231                       if (PREDICT_FALSE
1232                           (nat44_ed_not_translate_output_feature
1233                            (sm, ip1, ip1->protocol, udp1->src_port,
1234                             udp1->dst_port, thread_index, sw_if_index1,
1235                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1236                         goto trace01;
1237                     }
1238                   else
1239                     {
1240                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1241                                                                  sw_if_index1,
1242                                                                  ip1, proto1,
1243                                                                  rx_fib_index1,
1244                                                                  thread_index)))
1245                         goto trace01;
1246                     }
1247
1248                   if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1249                     {
1250                       b1->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
1251                       next1 = NAT_IN2OUT_ED_NEXT_DROP;
1252                       goto trace01;
1253                     }
1254
1255                   next1 =
1256                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1257                                   next1, thread_index, now);
1258
1259                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1260                     goto trace01;
1261
1262                   if (PREDICT_FALSE (!s1))
1263                     goto trace01;
1264                 }
1265               else
1266                 {
1267                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1268                   goto trace01;
1269                 }
1270             }
1271           else
1272             {
1273               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1274             }
1275
1276           b1->flags |= VNET_BUFFER_F_IS_NATED;
1277
1278           if (!is_output_feature)
1279             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1280
1281           old_addr1 = ip1->src_address.as_u32;
1282           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1283           sum1 = ip1->checksum;
1284           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1285                                  src_address);
1286           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1287             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1288                                    s1->ext_host_addr.as_u32, ip4_header_t,
1289                                    dst_address);
1290           ip1->checksum = ip_csum_fold (sum1);
1291
1292           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1293             {
1294               old_port1 = tcp1->src_port;
1295               new_port1 = tcp1->src_port = s1->out2in.port;
1296
1297               sum1 = tcp1->checksum;
1298               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1299                                      dst_address);
1300               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1301                                      length);
1302               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1303                 {
1304                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1305                                          s1->ext_host_addr.as_u32,
1306                                          ip4_header_t, dst_address);
1307                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1308                                          s1->ext_host_port, ip4_header_t,
1309                                          length);
1310                   tcp1->dst_port = s1->ext_host_port;
1311                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1312                 }
1313               tcp1->checksum = ip_csum_fold (sum1);
1314               mss_clamping (sm, tcp1, &sum1);
1315               if (nat44_set_tcp_session_state_i2o
1316                   (sm, s1, tcp1, thread_index))
1317                 goto trace01;
1318             }
1319           else
1320             {
1321               udp1->src_port = s1->out2in.port;
1322               udp1->checksum = 0;
1323               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1324                 {
1325                   udp1->dst_port = s1->ext_host_port;
1326                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1327                 }
1328             }
1329
1330           /* Accounting */
1331           nat44_session_update_counters (s1, now,
1332                                          vlib_buffer_length_in_chain (vm,
1333                                                                       b1));
1334           /* Per-user LRU list maintenance */
1335           nat44_session_update_lru (sm, s1, thread_index);
1336
1337         trace01:
1338           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1339                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1340             {
1341               nat_in2out_ed_trace_t *t =
1342                 vlib_add_trace (vm, node, b1, sizeof (*t));
1343               t->is_slow_path = is_slow_path;
1344               t->sw_if_index = sw_if_index1;
1345               t->next_index = next1;
1346               t->session_index = ~0;
1347               if (s1)
1348                 t->session_index = s1 - tsm->sessions;
1349             }
1350
1351           pkts_processed += next1 != NAT_IN2OUT_ED_NEXT_DROP;
1352
1353           /* verify speculative enqueues, maybe switch current next frame */
1354           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1355                                            to_next, n_left_to_next,
1356                                            bi0, bi1, next0, next1);
1357         }
1358
1359       while (n_left_from > 0 && n_left_to_next > 0)
1360         {
1361           u32 bi0;
1362           vlib_buffer_t *b0;
1363           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1364             new_addr0, old_addr0;
1365           u16 old_port0, new_port0;
1366           ip4_header_t *ip0;
1367           udp_header_t *udp0;
1368           tcp_header_t *tcp0;
1369           icmp46_header_t *icmp0;
1370           snat_session_t *s0 = 0;
1371           clib_bihash_kv_16_8_t kv0, value0;
1372           ip_csum_t sum0;
1373
1374           /* speculatively enqueue b0 to the current next frame */
1375           bi0 = from[0];
1376           to_next[0] = bi0;
1377           from += 1;
1378           to_next += 1;
1379           n_left_from -= 1;
1380           n_left_to_next -= 1;
1381
1382           b0 = vlib_get_buffer (vm, bi0);
1383           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1384
1385           if (is_output_feature)
1386             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1387
1388           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1389                                   iph_offset0);
1390
1391           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1392           rx_fib_index0 =
1393             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1394                                                  sw_if_index0);
1395
1396           if (PREDICT_FALSE (ip0->ttl == 1))
1397             {
1398               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1399               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1400                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1401                                            0);
1402               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1403               goto trace0;
1404             }
1405
1406           udp0 = ip4_next_header (ip0);
1407           tcp0 = (tcp_header_t *) udp0;
1408           icmp0 = (icmp46_header_t *) udp0;
1409           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1410
1411           if (is_slow_path)
1412             {
1413               if (PREDICT_FALSE (proto0 == ~0))
1414                 {
1415                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1416                                                       rx_fib_index0,
1417                                                       thread_index, now, vm,
1418                                                       node);
1419                   if (!s0)
1420                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1421                   goto trace0;
1422                 }
1423
1424               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1425                 {
1426                   next0 = icmp_in2out_ed_slow_path
1427                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1428                      next0, now, thread_index, &s0);
1429                   goto trace0;
1430                 }
1431             }
1432           else
1433             {
1434               if (PREDICT_FALSE (proto0 == ~0))
1435                 {
1436                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1437                   goto trace0;
1438                 }
1439
1440               if (ip4_is_fragment (ip0))
1441                 {
1442                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1443                   goto trace0;
1444                 }
1445
1446               if (is_output_feature)
1447                 {
1448                   if (PREDICT_FALSE
1449                       (nat_not_translate_output_feature_fwd
1450                        (sm, ip0, thread_index, now, vm, b0)))
1451                     goto trace0;
1452                 }
1453
1454               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1455                 {
1456                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1457                   goto trace0;
1458                 }
1459             }
1460
1461           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1462                       ip0->protocol, rx_fib_index0, udp0->src_port,
1463                       udp0->dst_port);
1464
1465           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1466             {
1467               if (is_slow_path)
1468                 {
1469                   if (is_output_feature)
1470                     {
1471                       if (PREDICT_FALSE
1472                           (nat44_ed_not_translate_output_feature
1473                            (sm, ip0, ip0->protocol, udp0->src_port,
1474                             udp0->dst_port, thread_index, sw_if_index0,
1475                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1476                         goto trace0;
1477                     }
1478                   else
1479                     {
1480                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1481                                                                  sw_if_index0,
1482                                                                  ip0, proto0,
1483                                                                  rx_fib_index0,
1484                                                                  thread_index)))
1485                         goto trace0;
1486                     }
1487
1488                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1489                     {
1490                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
1491                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1492                       goto trace0;
1493                     }
1494
1495                   next0 =
1496                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1497                                   next0, thread_index, now);
1498
1499                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1500                     goto trace0;
1501
1502                   if (PREDICT_FALSE (!s0))
1503                     goto trace0;
1504                 }
1505               else
1506                 {
1507                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1508                   goto trace0;
1509                 }
1510             }
1511           else
1512             {
1513               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1514             }
1515
1516           b0->flags |= VNET_BUFFER_F_IS_NATED;
1517
1518           if (!is_output_feature)
1519             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1520
1521           old_addr0 = ip0->src_address.as_u32;
1522           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1523           sum0 = ip0->checksum;
1524           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1525                                  src_address);
1526           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1527             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1528                                    s0->ext_host_addr.as_u32, ip4_header_t,
1529                                    dst_address);
1530           ip0->checksum = ip_csum_fold (sum0);
1531
1532           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1533             {
1534               old_port0 = tcp0->src_port;
1535               new_port0 = tcp0->src_port = s0->out2in.port;
1536
1537               sum0 = tcp0->checksum;
1538               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1539                                      dst_address);
1540               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1541                                      length);
1542               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1543                 {
1544                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1545                                          s0->ext_host_addr.as_u32,
1546                                          ip4_header_t, dst_address);
1547                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1548                                          s0->ext_host_port, ip4_header_t,
1549                                          length);
1550                   tcp0->dst_port = s0->ext_host_port;
1551                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1552                 }
1553               mss_clamping (sm, tcp0, &sum0);
1554               tcp0->checksum = ip_csum_fold (sum0);
1555               if (nat44_set_tcp_session_state_i2o
1556                   (sm, s0, tcp0, thread_index))
1557                 goto trace0;
1558             }
1559           else
1560             {
1561               udp0->src_port = s0->out2in.port;
1562               udp0->checksum = 0;
1563               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1564                 {
1565                   udp0->dst_port = s0->ext_host_port;
1566                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1567                 }
1568             }
1569
1570           /* Accounting */
1571           nat44_session_update_counters (s0, now,
1572                                          vlib_buffer_length_in_chain (vm,
1573                                                                       b0));
1574           /* Per-user LRU list maintenance */
1575           nat44_session_update_lru (sm, s0, thread_index);
1576
1577         trace0:
1578           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1579                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1580             {
1581               nat_in2out_ed_trace_t *t =
1582                 vlib_add_trace (vm, node, b0, sizeof (*t));
1583               t->is_slow_path = is_slow_path;
1584               t->sw_if_index = sw_if_index0;
1585               t->next_index = next0;
1586               t->session_index = ~0;
1587               if (s0)
1588                 t->session_index = s0 - tsm->sessions;
1589             }
1590
1591           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1592
1593           /* verify speculative enqueue, maybe switch current next frame */
1594           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1595                                            to_next, n_left_to_next,
1596                                            bi0, next0);
1597         }
1598
1599       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1600     }
1601
1602   vlib_node_increment_counter (vm, stats_node_index,
1603                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1604                                pkts_processed);
1605   return frame->n_vectors;
1606 }
1607
1608 static uword
1609 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1610                               vlib_node_runtime_t * node,
1611                               vlib_frame_t * frame)
1612 {
1613   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1614 }
1615
1616 /* *INDENT-OFF* */
1617 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1618   .function = nat44_ed_in2out_fast_path_fn,
1619   .name = "nat44-ed-in2out",
1620   .vector_size = sizeof (u32),
1621   .format_trace = format_nat_in2out_ed_trace,
1622   .type = VLIB_NODE_TYPE_INTERNAL,
1623   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1624   .error_strings = nat_in2out_ed_error_strings,
1625   .runtime_data_bytes = sizeof (snat_runtime_t),
1626   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1627   .next_nodes = {
1628     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1629     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1630     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1631     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1632     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1633   },
1634 };
1635 /* *INDENT-ON* */
1636
1637 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1638                               nat44_ed_in2out_fast_path_fn);
1639
1640 static uword
1641 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1642                                      vlib_node_runtime_t * node,
1643                                      vlib_frame_t * frame)
1644 {
1645   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1646 }
1647
1648 /* *INDENT-OFF* */
1649 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1650   .function = nat44_ed_in2out_output_fast_path_fn,
1651   .name = "nat44-ed-in2out-output",
1652   .vector_size = sizeof (u32),
1653   .format_trace = format_nat_in2out_ed_trace,
1654   .type = VLIB_NODE_TYPE_INTERNAL,
1655   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1656   .error_strings = nat_in2out_ed_error_strings,
1657   .runtime_data_bytes = sizeof (snat_runtime_t),
1658   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1659   .next_nodes = {
1660     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1661     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1662     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1663     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1664     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1665   },
1666 };
1667 /* *INDENT-ON* */
1668
1669 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1670                               nat44_ed_in2out_output_fast_path_fn);
1671
1672 static uword
1673 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1674                               vlib_node_runtime_t * node,
1675                               vlib_frame_t * frame)
1676 {
1677   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1678 }
1679
1680 /* *INDENT-OFF* */
1681 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1682   .function = nat44_ed_in2out_slow_path_fn,
1683   .name = "nat44-ed-in2out-slowpath",
1684   .vector_size = sizeof (u32),
1685   .format_trace = format_nat_in2out_ed_trace,
1686   .type = VLIB_NODE_TYPE_INTERNAL,
1687   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1688   .error_strings = nat_in2out_ed_error_strings,
1689   .runtime_data_bytes = sizeof (snat_runtime_t),
1690   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1691   .next_nodes = {
1692     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1693     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1694     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1695     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1696     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1697   },
1698 };
1699 /* *INDENT-ON* */
1700
1701 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1702                               nat44_ed_in2out_slow_path_fn);
1703
1704 static uword
1705 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1706                                      vlib_node_runtime_t * node,
1707                                      vlib_frame_t * frame)
1708 {
1709   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1710 }
1711
1712 /* *INDENT-OFF* */
1713 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1714   .function = nat44_ed_in2out_output_slow_path_fn,
1715   .name = "nat44-ed-in2out-output-slowpath",
1716   .vector_size = sizeof (u32),
1717   .format_trace = format_nat_in2out_ed_trace,
1718   .type = VLIB_NODE_TYPE_INTERNAL,
1719   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1720   .error_strings = nat_in2out_ed_error_strings,
1721   .runtime_data_bytes = sizeof (snat_runtime_t),
1722   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1723   .next_nodes = {
1724     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1725     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1726     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1727     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1728     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1729   },
1730 };
1731 /* *INDENT-ON* */
1732
1733 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1734                               nat44_ed_in2out_output_slow_path_fn);
1735
1736 static inline uword
1737 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1738                                       vlib_node_runtime_t * node,
1739                                       vlib_frame_t * frame,
1740                                       int is_output_feature)
1741 {
1742   u32 n_left_from, *from, *to_next;
1743   nat_in2out_ed_next_t next_index;
1744   u32 pkts_processed = 0;
1745   snat_main_t *sm = &snat_main;
1746   f64 now = vlib_time_now (vm);
1747   u32 thread_index = vm->thread_index;
1748   snat_main_per_thread_data_t *per_thread_data =
1749     &sm->per_thread_data[thread_index];
1750   u32 *fragments_to_drop = 0;
1751   u32 *fragments_to_loopback = 0;
1752
1753   from = vlib_frame_vector_args (frame);
1754   n_left_from = frame->n_vectors;
1755   next_index = node->cached_next_index;
1756
1757   while (n_left_from > 0)
1758     {
1759       u32 n_left_to_next;
1760
1761       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1762
1763       while (n_left_from > 0 && n_left_to_next > 0)
1764         {
1765           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1766           u32 iph_offset0 = 0;
1767           vlib_buffer_t *b0;
1768           u32 next0;
1769           u8 cached0 = 0;
1770           ip4_header_t *ip0 = 0;
1771           nat_reass_ip4_t *reass0;
1772           udp_header_t *udp0;
1773           tcp_header_t *tcp0;
1774           icmp46_header_t *icmp0;
1775           clib_bihash_kv_16_8_t kv0, value0;
1776           snat_session_t *s0 = 0;
1777           u16 old_port0, new_port0;
1778           ip_csum_t sum0;
1779
1780           /* speculatively enqueue b0 to the current next frame */
1781           bi0 = from[0];
1782           to_next[0] = bi0;
1783           from += 1;
1784           to_next += 1;
1785           n_left_from -= 1;
1786           n_left_to_next -= 1;
1787
1788           b0 = vlib_get_buffer (vm, bi0);
1789
1790           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1791
1792           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1793           rx_fib_index0 =
1794             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1795                                                  sw_if_index0);
1796
1797           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1798             {
1799               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1800               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1801               goto trace0;
1802             }
1803
1804           if (is_output_feature)
1805             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1806
1807           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1808                                   iph_offset0);
1809
1810           udp0 = ip4_next_header (ip0);
1811           tcp0 = (tcp_header_t *) udp0;
1812           icmp0 = (icmp46_header_t *) udp0;
1813           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1814
1815           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1816                                                  ip0->dst_address,
1817                                                  ip0->fragment_id,
1818                                                  ip0->protocol,
1819                                                  1, &fragments_to_drop);
1820
1821           if (PREDICT_FALSE (!reass0))
1822             {
1823               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1824               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1825               nat_log_notice ("maximum reassemblies exceeded");
1826               goto trace0;
1827             }
1828
1829           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1830             {
1831               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1832                 {
1833                   if (is_output_feature)
1834                     {
1835                       if (PREDICT_FALSE
1836                           (nat_not_translate_output_feature_fwd
1837                            (sm, ip0, thread_index, now, vm, b0)))
1838                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1839                       goto trace0;
1840                     }
1841
1842                   next0 = icmp_in2out_ed_slow_path
1843                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1844                      next0, now, thread_index, &s0);
1845
1846                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1847                     {
1848                       if (s0)
1849                         reass0->sess_index = s0 - per_thread_data->sessions;
1850                       else
1851                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1852                       nat_ip4_reass_get_frags (reass0,
1853                                                &fragments_to_loopback);
1854                     }
1855
1856                   goto trace0;
1857                 }
1858
1859               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1860                           ip0->protocol, rx_fib_index0, udp0->src_port,
1861                           udp0->dst_port);
1862
1863               if (clib_bihash_search_16_8
1864                   (&per_thread_data->in2out_ed, &kv0, &value0))
1865                 {
1866                   if (is_output_feature)
1867                     {
1868                       if (PREDICT_FALSE
1869                           (nat44_ed_not_translate_output_feature
1870                            (sm, ip0, ip0->protocol, udp0->src_port,
1871                             udp0->dst_port, thread_index, sw_if_index0,
1872                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1873                         {
1874                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1875                           nat_ip4_reass_get_frags (reass0,
1876                                                    &fragments_to_loopback);
1877                           goto trace0;
1878                         }
1879                     }
1880                   else
1881                     {
1882                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1883                                                                  sw_if_index0,
1884                                                                  ip0, proto0,
1885                                                                  rx_fib_index0,
1886                                                                  thread_index)))
1887                         {
1888                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1889                           nat_ip4_reass_get_frags (reass0,
1890                                                    &fragments_to_loopback);
1891                           goto trace0;
1892                         }
1893                     }
1894
1895                   if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1896                     {
1897                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
1898                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1899                       goto trace0;
1900                     }
1901
1902                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1903                                         &s0, node, next0, thread_index, now);
1904
1905                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1906                     goto trace0;
1907
1908                   if (PREDICT_FALSE (!s0))
1909                     {
1910                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1911                       goto trace0;
1912                     }
1913
1914                   reass0->sess_index = s0 - per_thread_data->sessions;
1915                 }
1916               else
1917                 {
1918                   s0 = pool_elt_at_index (per_thread_data->sessions,
1919                                           value0.value);
1920                   reass0->sess_index = value0.value;
1921                 }
1922               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1923             }
1924           else
1925             {
1926               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1927                 goto trace0;
1928               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1929                 {
1930                   if (nat_ip4_reass_add_fragment
1931                       (reass0, bi0, &fragments_to_drop))
1932                     {
1933                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1934                       nat_log_notice
1935                         ("maximum fragments per reassembly exceeded");
1936                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1937                       goto trace0;
1938                     }
1939                   cached0 = 1;
1940                   goto trace0;
1941                 }
1942               s0 = pool_elt_at_index (per_thread_data->sessions,
1943                                       reass0->sess_index);
1944             }
1945
1946           old_addr0 = ip0->src_address.as_u32;
1947           ip0->src_address = s0->out2in.addr;
1948           new_addr0 = ip0->src_address.as_u32;
1949           if (!is_output_feature)
1950             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1951
1952           sum0 = ip0->checksum;
1953           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1954                                  ip4_header_t,
1955                                  src_address /* changed member */ );
1956           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1957             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1958                                    s0->ext_host_addr.as_u32, ip4_header_t,
1959                                    dst_address);
1960           ip0->checksum = ip_csum_fold (sum0);
1961
1962           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1963             {
1964               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1965                 {
1966                   old_port0 = tcp0->src_port;
1967                   tcp0->src_port = s0->out2in.port;
1968                   new_port0 = tcp0->src_port;
1969
1970                   sum0 = tcp0->checksum;
1971                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1972                                          ip4_header_t,
1973                                          dst_address /* changed member */ );
1974                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1975                                          ip4_header_t /* cheat */ ,
1976                                          length /* changed member */ );
1977                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1978                     {
1979                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1980                                              s0->ext_host_addr.as_u32,
1981                                              ip4_header_t, dst_address);
1982                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1983                                              s0->ext_host_port, ip4_header_t,
1984                                              length);
1985                       tcp0->dst_port = s0->ext_host_port;
1986                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1987                     }
1988                   tcp0->checksum = ip_csum_fold (sum0);
1989                 }
1990               else
1991                 {
1992                   old_port0 = udp0->src_port;
1993                   udp0->src_port = s0->out2in.port;
1994                   udp0->checksum = 0;
1995                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1996                     {
1997                       udp0->dst_port = s0->ext_host_port;
1998                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1999                     }
2000                 }
2001             }
2002
2003           /* Hairpinning */
2004           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2005                                    s0->ext_host_port, proto0, 1);
2006
2007           /* Accounting */
2008           nat44_session_update_counters (s0, now,
2009                                          vlib_buffer_length_in_chain (vm,
2010                                                                       b0));
2011           /* Per-user LRU list maintenance */
2012           nat44_session_update_lru (sm, s0, thread_index);
2013
2014         trace0:
2015           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2016                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2017             {
2018               nat44_reass_trace_t *t =
2019                 vlib_add_trace (vm, node, b0, sizeof (*t));
2020               t->cached = cached0;
2021               t->sw_if_index = sw_if_index0;
2022               t->next_index = next0;
2023             }
2024
2025           if (cached0)
2026             {
2027               n_left_to_next++;
2028               to_next--;
2029             }
2030           else
2031             {
2032               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
2033
2034               /* verify speculative enqueue, maybe switch current next frame */
2035               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2036                                                to_next, n_left_to_next,
2037                                                bi0, next0);
2038             }
2039
2040           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2041             {
2042               from = vlib_frame_vector_args (frame);
2043               u32 len = vec_len (fragments_to_loopback);
2044               if (len <= VLIB_FRAME_SIZE)
2045                 {
2046                   clib_memcpy_fast (from, fragments_to_loopback,
2047                                     sizeof (u32) * len);
2048                   n_left_from = len;
2049                   vec_reset_length (fragments_to_loopback);
2050                 }
2051               else
2052                 {
2053                   clib_memcpy_fast (from, fragments_to_loopback +
2054                                     (len - VLIB_FRAME_SIZE),
2055                                     sizeof (u32) * VLIB_FRAME_SIZE);
2056                   n_left_from = VLIB_FRAME_SIZE;
2057                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2058                 }
2059             }
2060         }
2061
2062       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2063     }
2064
2065   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
2066                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
2067                                pkts_processed);
2068
2069   nat_send_all_to_node (vm, fragments_to_drop, node,
2070                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
2071                         NAT_IN2OUT_ED_NEXT_DROP);
2072
2073   vec_free (fragments_to_drop);
2074   vec_free (fragments_to_loopback);
2075   return frame->n_vectors;
2076 }
2077
2078 static uword
2079 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
2080                                vlib_node_runtime_t * node,
2081                                vlib_frame_t * frame)
2082 {
2083   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2084 }
2085
2086 /* *INDENT-OFF* */
2087 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2088   .function = nat44_ed_in2out_reass_node_fn,
2089   .name = "nat44-ed-in2out-reass",
2090   .vector_size = sizeof (u32),
2091   .format_trace = format_nat44_reass_trace,
2092   .type = VLIB_NODE_TYPE_INTERNAL,
2093   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2094   .error_strings = nat_in2out_ed_error_strings,
2095   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2096   .next_nodes = {
2097     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2098     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2099     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2100     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2101     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2102   },
2103 };
2104 /* *INDENT-ON* */
2105
2106 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2107                               nat44_ed_in2out_reass_node_fn);
2108
2109 static uword
2110 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2111                                       vlib_node_runtime_t * node,
2112                                       vlib_frame_t * frame)
2113 {
2114   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2115 }
2116
2117 /* *INDENT-OFF* */
2118 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2119   .function = nat44_ed_in2out_reass_output_node_fn,
2120   .name = "nat44-ed-in2out-reass-output",
2121   .vector_size = sizeof (u32),
2122   .format_trace = format_nat44_reass_trace,
2123   .type = VLIB_NODE_TYPE_INTERNAL,
2124   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2125   .error_strings = nat_in2out_ed_error_strings,
2126   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2127   .next_nodes = {
2128     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2129     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2130     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2131     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2132     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2133   },
2134 };
2135 /* *INDENT-ON* */
2136
2137 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2138                               nat44_ed_in2out_reass_output_node_fn);
2139
2140 /*
2141  * fd.io coding-style-patch-verification: ON
2142  *
2143  * Local Variables:
2144  * eval: (c-set-style "gnu")
2145  * End:
2146  */