src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_syslog.h>
  32
  33 #define foreach_nat_in2out_ed_error                     \
  34 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  35 _(IN2OUT_PACKETS, "good in2out packets processed")      \
  36 _(OUT_OF_PORTS, "out of ports")                         \
  37 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  38 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  39 _(DROP_FRAGMENT, "drop fragment")                       \
  40 _(MAX_REASS, "maximum reassemblies exceeded")           \
  41 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  42 _(NON_SYN, "non-SYN packet try to create session")      \
  43 _(TCP_PACKETS, "TCP packets")                           \
  44 _(UDP_PACKETS, "UDP packets")                           \
  45 _(ICMP_PACKETS, "ICMP packets")                         \
  46 _(OTHER_PACKETS, "other protocol packets")              \
  47 _(FRAGMENTS, "fragments")                               \
  48 _(CACHED_FRAGMENTS, "cached fragments")                 \
  49 _(PROCESSED_FRAGMENTS, "processed fragments")
  50
  51
  52 typedef enum
  53 {
  54 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
  55   foreach_nat_in2out_ed_error
  56 #undef _
  57     NAT_IN2OUT_ED_N_ERROR,
  58 } nat_in2out_ed_error_t;
  59
  60 static char *nat_in2out_ed_error_strings[] = {
  61 #define _(sym,string) string,
  62   foreach_nat_in2out_ed_error
  63 #undef _
  64 };
  65
  66 typedef enum
  67 {
  68   NAT_IN2OUT_ED_NEXT_LOOKUP,
  69   NAT_IN2OUT_ED_NEXT_DROP,
  70   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
  71   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
  72   NAT_IN2OUT_ED_NEXT_REASS,
  73   NAT_IN2OUT_ED_N_NEXT,
  74 } nat_in2out_ed_next_t;
  75
  76 typedef struct
  77 {
  78   u32 sw_if_index;
  79   u32 next_index;
  80   u32 session_index;
  81   u32 is_slow_path;
  82 } nat_in2out_ed_trace_t;
  83
  84 vlib_node_registration_t nat44_ed_in2out_node;
  85 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
  86 vlib_node_registration_t nat44_ed_in2out_output_node;
  87 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
  88 vlib_node_registration_t nat44_ed_in2out_reass_node;
  89
  90 static u8 *
  91 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  92 {
  93   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  94   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  95   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  96   char *tag;
  97
  98   tag =
  99     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
 100     "NAT44_IN2OUT_ED_FAST_PATH";
 101
 102   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
 103               t->sw_if_index, t->next_index, t->session_index);
 104
 105   return s;
 106 }
 107
 108 static_always_inline int
 109 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
 110 {
 111   icmp46_header_t *icmp0;
 112   nat_ed_ses_key_t key0;
 113   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 114   ip4_header_t *inner_ip0 = 0;
 115   void *l4_header = 0;
 116   icmp46_header_t *inner_icmp0;
 117
 118   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 119   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 120
 121   if (!icmp_is_error_message (icmp0))
 122     {
 123       key0.proto = IP_PROTOCOL_ICMP;
 124       key0.l_addr = ip0->src_address;
 125       key0.r_addr = ip0->dst_address;
 126       key0.l_port = echo0->identifier;
 127       key0.r_port = 0;
 128     }
 129   else
 130     {
 131       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 132       l4_header = ip4_next_header (inner_ip0);
 133       key0.proto = inner_ip0->protocol;
 134       key0.r_addr = inner_ip0->src_address;
 135       key0.l_addr = inner_ip0->dst_address;
 136       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 137         {
 138         case SNAT_PROTOCOL_ICMP:
 139           inner_icmp0 = (icmp46_header_t *) l4_header;
 140           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 141           key0.r_port = 0;
 142           key0.l_port = inner_echo0->identifier;
 143           break;
 144         case SNAT_PROTOCOL_UDP:
 145         case SNAT_PROTOCOL_TCP:
 146           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
 147           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
 148           break;
 149         default:
 150           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
 151         }
 152     }
 153   *p_key0 = key0;
 154   return 0;
 155 }
 156
 157 int
 158 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
 159 {
 160   snat_main_t *sm = &snat_main;
 161   nat44_is_idle_session_ctx_t *ctx = arg;
 162   snat_session_t *s;
 163   u64 sess_timeout_time;
 164   nat_ed_ses_key_t ed_key;
 165   clib_bihash_kv_16_8_t ed_kv;
 166   int i;
 167   snat_address_t *a;
 168   snat_session_key_t key;
 169   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 170                                                        ctx->thread_index);
 171
 172   s = pool_elt_at_index (tsm->sessions, kv->value);
 173   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 174   if (ctx->now >= sess_timeout_time)
 175     {
 176       if (is_fwd_bypass_session (s))
 177         goto delete;
 178
 179       ed_key.l_addr = s->out2in.addr;
 180       ed_key.r_addr = s->ext_host_addr;
 181       ed_key.fib_index = s->out2in.fib_index;
 182       if (snat_is_unk_proto_session (s))
 183         {
 184           ed_key.proto = s->in2out.port;
 185           ed_key.r_port = 0;
 186           ed_key.l_port = 0;
 187         }
 188       else
 189         {
 190           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 191           ed_key.l_port = s->out2in.port;
 192           ed_key.r_port = s->ext_host_port;
 193         }
 194       ed_kv.key[0] = ed_key.as_u64[0];
 195       ed_kv.key[1] = ed_key.as_u64[1];
 196       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 197         nat_log_warn ("out2in_ed key del failed");
 198
 199       if (snat_is_unk_proto_session (s))
 200         goto delete;
 201
 202       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 203                                            s->in2out.addr.as_u32,
 204                                            s->out2in.addr.as_u32,
 205                                            s->in2out.protocol,
 206                                            s->in2out.port,
 207                                            s->out2in.port,
 208                                            s->in2out.fib_index);
 209
 210       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 211                              &s->in2out.addr, s->in2out.port,
 212                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 213                              &s->out2in.addr, s->out2in.port,
 214                              &s->ext_host_addr, s->ext_host_port,
 215                              s->in2out.protocol, is_twice_nat_session (s));
 216
 217       if (is_twice_nat_session (s))
 218         {
 219           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 220             {
 221               key.protocol = s->in2out.protocol;
 222               key.port = s->ext_host_nat_port;
 223               a = sm->twice_nat_addresses + i;
 224               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 225                 {
 226                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 227                                                       ctx->thread_index,
 228                                                       &key);
 229                   break;
 230                 }
 231             }
 232         }
 233
 234       if (snat_is_session_static (s))
 235         goto delete;
 236
 237       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 238                                           &s->out2in);
 239     delete:
 240       nat44_delete_session (sm, s, ctx->thread_index);
 241       return 1;
 242     }
 243
 244   return 0;
 245 }
 246
 247 static inline u32
 248 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 249                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 250                           u32 sw_if_index0, u32 rx_fib_index0,
 251                           vlib_node_runtime_t * node, u32 next0, f64 now,
 252                           u32 thread_index, snat_session_t ** p_s0)
 253 {
 254   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 255                        next0, thread_index, p_s0, 0);
 256   snat_session_t *s0 = *p_s0;
 257   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
 258     {
 259       /* Accounting */
 260       nat44_session_update_counters (s0, now,
 261                                      vlib_buffer_length_in_chain
 262                                      (sm->vlib_main, b0));
 263       /* Per-user LRU list maintenance */
 264       nat44_session_update_lru (sm, s0, thread_index);
 265     }
 266   return next0;
 267 }
 268
 269 static u32
 270 slow_path_ed (snat_main_t * sm,
 271               vlib_buffer_t * b,
 272               u32 rx_fib_index,
 273               clib_bihash_kv_16_8_t * kv,
 274               snat_session_t ** sessionp,
 275               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now,
 276               tcp_header_t * tcp)
 277 {
 278   snat_session_t *s = 0;
 279   snat_user_t *u;
 280   snat_session_key_t key0, key1;
 281   lb_nat_type_t lb = 0, is_sm = 0;
 282   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 283   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 284   u32 proto = ip_proto_to_snat_proto (key->proto);
 285   nat_outside_fib_t *outside_fib;
 286   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 287   u8 identity_nat;
 288   fib_prefix_t pfx = {
 289     .fp_proto = FIB_PROTOCOL_IP4,
 290     .fp_len = 32,
 291     .fp_addr = {
 292                 .ip4.as_u32 = key->r_addr.as_u32,
 293                 },
 294   };
 295   nat44_is_idle_session_ctx_t ctx;
 296
 297   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 298     {
 299       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 300       nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 301       nat_log_notice ("maximum sessions exceeded");
 302       return NAT_IN2OUT_ED_NEXT_DROP;
 303     }
 304
 305   key0.addr = key->l_addr;
 306   key0.port = key->l_port;
 307   key1.protocol = key0.protocol = proto;
 308   key0.fib_index = rx_fib_index;
 309   key1.fib_index = sm->outside_fib_index;
 310   /* First try to match static mapping by local address and port */
 311   if (snat_static_mapping_match
 312       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 313     {
 314       /* Try to create dynamic translation */
 315       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 316                                                thread_index, &key1,
 317                                                sm->port_per_thread,
 318                                                tsm->snat_thread_index))
 319         {
 320           nat_log_notice ("addresses exhausted");
 321           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 322           return NAT_IN2OUT_ED_NEXT_DROP;
 323         }
 324     }
 325   else
 326     {
 327       if (PREDICT_FALSE (identity_nat))
 328         {
 329           *sessionp = s;
 330           return next;
 331         }
 332
 333       is_sm = 1;
 334     }
 335
 336   if (proto == SNAT_PROTOCOL_TCP)
 337     {
 338       if (!tcp_is_init (tcp))
 339         {
 340           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 341           return NAT_IN2OUT_ED_NEXT_DROP;
 342         }
 343     }
 344
 345   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 346   if (!u)
 347     {
 348       nat_log_warn ("create NAT user failed");
 349       if (!is_sm)
 350         snat_free_outside_address_and_port (sm->addresses,
 351                                             thread_index, &key1);
 352       return NAT_IN2OUT_ED_NEXT_DROP;
 353     }
 354
 355   s = nat_ed_session_alloc (sm, u, thread_index, now);
 356   if (!s)
 357     {
 358       nat44_delete_user_with_no_session (sm, u, thread_index);
 359       nat_log_warn ("create NAT session failed");
 360       if (!is_sm)
 361         snat_free_outside_address_and_port (sm->addresses,
 362                                             thread_index, &key1);
 363       return NAT_IN2OUT_ED_NEXT_DROP;
 364     }
 365
 366   user_session_increment (sm, u, is_sm);
 367   if (is_sm)
 368     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 369   if (lb)
 370     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 371   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 372   s->ext_host_addr = key->r_addr;
 373   s->ext_host_port = key->r_port;
 374   s->in2out = key0;
 375   s->out2in = key1;
 376   s->out2in.protocol = key0.protocol;
 377
 378   switch (vec_len (sm->outside_fibs))
 379     {
 380     case 0:
 381       s->out2in.fib_index = sm->outside_fib_index;
 382       break;
 383     case 1:
 384       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 385       break;
 386     default:
 387       /* *INDENT-OFF* */
 388       vec_foreach (outside_fib, sm->outside_fibs)
 389        {
 390           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 391           if (FIB_NODE_INDEX_INVALID != fei)
 392             {
 393               if (fib_entry_get_resolving_interface (fei) != ~0)
 394                 {
 395                   s->out2in.fib_index = outside_fib->fib_index;
 396                   break;
 397                 }
 398             }
 399         }
 400       /* *INDENT-ON* */
 401       break;
 402     }
 403
 404   /* Add to lookup tables */
 405   kv->value = s - tsm->sessions;
 406   ctx.now = now;
 407   ctx.thread_index = thread_index;
 408   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 409                                                nat44_i2o_ed_is_idle_session_cb,
 410                                                &ctx))
 411     nat_log_notice ("in2out-ed key add failed");
 412
 413   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 414               key1.port, key->r_port);
 415   kv->value = s - tsm->sessions;
 416   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 417                                                nat44_o2i_ed_is_idle_session_cb,
 418                                                &ctx))
 419     nat_log_notice ("out2in-ed key add failed");
 420
 421   *sessionp = s;
 422
 423   /* log NAT event */
 424   snat_ipfix_logging_nat44_ses_create (thread_index,
 425                                        s->in2out.addr.as_u32,
 426                                        s->out2in.addr.as_u32,
 427                                        s->in2out.protocol,
 428                                        s->in2out.port,
 429                                        s->out2in.port, s->in2out.fib_index);
 430
 431   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 432                          &s->in2out.addr, s->in2out.port,
 433                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 434                          &s->out2in.addr, s->out2in.port,
 435                          &s->ext_host_addr, s->ext_host_port,
 436                          s->in2out.protocol, 0);
 437
 438   return next;
 439 }
 440
 441 static_always_inline int
 442 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 443                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 444                         u32 rx_fib_index, u32 thread_index)
 445 {
 446   udp_header_t *udp = ip4_next_header (ip);
 447   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 448   clib_bihash_kv_16_8_t kv, value;
 449   snat_session_key_t key0, key1;
 450
 451   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 452               sm->outside_fib_index, udp->dst_port, udp->src_port);
 453
 454   /* NAT packet aimed at external address if */
 455   /* has active sessions */
 456   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 457     {
 458       key0.addr = ip->dst_address;
 459       key0.port = udp->dst_port;
 460       key0.protocol = proto;
 461       key0.fib_index = sm->outside_fib_index;
 462       /* or is static mappings */
 463       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 464         return 0;
 465     }
 466   else
 467     return 0;
 468
 469   if (sm->forwarding_enabled)
 470     return 1;
 471
 472   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 473                                   rx_fib_index);
 474 }
 475
 476 static_always_inline int
 477 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 478                                       u32 thread_index, f64 now,
 479                                       vlib_main_t * vm, vlib_buffer_t * b)
 480 {
 481   nat_ed_ses_key_t key;
 482   clib_bihash_kv_16_8_t kv, value;
 483   udp_header_t *udp;
 484   snat_session_t *s = 0;
 485   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 486
 487   if (!sm->forwarding_enabled)
 488     return 0;
 489
 490   if (ip->protocol == IP_PROTOCOL_ICMP)
 491     {
 492       key.as_u64[0] = key.as_u64[1] = 0;
 493       if (icmp_get_ed_key (ip, &key))
 494         return 0;
 495       key.fib_index = 0;
 496       kv.key[0] = key.as_u64[0];
 497       kv.key[1] = key.as_u64[1];
 498     }
 499   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 500     {
 501       udp = ip4_next_header (ip);
 502       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 503                   udp->src_port, udp->dst_port);
 504     }
 505   else
 506     {
 507       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 508                   0);
 509     }
 510
 511   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 512     {
 513       s = pool_elt_at_index (tsm->sessions, value.value);
 514       if (is_fwd_bypass_session (s))
 515         {
 516           if (ip->protocol == IP_PROTOCOL_TCP)
 517             {
 518               tcp_header_t *tcp = ip4_next_header (ip);
 519               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
 520                 return 1;
 521             }
 522           /* Accounting */
 523           nat44_session_update_counters (s, now,
 524                                          vlib_buffer_length_in_chain (vm, b));
 525           /* Per-user LRU list maintenance */
 526           nat44_session_update_lru (sm, s, thread_index);
 527           return 1;
 528         }
 529       else
 530         return 0;
 531     }
 532
 533   return 0;
 534 }
 535
 536 static_always_inline int
 537 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 538                                        u8 proto, u16 src_port, u16 dst_port,
 539                                        u32 thread_index, u32 rx_sw_if_index,
 540                                        u32 tx_sw_if_index)
 541 {
 542   clib_bihash_kv_16_8_t kv, value;
 543   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 544   snat_interface_t *i;
 545   snat_session_t *s;
 546   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 547   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 548
 549   /* src NAT check */
 550   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 551               src_port, dst_port);
 552   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 553     {
 554       s = pool_elt_at_index (tsm->sessions, value.value);
 555       if (nat44_is_ses_closed (s))
 556         {
 557           nat_log_debug ("TCP close connection %U", format_snat_session,
 558                          &sm->per_thread_data[thread_index], s);
 559           nat_free_session_data (sm, s, thread_index);
 560           nat44_delete_session (sm, s, thread_index);
 561         }
 562       else
 563         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 564       return 1;
 565     }
 566
 567   /* dst NAT check */
 568   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 569               dst_port, src_port);
 570   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 571     {
 572       s = pool_elt_at_index (tsm->sessions, value.value);
 573       if (is_fwd_bypass_session (s))
 574         return 0;
 575
 576       /* hairpinning */
 577       /* *INDENT-OFF* */
 578       pool_foreach (i, sm->output_feature_interfaces,
 579       ({
 580         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 581            return 0;
 582       }));
 583       /* *INDENT-ON* */
 584       return 1;
 585     }
 586
 587   return 0;
 588 }
 589
 590 u32
 591 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 592                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 593                       u8 * p_proto, snat_session_key_t * p_value,
 594                       u8 * p_dont_translate, void *d, void *e)
 595 {
 596   icmp46_header_t *icmp;
 597   u32 sw_if_index;
 598   u32 rx_fib_index;
 599   nat_ed_ses_key_t key;
 600   snat_session_t *s = 0;
 601   u8 dont_translate = 0;
 602   clib_bihash_kv_16_8_t kv, value;
 603   u32 next = ~0;
 604   int err;
 605   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 606
 607   icmp = (icmp46_header_t *) ip4_next_header (ip);
 608   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 609   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 610
 611   key.as_u64[0] = key.as_u64[1] = 0;
 612   err = icmp_get_ed_key (ip, &key);
 613   if (err != 0)
 614     {
 615       b->error = node->errors[err];
 616       next = NAT_IN2OUT_ED_NEXT_DROP;
 617       goto out;
 618     }
 619   key.fib_index = rx_fib_index;
 620
 621   kv.key[0] = key.as_u64[0];
 622   kv.key[1] = key.as_u64[1];
 623
 624   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 625     {
 626       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 627         {
 628           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
 629                                                                     key.proto,
 630                                                                     key.
 631                                                                     l_port,
 632                                                                     key.
 633                                                                     r_port,
 634                                                                     thread_index,
 635                                                                     sw_if_index,
 636                                                                     vnet_buffer
 637                                                                     (b)->
 638                                                                     sw_if_index
 639                                                                     [VLIB_TX])))
 640             {
 641               dont_translate = 1;
 642               goto out;
 643             }
 644         }
 645       else
 646         {
 647           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 648                                                      ip, SNAT_PROTOCOL_ICMP,
 649                                                      rx_fib_index,
 650                                                      thread_index)))
 651             {
 652               dont_translate = 1;
 653               goto out;
 654             }
 655         }
 656
 657       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
 658         {
 659           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 660           next = NAT_IN2OUT_ED_NEXT_DROP;
 661           goto out;
 662         }
 663
 664       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 665                            thread_index, vlib_time_now (sm->vlib_main), 0);
 666
 667       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
 668         goto out;
 669
 670       if (!s)
 671         {
 672           dont_translate = 1;
 673           goto out;
 674         }
 675     }
 676   else
 677     {
 678       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
 679                          icmp->type != ICMP4_echo_reply &&
 680                          !icmp_is_error_message (icmp)))
 681         {
 682           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 683           next = NAT_IN2OUT_ED_NEXT_DROP;
 684           goto out;
 685         }
 686
 687       s = pool_elt_at_index (tsm->sessions, value.value);
 688     }
 689
 690   *p_proto = ip_proto_to_snat_proto (key.proto);
 691 out:
 692   if (s)
 693     *p_value = s->out2in;
 694   *p_dont_translate = dont_translate;
 695   if (d)
 696     *(snat_session_t **) d = s;
 697   return next;
 698 }
 699
 700 static snat_session_t *
 701 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 702                                vlib_buffer_t * b,
 703                                ip4_header_t * ip,
 704                                u32 rx_fib_index,
 705                                u32 thread_index,
 706                                f64 now,
 707                                vlib_main_t * vm, vlib_node_runtime_t * node)
 708 {
 709   clib_bihash_kv_8_8_t kv, value;
 710   clib_bihash_kv_16_8_t s_kv, s_value;
 711   snat_static_mapping_t *m;
 712   u32 old_addr, new_addr = 0;
 713   ip_csum_t sum;
 714   snat_user_t *u;
 715   dlist_elt_t *head, *elt;
 716   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 717   u32 elt_index, head_index, ses_index;
 718   snat_session_t *s;
 719   u32 outside_fib_index = sm->outside_fib_index;
 720   int i;
 721   u8 is_sm = 0;
 722   nat_outside_fib_t *outside_fib;
 723   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 724   fib_prefix_t pfx = {
 725     .fp_proto = FIB_PROTOCOL_IP4,
 726     .fp_len = 32,
 727     .fp_addr = {
 728                 .ip4.as_u32 = ip->dst_address.as_u32,
 729                 },
 730   };
 731
 732   switch (vec_len (sm->outside_fibs))
 733     {
 734     case 0:
 735       outside_fib_index = sm->outside_fib_index;
 736       break;
 737     case 1:
 738       outside_fib_index = sm->outside_fibs[0].fib_index;
 739       break;
 740     default:
 741       /* *INDENT-OFF* */
 742       vec_foreach (outside_fib, sm->outside_fibs)
 743         {
 744           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 745           if (FIB_NODE_INDEX_INVALID != fei)
 746             {
 747               if (fib_entry_get_resolving_interface (fei) != ~0)
 748                 {
 749                   outside_fib_index = outside_fib->fib_index;
 750                   break;
 751                 }
 752             }
 753         }
 754       /* *INDENT-ON* */
 755       break;
 756     }
 757   old_addr = ip->src_address.as_u32;
 758
 759   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 760               rx_fib_index, 0, 0);
 761
 762   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 763     {
 764       s = pool_elt_at_index (tsm->sessions, s_value.value);
 765       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 766     }
 767   else
 768     {
 769       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 770         {
 771           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 772           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 773           nat_log_notice ("maximum sessions exceeded");
 774           return 0;
 775         }
 776
 777       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 778                                   thread_index);
 779       if (!u)
 780         {
 781           nat_log_warn ("create NAT user failed");
 782           return 0;
 783         }
 784
 785       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 786
 787       /* Try to find static mapping first */
 788       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 789         {
 790           m = pool_elt_at_index (sm->static_mappings, value.value);
 791           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 792           is_sm = 1;
 793           goto create_ses;
 794         }
 795       /* Fallback to 3-tuple key */
 796       else
 797         {
 798           /* Choose same out address as for TCP/UDP session to same destination */
 799           head_index = u->sessions_per_user_list_head_index;
 800           head = pool_elt_at_index (tsm->list_pool, head_index);
 801           elt_index = head->next;
 802           if (PREDICT_FALSE (elt_index == ~0))
 803             ses_index = ~0;
 804           else
 805             {
 806               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 807               ses_index = elt->value;
 808             }
 809
 810           while (ses_index != ~0)
 811             {
 812               s = pool_elt_at_index (tsm->sessions, ses_index);
 813               elt_index = elt->next;
 814               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 815               ses_index = elt->value;
 816
 817               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 818                 {
 819                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 820
 821                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 822                               ip->protocol, outside_fib_index, 0, 0);
 823                   if (clib_bihash_search_16_8
 824                       (&tsm->out2in_ed, &s_kv, &s_value))
 825                     goto create_ses;
 826
 827                   break;
 828                 }
 829             }
 830
 831           for (i = 0; i < vec_len (sm->addresses); i++)
 832             {
 833               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 834                           ip->protocol, outside_fib_index, 0, 0);
 835               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 836                 {
 837                   new_addr = ip->src_address.as_u32 =
 838                     sm->addresses[i].addr.as_u32;
 839                   goto create_ses;
 840                 }
 841             }
 842           return 0;
 843         }
 844
 845     create_ses:
 846       s = nat_ed_session_alloc (sm, u, thread_index, now);
 847       if (!s)
 848         {
 849           nat44_delete_user_with_no_session (sm, u, thread_index);
 850           nat_log_warn ("create NAT session failed");
 851           return 0;
 852         }
 853
 854       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 855       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 856       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 857       s->out2in.addr.as_u32 = new_addr;
 858       s->out2in.fib_index = outside_fib_index;
 859       s->in2out.addr.as_u32 = old_addr;
 860       s->in2out.fib_index = rx_fib_index;
 861       s->in2out.port = s->out2in.port = ip->protocol;
 862       if (is_sm)
 863         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 864       user_session_increment (sm, u, is_sm);
 865
 866       /* Add to lookup tables */
 867       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 868                   rx_fib_index, 0, 0);
 869       s_kv.value = s - tsm->sessions;
 870       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 871         nat_log_notice ("in2out key add failed");
 872
 873       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 874                   outside_fib_index, 0, 0);
 875       s_kv.value = s - tsm->sessions;
 876       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 877         nat_log_notice ("out2in key add failed");
 878     }
 879
 880   /* Update IP checksum */
 881   sum = ip->checksum;
 882   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 883   ip->checksum = ip_csum_fold (sum);
 884
 885   /* Accounting */
 886   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
 887   /* Per-user LRU list maintenance */
 888   nat44_session_update_lru (sm, s, thread_index);
 889
 890   /* Hairpinning */
 891   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 892     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 893
 894   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 895     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 896
 897   return s;
 898 }
 899
 900 static inline uword
 901 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
 902                                 vlib_node_runtime_t * node,
 903                                 vlib_frame_t * frame, int is_slow_path,
 904                                 int is_output_feature)
 905 {
 906   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 907   nat_in2out_ed_next_t next_index;
 908   snat_main_t *sm = &snat_main;
 909   f64 now = vlib_time_now (vm);
 910   u32 thread_index = vm->thread_index;
 911   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 912   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 913     0, fragments = 0;
 914
 915   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
 916     nat44_ed_in2out_node.index;
 917
 918   from = vlib_frame_vector_args (frame);
 919   n_left_from = frame->n_vectors;
 920   next_index = node->cached_next_index;
 921
 922   while (n_left_from > 0)
 923     {
 924       u32 n_left_to_next;
 925
 926       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 927
 928       while (n_left_from >= 4 && n_left_to_next >= 2)
 929         {
 930           u32 bi0, bi1;
 931           vlib_buffer_t *b0, *b1;
 932           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 933             new_addr0, old_addr0;
 934           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
 935             new_addr1, old_addr1;
 936           u16 old_port0, new_port0, old_port1, new_port1;
 937           ip4_header_t *ip0, *ip1;
 938           udp_header_t *udp0, *udp1;
 939           tcp_header_t *tcp0, *tcp1;
 940           icmp46_header_t *icmp0, *icmp1;
 941           snat_session_t *s0 = 0, *s1 = 0;
 942           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
 943           ip_csum_t sum0, sum1;
 944
 945           /* Prefetch next iteration. */
 946           {
 947             vlib_buffer_t *p2, *p3;
 948
 949             p2 = vlib_get_buffer (vm, from[2]);
 950             p3 = vlib_get_buffer (vm, from[3]);
 951
 952             vlib_prefetch_buffer_header (p2, LOAD);
 953             vlib_prefetch_buffer_header (p3, LOAD);
 954
 955             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 956             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 957           }
 958
 959           /* speculatively enqueue b0 and b1 to the current next frame */
 960           to_next[0] = bi0 = from[0];
 961           to_next[1] = bi1 = from[1];
 962           from += 2;
 963           to_next += 2;
 964           n_left_from -= 2;
 965           n_left_to_next -= 2;
 966
 967           b0 = vlib_get_buffer (vm, bi0);
 968           b1 = vlib_get_buffer (vm, bi1);
 969
 970           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
 971
 972           if (is_output_feature)
 973             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 974
 975           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 976                                   iph_offset0);
 977
 978           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 979           rx_fib_index0 =
 980             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 981                                                  sw_if_index0);
 982
 983           if (PREDICT_FALSE (ip0->ttl == 1))
 984             {
 985               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 986               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 987                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 988                                            0);
 989               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
 990               goto trace00;
 991             }
 992
 993           udp0 = ip4_next_header (ip0);
 994           tcp0 = (tcp_header_t *) udp0;
 995           icmp0 = (icmp46_header_t *) udp0;
 996           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 997
 998           if (is_slow_path)
 999             {
1000               if (PREDICT_FALSE (proto0 == ~0))
1001                 {
1002                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1003                                                       rx_fib_index0,
1004                                                       thread_index, now, vm,
1005                                                       node);
1006                   if (!s0)
1007                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1008                   other_packets++;
1009                   goto trace00;
1010                 }
1011
1012               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1013                 {
1014                   next0 = icmp_in2out_ed_slow_path
1015                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1016                      next0, now, thread_index, &s0);
1017                   icmp_packets++;
1018                   goto trace00;
1019                 }
1020             }
1021           else
1022             {
1023               if (PREDICT_FALSE (proto0 == ~0))
1024                 {
1025                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1026                   goto trace00;
1027                 }
1028
1029               if (ip4_is_fragment (ip0))
1030                 {
1031                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1032                   fragments++;
1033                   goto trace00;
1034                 }
1035
1036               if (is_output_feature)
1037                 {
1038                   if (PREDICT_FALSE
1039                       (nat_not_translate_output_feature_fwd
1040                        (sm, ip0, thread_index, now, vm, b0)))
1041                     goto trace00;
1042                 }
1043
1044               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1045                 {
1046                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1047                   goto trace00;
1048                 }
1049             }
1050
1051           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1052                       ip0->protocol, rx_fib_index0, udp0->src_port,
1053                       udp0->dst_port);
1054
1055           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1056             {
1057               if (is_slow_path)
1058                 {
1059                   if (is_output_feature)
1060                     {
1061                       if (PREDICT_FALSE
1062                           (nat44_ed_not_translate_output_feature
1063                            (sm, ip0, ip0->protocol, udp0->src_port,
1064                             udp0->dst_port, thread_index, sw_if_index0,
1065                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1066                         goto trace00;
1067                     }
1068                   else
1069                     {
1070                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1071                                                                  sw_if_index0,
1072                                                                  ip0, proto0,
1073                                                                  rx_fib_index0,
1074                                                                  thread_index)))
1075                         goto trace00;
1076                     }
1077
1078                   next0 =
1079                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1080                                   next0, thread_index, now, tcp0);
1081
1082                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1083                     goto trace00;
1084
1085                   if (PREDICT_FALSE (!s0))
1086                     goto trace00;
1087                 }
1088               else
1089                 {
1090                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1091                   goto trace00;
1092                 }
1093             }
1094           else
1095             {
1096               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1097             }
1098
1099           b0->flags |= VNET_BUFFER_F_IS_NATED;
1100
1101           if (!is_output_feature)
1102             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1103
1104           old_addr0 = ip0->src_address.as_u32;
1105           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1106           sum0 = ip0->checksum;
1107           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1108                                  src_address);
1109           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1110             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1111                                    s0->ext_host_addr.as_u32, ip4_header_t,
1112                                    dst_address);
1113           ip0->checksum = ip_csum_fold (sum0);
1114
1115           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1116             {
1117               old_port0 = tcp0->src_port;
1118               new_port0 = tcp0->src_port = s0->out2in.port;
1119
1120               sum0 = tcp0->checksum;
1121               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1122                                      dst_address);
1123               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1124                                      length);
1125               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1126                 {
1127                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1128                                          s0->ext_host_addr.as_u32,
1129                                          ip4_header_t, dst_address);
1130                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1131                                          s0->ext_host_port, ip4_header_t,
1132                                          length);
1133                   tcp0->dst_port = s0->ext_host_port;
1134                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1135                 }
1136               mss_clamping (sm, tcp0, &sum0);
1137               tcp0->checksum = ip_csum_fold (sum0);
1138               tcp_packets++;
1139               if (nat44_set_tcp_session_state_i2o
1140                   (sm, s0, tcp0, thread_index))
1141                 goto trace00;
1142             }
1143           else
1144             {
1145               udp0->src_port = s0->out2in.port;
1146               udp0->checksum = 0;
1147               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1148                 {
1149                   udp0->dst_port = s0->ext_host_port;
1150                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1151                 }
1152               udp_packets++;
1153             }
1154
1155           /* Accounting */
1156           nat44_session_update_counters (s0, now,
1157                                          vlib_buffer_length_in_chain (vm,
1158                                                                       b0));
1159           /* Per-user LRU list maintenance */
1160           nat44_session_update_lru (sm, s0, thread_index);
1161
1162         trace00:
1163           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1164                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1165             {
1166               nat_in2out_ed_trace_t *t =
1167                 vlib_add_trace (vm, node, b0, sizeof (*t));
1168               t->is_slow_path = is_slow_path;
1169               t->sw_if_index = sw_if_index0;
1170               t->next_index = next0;
1171               t->session_index = ~0;
1172               if (s0)
1173                 t->session_index = s0 - tsm->sessions;
1174             }
1175
1176           pkts_processed += next0 == NAT_IN2OUT_ED_NEXT_LOOKUP;
1177
1178
1179           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1180
1181           if (is_output_feature)
1182             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1183
1184           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1185                                   iph_offset1);
1186
1187           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1188           rx_fib_index1 =
1189             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1190                                                  sw_if_index1);
1191
1192           if (PREDICT_FALSE (ip1->ttl == 1))
1193             {
1194               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1195               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1196                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1197                                            0);
1198               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1199               goto trace01;
1200             }
1201
1202           udp1 = ip4_next_header (ip1);
1203           tcp1 = (tcp_header_t *) udp1;
1204           icmp1 = (icmp46_header_t *) udp1;
1205           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1206
1207           if (is_slow_path)
1208             {
1209               if (PREDICT_FALSE (proto1 == ~0))
1210                 {
1211                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1212                                                       rx_fib_index1,
1213                                                       thread_index, now, vm,
1214                                                       node);
1215                   if (!s1)
1216                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1217                   other_packets++;
1218                   goto trace01;
1219                 }
1220
1221               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1222                 {
1223                   next1 = icmp_in2out_ed_slow_path
1224                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1225                      next1, now, thread_index, &s1);
1226                   icmp_packets++;
1227                   goto trace01;
1228                 }
1229             }
1230           else
1231             {
1232               if (PREDICT_FALSE (proto1 == ~0))
1233                 {
1234                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1235                   goto trace01;
1236                 }
1237
1238               if (ip4_is_fragment (ip1))
1239                 {
1240                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1241                   fragments++;
1242                   goto trace01;
1243                 }
1244
1245               if (is_output_feature)
1246                 {
1247                   if (PREDICT_FALSE
1248                       (nat_not_translate_output_feature_fwd
1249                        (sm, ip1, thread_index, now, vm, b1)))
1250                     goto trace01;
1251                 }
1252
1253               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1254                 {
1255                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1256                   goto trace01;
1257                 }
1258             }
1259
1260           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1261                       ip1->protocol, rx_fib_index1, udp1->src_port,
1262                       udp1->dst_port);
1263
1264           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1265             {
1266               if (is_slow_path)
1267                 {
1268                   if (is_output_feature)
1269                     {
1270                       if (PREDICT_FALSE
1271                           (nat44_ed_not_translate_output_feature
1272                            (sm, ip1, ip1->protocol, udp1->src_port,
1273                             udp1->dst_port, thread_index, sw_if_index1,
1274                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1275                         goto trace01;
1276                     }
1277                   else
1278                     {
1279                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1280                                                                  sw_if_index1,
1281                                                                  ip1, proto1,
1282                                                                  rx_fib_index1,
1283                                                                  thread_index)))
1284                         goto trace01;
1285                     }
1286
1287                   next1 =
1288                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1289                                   next1, thread_index, now, tcp1);
1290
1291                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1292                     goto trace01;
1293
1294                   if (PREDICT_FALSE (!s1))
1295                     goto trace01;
1296                 }
1297               else
1298                 {
1299                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1300                   goto trace01;
1301                 }
1302             }
1303           else
1304             {
1305               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1306             }
1307
1308           b1->flags |= VNET_BUFFER_F_IS_NATED;
1309
1310           if (!is_output_feature)
1311             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1312
1313           old_addr1 = ip1->src_address.as_u32;
1314           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1315           sum1 = ip1->checksum;
1316           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1317                                  src_address);
1318           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1319             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1320                                    s1->ext_host_addr.as_u32, ip4_header_t,
1321                                    dst_address);
1322           ip1->checksum = ip_csum_fold (sum1);
1323
1324           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1325             {
1326               old_port1 = tcp1->src_port;
1327               new_port1 = tcp1->src_port = s1->out2in.port;
1328
1329               sum1 = tcp1->checksum;
1330               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1331                                      dst_address);
1332               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1333                                      length);
1334               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1335                 {
1336                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1337                                          s1->ext_host_addr.as_u32,
1338                                          ip4_header_t, dst_address);
1339                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1340                                          s1->ext_host_port, ip4_header_t,
1341                                          length);
1342                   tcp1->dst_port = s1->ext_host_port;
1343                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1344                 }
1345               tcp1->checksum = ip_csum_fold (sum1);
1346               mss_clamping (sm, tcp1, &sum1);
1347               tcp_packets++;
1348               if (nat44_set_tcp_session_state_i2o
1349                   (sm, s1, tcp1, thread_index))
1350                 goto trace01;
1351             }
1352           else
1353             {
1354               udp1->src_port = s1->out2in.port;
1355               udp1->checksum = 0;
1356               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1357                 {
1358                   udp1->dst_port = s1->ext_host_port;
1359                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1360                 }
1361               udp_packets++;
1362             }
1363
1364           /* Accounting */
1365           nat44_session_update_counters (s1, now,
1366                                          vlib_buffer_length_in_chain (vm,
1367                                                                       b1));
1368           /* Per-user LRU list maintenance */
1369           nat44_session_update_lru (sm, s1, thread_index);
1370
1371         trace01:
1372           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1373                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1374             {
1375               nat_in2out_ed_trace_t *t =
1376                 vlib_add_trace (vm, node, b1, sizeof (*t));
1377               t->is_slow_path = is_slow_path;
1378               t->sw_if_index = sw_if_index1;
1379               t->next_index = next1;
1380               t->session_index = ~0;
1381               if (s1)
1382                 t->session_index = s1 - tsm->sessions;
1383             }
1384
1385           pkts_processed += next1 == NAT_IN2OUT_ED_NEXT_LOOKUP;
1386
1387           /* verify speculative enqueues, maybe switch current next frame */
1388           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1389                                            to_next, n_left_to_next,
1390                                            bi0, bi1, next0, next1);
1391         }
1392
1393       while (n_left_from > 0 && n_left_to_next > 0)
1394         {
1395           u32 bi0;
1396           vlib_buffer_t *b0;
1397           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1398             new_addr0, old_addr0;
1399           u16 old_port0, new_port0;
1400           ip4_header_t *ip0;
1401           udp_header_t *udp0;
1402           tcp_header_t *tcp0;
1403           icmp46_header_t *icmp0;
1404           snat_session_t *s0 = 0;
1405           clib_bihash_kv_16_8_t kv0, value0;
1406           ip_csum_t sum0;
1407
1408           /* speculatively enqueue b0 to the current next frame */
1409           bi0 = from[0];
1410           to_next[0] = bi0;
1411           from += 1;
1412           to_next += 1;
1413           n_left_from -= 1;
1414           n_left_to_next -= 1;
1415
1416           b0 = vlib_get_buffer (vm, bi0);
1417           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1418
1419           if (is_output_feature)
1420             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1421
1422           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1423                                   iph_offset0);
1424
1425           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1426           rx_fib_index0 =
1427             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1428                                                  sw_if_index0);
1429
1430           if (PREDICT_FALSE (ip0->ttl == 1))
1431             {
1432               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1433               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1434                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1435                                            0);
1436               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1437               goto trace0;
1438             }
1439
1440           udp0 = ip4_next_header (ip0);
1441           tcp0 = (tcp_header_t *) udp0;
1442           icmp0 = (icmp46_header_t *) udp0;
1443           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1444
1445           if (is_slow_path)
1446             {
1447               if (PREDICT_FALSE (proto0 == ~0))
1448                 {
1449                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1450                                                       rx_fib_index0,
1451                                                       thread_index, now, vm,
1452                                                       node);
1453                   if (!s0)
1454                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1455                   other_packets++;
1456                   goto trace0;
1457                 }
1458
1459               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1460                 {
1461                   next0 = icmp_in2out_ed_slow_path
1462                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1463                      next0, now, thread_index, &s0);
1464                   icmp_packets++;
1465                   goto trace0;
1466                 }
1467             }
1468           else
1469             {
1470               if (PREDICT_FALSE (proto0 == ~0))
1471                 {
1472                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1473                   goto trace0;
1474                 }
1475
1476               if (ip4_is_fragment (ip0))
1477                 {
1478                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1479                   fragments++;
1480                   goto trace0;
1481                 }
1482
1483               if (is_output_feature)
1484                 {
1485                   if (PREDICT_FALSE
1486                       (nat_not_translate_output_feature_fwd
1487                        (sm, ip0, thread_index, now, vm, b0)))
1488                     goto trace0;
1489                 }
1490
1491               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1492                 {
1493                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1494                   goto trace0;
1495                 }
1496             }
1497
1498           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1499                       ip0->protocol, rx_fib_index0, udp0->src_port,
1500                       udp0->dst_port);
1501
1502           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1503             {
1504               if (is_slow_path)
1505                 {
1506                   if (is_output_feature)
1507                     {
1508                       if (PREDICT_FALSE
1509                           (nat44_ed_not_translate_output_feature
1510                            (sm, ip0, ip0->protocol, udp0->src_port,
1511                             udp0->dst_port, thread_index, sw_if_index0,
1512                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1513                         goto trace0;
1514                     }
1515                   else
1516                     {
1517                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1518                                                                  sw_if_index0,
1519                                                                  ip0, proto0,
1520                                                                  rx_fib_index0,
1521                                                                  thread_index)))
1522                         goto trace0;
1523                     }
1524
1525                   next0 =
1526                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1527                                   next0, thread_index, now, tcp0);
1528
1529                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1530                     goto trace0;
1531
1532                   if (PREDICT_FALSE (!s0))
1533                     goto trace0;
1534                 }
1535               else
1536                 {
1537                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1538                   goto trace0;
1539                 }
1540             }
1541           else
1542             {
1543               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1544             }
1545
1546           b0->flags |= VNET_BUFFER_F_IS_NATED;
1547
1548           if (!is_output_feature)
1549             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1550
1551           old_addr0 = ip0->src_address.as_u32;
1552           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1553           sum0 = ip0->checksum;
1554           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1555                                  src_address);
1556           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1557             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1558                                    s0->ext_host_addr.as_u32, ip4_header_t,
1559                                    dst_address);
1560           ip0->checksum = ip_csum_fold (sum0);
1561
1562           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1563             {
1564               old_port0 = tcp0->src_port;
1565               new_port0 = tcp0->src_port = s0->out2in.port;
1566
1567               sum0 = tcp0->checksum;
1568               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1569                                      dst_address);
1570               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1571                                      length);
1572               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1573                 {
1574                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1575                                          s0->ext_host_addr.as_u32,
1576                                          ip4_header_t, dst_address);
1577                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1578                                          s0->ext_host_port, ip4_header_t,
1579                                          length);
1580                   tcp0->dst_port = s0->ext_host_port;
1581                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1582                 }
1583               mss_clamping (sm, tcp0, &sum0);
1584               tcp0->checksum = ip_csum_fold (sum0);
1585               tcp_packets++;
1586               if (nat44_set_tcp_session_state_i2o
1587                   (sm, s0, tcp0, thread_index))
1588                 goto trace0;
1589             }
1590           else
1591             {
1592               udp0->src_port = s0->out2in.port;
1593               udp0->checksum = 0;
1594               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1595                 {
1596                   udp0->dst_port = s0->ext_host_port;
1597                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1598                 }
1599               udp_packets++;
1600             }
1601
1602           /* Accounting */
1603           nat44_session_update_counters (s0, now,
1604                                          vlib_buffer_length_in_chain (vm,
1605                                                                       b0));
1606           /* Per-user LRU list maintenance */
1607           nat44_session_update_lru (sm, s0, thread_index);
1608
1609         trace0:
1610           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1611                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1612             {
1613               nat_in2out_ed_trace_t *t =
1614                 vlib_add_trace (vm, node, b0, sizeof (*t));
1615               t->is_slow_path = is_slow_path;
1616               t->sw_if_index = sw_if_index0;
1617               t->next_index = next0;
1618               t->session_index = ~0;
1619               if (s0)
1620                 t->session_index = s0 - tsm->sessions;
1621             }
1622
1623           pkts_processed += next0 == NAT_IN2OUT_ED_NEXT_LOOKUP;
1624
1625           /* verify speculative enqueue, maybe switch current next frame */
1626           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1627                                            to_next, n_left_to_next,
1628                                            bi0, next0);
1629         }
1630
1631       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1632     }
1633
1634   vlib_node_increment_counter (vm, stats_node_index,
1635                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1636                                pkts_processed);
1637   vlib_node_increment_counter (vm, stats_node_index,
1638                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1639   vlib_node_increment_counter (vm, stats_node_index,
1640                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, tcp_packets);
1641   vlib_node_increment_counter (vm, stats_node_index,
1642                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1643                                icmp_packets);
1644   vlib_node_increment_counter (vm, stats_node_index,
1645                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1646                                other_packets);
1647   vlib_node_increment_counter (vm, stats_node_index,
1648                                NAT_IN2OUT_ED_ERROR_FRAGMENTS, fragments);
1649
1650   return frame->n_vectors;
1651 }
1652
1653 static uword
1654 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1655                               vlib_node_runtime_t * node,
1656                               vlib_frame_t * frame)
1657 {
1658   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1659 }
1660
1661 /* *INDENT-OFF* */
1662 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1663   .function = nat44_ed_in2out_fast_path_fn,
1664   .name = "nat44-ed-in2out",
1665   .vector_size = sizeof (u32),
1666   .format_trace = format_nat_in2out_ed_trace,
1667   .type = VLIB_NODE_TYPE_INTERNAL,
1668   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1669   .error_strings = nat_in2out_ed_error_strings,
1670   .runtime_data_bytes = sizeof (snat_runtime_t),
1671   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1672   .next_nodes = {
1673     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1674     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1675     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1676     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1677     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1678   },
1679 };
1680 /* *INDENT-ON* */
1681
1682 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1683                               nat44_ed_in2out_fast_path_fn);
1684
1685 static uword
1686 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1687                                      vlib_node_runtime_t * node,
1688                                      vlib_frame_t * frame)
1689 {
1690   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1691 }
1692
1693 /* *INDENT-OFF* */
1694 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1695   .function = nat44_ed_in2out_output_fast_path_fn,
1696   .name = "nat44-ed-in2out-output",
1697   .vector_size = sizeof (u32),
1698   .format_trace = format_nat_in2out_ed_trace,
1699   .type = VLIB_NODE_TYPE_INTERNAL,
1700   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1701   .error_strings = nat_in2out_ed_error_strings,
1702   .runtime_data_bytes = sizeof (snat_runtime_t),
1703   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1704   .next_nodes = {
1705     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1706     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1707     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1708     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1709     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1710   },
1711 };
1712 /* *INDENT-ON* */
1713
1714 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1715                               nat44_ed_in2out_output_fast_path_fn);
1716
1717 static uword
1718 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1719                               vlib_node_runtime_t * node,
1720                               vlib_frame_t * frame)
1721 {
1722   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1723 }
1724
1725 /* *INDENT-OFF* */
1726 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1727   .function = nat44_ed_in2out_slow_path_fn,
1728   .name = "nat44-ed-in2out-slowpath",
1729   .vector_size = sizeof (u32),
1730   .format_trace = format_nat_in2out_ed_trace,
1731   .type = VLIB_NODE_TYPE_INTERNAL,
1732   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1733   .error_strings = nat_in2out_ed_error_strings,
1734   .runtime_data_bytes = sizeof (snat_runtime_t),
1735   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1736   .next_nodes = {
1737     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1738     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1739     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1740     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1741     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1742   },
1743 };
1744 /* *INDENT-ON* */
1745
1746 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1747                               nat44_ed_in2out_slow_path_fn);
1748
1749 static uword
1750 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1751                                      vlib_node_runtime_t * node,
1752                                      vlib_frame_t * frame)
1753 {
1754   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1755 }
1756
1757 /* *INDENT-OFF* */
1758 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1759   .function = nat44_ed_in2out_output_slow_path_fn,
1760   .name = "nat44-ed-in2out-output-slowpath",
1761   .vector_size = sizeof (u32),
1762   .format_trace = format_nat_in2out_ed_trace,
1763   .type = VLIB_NODE_TYPE_INTERNAL,
1764   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1765   .error_strings = nat_in2out_ed_error_strings,
1766   .runtime_data_bytes = sizeof (snat_runtime_t),
1767   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1768   .next_nodes = {
1769     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1770     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1771     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1772     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1773     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1774   },
1775 };
1776 /* *INDENT-ON* */
1777
1778 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1779                               nat44_ed_in2out_output_slow_path_fn);
1780
1781 static inline uword
1782 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1783                                       vlib_node_runtime_t * node,
1784                                       vlib_frame_t * frame,
1785                                       int is_output_feature)
1786 {
1787   u32 n_left_from, *from, *to_next;
1788   nat_in2out_ed_next_t next_index;
1789   u32 pkts_processed = 0, cached_fragments = 0;
1790   snat_main_t *sm = &snat_main;
1791   f64 now = vlib_time_now (vm);
1792   u32 thread_index = vm->thread_index;
1793   snat_main_per_thread_data_t *per_thread_data =
1794     &sm->per_thread_data[thread_index];
1795   u32 *fragments_to_drop = 0;
1796   u32 *fragments_to_loopback = 0;
1797
1798   from = vlib_frame_vector_args (frame);
1799   n_left_from = frame->n_vectors;
1800   next_index = node->cached_next_index;
1801
1802   while (n_left_from > 0)
1803     {
1804       u32 n_left_to_next;
1805
1806       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1807
1808       while (n_left_from > 0 && n_left_to_next > 0)
1809         {
1810           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1811           u32 iph_offset0 = 0;
1812           vlib_buffer_t *b0;
1813           u32 next0;
1814           u8 cached0 = 0;
1815           ip4_header_t *ip0 = 0;
1816           nat_reass_ip4_t *reass0;
1817           udp_header_t *udp0;
1818           tcp_header_t *tcp0;
1819           icmp46_header_t *icmp0;
1820           clib_bihash_kv_16_8_t kv0, value0;
1821           snat_session_t *s0 = 0;
1822           u16 old_port0, new_port0;
1823           ip_csum_t sum0;
1824
1825           /* speculatively enqueue b0 to the current next frame */
1826           bi0 = from[0];
1827           to_next[0] = bi0;
1828           from += 1;
1829           to_next += 1;
1830           n_left_from -= 1;
1831           n_left_to_next -= 1;
1832
1833           b0 = vlib_get_buffer (vm, bi0);
1834
1835           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1836
1837           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1838           rx_fib_index0 =
1839             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1840                                                  sw_if_index0);
1841
1842           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1843             {
1844               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1845               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1846               goto trace0;
1847             }
1848
1849           if (is_output_feature)
1850             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1851
1852           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1853                                   iph_offset0);
1854
1855           udp0 = ip4_next_header (ip0);
1856           tcp0 = (tcp_header_t *) udp0;
1857           icmp0 = (icmp46_header_t *) udp0;
1858           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1859
1860           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1861                                                  ip0->dst_address,
1862                                                  ip0->fragment_id,
1863                                                  ip0->protocol,
1864                                                  1, &fragments_to_drop);
1865
1866           if (PREDICT_FALSE (!reass0))
1867             {
1868               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1869               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1870               nat_log_notice ("maximum reassemblies exceeded");
1871               goto trace0;
1872             }
1873
1874           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1875             {
1876               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1877                 {
1878                   if (is_output_feature)
1879                     {
1880                       if (PREDICT_FALSE
1881                           (nat_not_translate_output_feature_fwd
1882                            (sm, ip0, thread_index, now, vm, b0)))
1883                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1884                       goto trace0;
1885                     }
1886
1887                   next0 = icmp_in2out_ed_slow_path
1888                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1889                      next0, now, thread_index, &s0);
1890
1891                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1892                     {
1893                       if (s0)
1894                         reass0->sess_index = s0 - per_thread_data->sessions;
1895                       else
1896                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1897                       nat_ip4_reass_get_frags (reass0,
1898                                                &fragments_to_loopback);
1899                     }
1900
1901                   goto trace0;
1902                 }
1903
1904               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1905                           ip0->protocol, rx_fib_index0, udp0->src_port,
1906                           udp0->dst_port);
1907
1908               if (clib_bihash_search_16_8
1909                   (&per_thread_data->in2out_ed, &kv0, &value0))
1910                 {
1911                   if (is_output_feature)
1912                     {
1913                       if (PREDICT_FALSE
1914                           (nat44_ed_not_translate_output_feature
1915                            (sm, ip0, ip0->protocol, udp0->src_port,
1916                             udp0->dst_port, thread_index, sw_if_index0,
1917                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1918                         {
1919                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1920                           nat_ip4_reass_get_frags (reass0,
1921                                                    &fragments_to_loopback);
1922                           goto trace0;
1923                         }
1924                     }
1925                   else
1926                     {
1927                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1928                                                                  sw_if_index0,
1929                                                                  ip0, proto0,
1930                                                                  rx_fib_index0,
1931                                                                  thread_index)))
1932                         {
1933                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1934                           nat_ip4_reass_get_frags (reass0,
1935                                                    &fragments_to_loopback);
1936                           goto trace0;
1937                         }
1938                     }
1939
1940                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1941                                         &s0, node, next0, thread_index, now,
1942                                         tcp0);
1943
1944                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1945                     goto trace0;
1946
1947                   if (PREDICT_FALSE (!s0))
1948                     {
1949                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1950                       goto trace0;
1951                     }
1952
1953                   reass0->sess_index = s0 - per_thread_data->sessions;
1954                 }
1955               else
1956                 {
1957                   s0 = pool_elt_at_index (per_thread_data->sessions,
1958                                           value0.value);
1959                   reass0->sess_index = value0.value;
1960                 }
1961               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1962             }
1963           else
1964             {
1965               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1966                 goto trace0;
1967               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1968                 {
1969                   if (nat_ip4_reass_add_fragment
1970                       (thread_index, reass0, bi0, &fragments_to_drop))
1971                     {
1972                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1973                       nat_log_notice
1974                         ("maximum fragments per reassembly exceeded");
1975                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1976                       goto trace0;
1977                     }
1978                   cached0 = 1;
1979                   goto trace0;
1980                 }
1981               s0 = pool_elt_at_index (per_thread_data->sessions,
1982                                       reass0->sess_index);
1983             }
1984
1985           old_addr0 = ip0->src_address.as_u32;
1986           ip0->src_address = s0->out2in.addr;
1987           new_addr0 = ip0->src_address.as_u32;
1988           if (!is_output_feature)
1989             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1990
1991           sum0 = ip0->checksum;
1992           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1993                                  ip4_header_t,
1994                                  src_address /* changed member */ );
1995           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1996             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1997                                    s0->ext_host_addr.as_u32, ip4_header_t,
1998                                    dst_address);
1999           ip0->checksum = ip_csum_fold (sum0);
2000
2001           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2002             {
2003               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2004                 {
2005                   old_port0 = tcp0->src_port;
2006                   tcp0->src_port = s0->out2in.port;
2007                   new_port0 = tcp0->src_port;
2008
2009                   sum0 = tcp0->checksum;
2010                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2011                                          ip4_header_t,
2012                                          dst_address /* changed member */ );
2013                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2014                                          ip4_header_t /* cheat */ ,
2015                                          length /* changed member */ );
2016                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
2017                     {
2018                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
2019                                              s0->ext_host_addr.as_u32,
2020                                              ip4_header_t, dst_address);
2021                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
2022                                              s0->ext_host_port, ip4_header_t,
2023                                              length);
2024                       tcp0->dst_port = s0->ext_host_port;
2025                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
2026                     }
2027                   tcp0->checksum = ip_csum_fold (sum0);
2028                 }
2029               else
2030                 {
2031                   old_port0 = udp0->src_port;
2032                   udp0->src_port = s0->out2in.port;
2033                   udp0->checksum = 0;
2034                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
2035                     {
2036                       udp0->dst_port = s0->ext_host_port;
2037                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
2038                     }
2039                 }
2040             }
2041
2042           /* Hairpinning */
2043           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2044                                    s0->ext_host_port, proto0, 1);
2045
2046           /* Accounting */
2047           nat44_session_update_counters (s0, now,
2048                                          vlib_buffer_length_in_chain (vm,
2049                                                                       b0));
2050           /* Per-user LRU list maintenance */
2051           nat44_session_update_lru (sm, s0, thread_index);
2052
2053         trace0:
2054           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2055                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2056             {
2057               nat44_reass_trace_t *t =
2058                 vlib_add_trace (vm, node, b0, sizeof (*t));
2059               t->cached = cached0;
2060               t->sw_if_index = sw_if_index0;
2061               t->next_index = next0;
2062             }
2063
2064           if (cached0)
2065             {
2066               n_left_to_next++;
2067               to_next--;
2068               cached_fragments++;
2069             }
2070           else
2071             {
2072               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
2073
2074               /* verify speculative enqueue, maybe switch current next frame */
2075               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2076                                                to_next, n_left_to_next,
2077                                                bi0, next0);
2078             }
2079
2080           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2081             {
2082               from = vlib_frame_vector_args (frame);
2083               u32 len = vec_len (fragments_to_loopback);
2084               if (len <= VLIB_FRAME_SIZE)
2085                 {
2086                   clib_memcpy_fast (from, fragments_to_loopback,
2087                                     sizeof (u32) * len);
2088                   n_left_from = len;
2089                   vec_reset_length (fragments_to_loopback);
2090                 }
2091               else
2092                 {
2093                   clib_memcpy_fast (from, fragments_to_loopback +
2094                                     (len - VLIB_FRAME_SIZE),
2095                                     sizeof (u32) * VLIB_FRAME_SIZE);
2096                   n_left_from = VLIB_FRAME_SIZE;
2097                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2098                 }
2099             }
2100         }
2101
2102       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2103     }
2104
2105   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
2106                                NAT_IN2OUT_ED_ERROR_PROCESSED_FRAGMENTS,
2107                                pkts_processed);
2108   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
2109                                NAT_IN2OUT_ED_ERROR_CACHED_FRAGMENTS,
2110                                cached_fragments);
2111
2112   nat_send_all_to_node (vm, fragments_to_drop, node,
2113                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
2114                         NAT_IN2OUT_ED_NEXT_DROP);
2115
2116   vec_free (fragments_to_drop);
2117   vec_free (fragments_to_loopback);
2118   return frame->n_vectors;
2119 }
2120
2121 static uword
2122 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
2123                                vlib_node_runtime_t * node,
2124                                vlib_frame_t * frame)
2125 {
2126   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2127 }
2128
2129 /* *INDENT-OFF* */
2130 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2131   .function = nat44_ed_in2out_reass_node_fn,
2132   .name = "nat44-ed-in2out-reass",
2133   .vector_size = sizeof (u32),
2134   .format_trace = format_nat44_reass_trace,
2135   .type = VLIB_NODE_TYPE_INTERNAL,
2136   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2137   .error_strings = nat_in2out_ed_error_strings,
2138   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2139   .next_nodes = {
2140     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2141     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2142     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2143     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2144     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2145   },
2146 };
2147 /* *INDENT-ON* */
2148
2149 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2150                               nat44_ed_in2out_reass_node_fn);
2151
2152 static uword
2153 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2154                                       vlib_node_runtime_t * node,
2155                                       vlib_frame_t * frame)
2156 {
2157   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2158 }
2159
2160 /* *INDENT-OFF* */
2161 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2162   .function = nat44_ed_in2out_reass_output_node_fn,
2163   .name = "nat44-ed-in2out-reass-output",
2164   .vector_size = sizeof (u32),
2165   .format_trace = format_nat44_reass_trace,
2166   .type = VLIB_NODE_TYPE_INTERNAL,
2167   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2168   .error_strings = nat_in2out_ed_error_strings,
2169   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2170   .next_nodes = {
2171     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2172     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2173     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2174     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2175     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2176   },
2177 };
2178 /* *INDENT-ON* */
2179
2180 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2181                               nat44_ed_in2out_reass_output_node_fn);
2182
2183 /*
2184  * fd.io coding-style-patch-verification: ON
2185  *
2186  * Local Variables:
2187  * eval: (c-set-style "gnu")
2188  * End:
2189  */