src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27 #include <nat/nat.h>
  28 #include <nat/lib/ipfix_logging.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/nat44/inlines.h>
  31 #include <nat/lib/nat_syslog.h>
  32 #include <nat/nat44/ed_inlines.h>
  33 #include <nat/lib/nat_inlines.h>
  34
  35 /* number of attempts to get a port for ED overloading algorithm, if rolling
  36  * a dice this many times doesn't produce a free port, it's treated
  37  * as if there were no free ports available to conserve resources */
  38 #define ED_PORT_ALLOC_ATTEMPTS (10)
  39
  40 static char *nat_in2out_ed_error_strings[] = {
  41 #define _(sym,string) string,
  42   foreach_nat_in2out_ed_error
  43 #undef _
  44 };
  45
  46 typedef struct
  47 {
  48   u32 sw_if_index;
  49   u32 next_index;
  50   u32 session_index;
  51   nat_translation_error_e translation_error;
  52   nat_6t_flow_t i2of;
  53   nat_6t_flow_t o2if;
  54   clib_bihash_kv_16_8_t search_key;
  55   u8 is_slow_path;
  56   u8 translation_via_i2of;
  57   u8 lookup_skipped;
  58 } nat_in2out_ed_trace_t;
  59
  60 static u8 *
  61 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  66   char *tag;
  67
  68   tag =
  69     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  70     "NAT44_IN2OUT_ED_FAST_PATH";
  71
  72   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  73               t->sw_if_index, t->next_index, t->session_index);
  74   if (~0 != t->session_index)
  75     {
  76       s = format (s, ", translation result '%U' via %s",
  77                   format_nat_ed_translation_error, t->translation_error,
  78                   t->translation_via_i2of ? "i2of" : "o2if");
  79       s = format (s, "\n  i2of %U", format_nat_6t_flow, &t->i2of);
  80       s = format (s, "\n  o2if %U", format_nat_6t_flow, &t->o2if);
  81     }
  82   if (!t->is_slow_path)
  83     {
  84       if (t->lookup_skipped)
  85         {
  86           s = format (s, "\n lookup skipped - cached session index used");
  87         }
  88       else
  89         {
  90           s = format (s, "\n  search key %U", format_ed_session_kvp,
  91                       &t->search_key);
  92         }
  93     }
  94
  95   return s;
  96 }
  97
  98 static int
  99 nat_ed_alloc_addr_and_port (snat_main_t *sm, u32 rx_fib_index, u32 nat_proto,
 100                             u32 thread_index, ip4_address_t r_addr, u16 r_port,
 101                             u8 proto, u16 port_per_thread,
 102                             u32 snat_thread_index, snat_session_t *s,
 103                             ip4_address_t *outside_addr, u16 *outside_port)
 104 {
 105   int i;
 106   snat_address_t *a, *ga = 0;
 107
 108   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 109
 110   for (i = 0; i < vec_len (sm->addresses); i++)
 111     {
 112       a = sm->addresses + i;
 113       switch (nat_proto)
 114         {
 115 #define _(N, j, n, unused)                                                    \
 116   case NAT_PROTOCOL_##N:                                                      \
 117     if (a->fib_index == rx_fib_index)                                         \
 118       {                                                                       \
 119         s->o2i.match.daddr = a->addr;                                         \
 120         /* first try port suggested by caller */                              \
 121         u16 port = clib_net_to_host_u16 (*outside_port);                      \
 122         u16 port_offset = port - port_thread_offset;                          \
 123         if (port <= port_thread_offset ||                                     \
 124             port > port_thread_offset + port_per_thread)                      \
 125           {                                                                   \
 126             /* need to pick a different port, suggested port doesn't fit in   \
 127              * this thread's port range */                                    \
 128             port_offset = snat_random_port (0, port_per_thread - 1);          \
 129             port = port_thread_offset + port_offset;                          \
 130           }                                                                   \
 131         u16 attempts = ED_PORT_ALLOC_ATTEMPTS;                                \
 132         do                                                                    \
 133           {                                                                   \
 134             if (NAT_PROTOCOL_ICMP == nat_proto)                               \
 135               {                                                               \
 136                 s->o2i.match.sport = clib_host_to_net_u16 (port);             \
 137               }                                                               \
 138             s->o2i.match.dport = clib_host_to_net_u16 (port);                 \
 139             if (0 ==                                                          \
 140                 nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 2))    \
 141               {                                                               \
 142                 ++a->busy_##n##_port_refcounts[port];                         \
 143                 a->busy_##n##_ports_per_thread[thread_index]++;               \
 144                 a->busy_##n##_ports++;                                        \
 145                 *outside_addr = a->addr;                                      \
 146                 *outside_port = clib_host_to_net_u16 (port);                  \
 147                 return 0;                                                     \
 148               }                                                               \
 149             port_offset = snat_random_port (0, port_per_thread - 1);          \
 150             port = port_thread_offset + port_offset;                          \
 151             --attempts;                                                       \
 152           }                                                                   \
 153         while (attempts > 0);                                                 \
 154       }                                                                       \
 155     else if (a->fib_index == ~0)                                              \
 156       {                                                                       \
 157         ga = a;                                                               \
 158       }                                                                       \
 159     break;
 160
 161           foreach_nat_protocol;
 162         default:
 163           nat_elog_info ("unknown protocol");
 164           return 1;
 165         }
 166     }
 167
 168   if (ga)
 169     {
 170       /* fake fib_index to reuse macro */
 171       rx_fib_index = ~0;
 172       a = ga;
 173       switch (nat_proto)
 174         {
 175           foreach_nat_protocol;
 176         default:
 177           nat_elog_info ("unknown protocol");
 178           return 1;
 179         }
 180     }
 181
 182 #undef _
 183
 184   /* Totally out of translations to use... */
 185   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 186   return 1;
 187 }
 188
 189 static_always_inline u32
 190 nat_outside_fib_index_lookup (snat_main_t * sm, ip4_address_t addr)
 191 {
 192   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 193   nat_outside_fib_t *outside_fib;
 194   fib_prefix_t pfx = {
 195     .fp_proto = FIB_PROTOCOL_IP4,
 196     .fp_len = 32,
 197     .fp_addr = {.ip4.as_u32 = addr.as_u32,}
 198     ,
 199   };
 200   // TODO: multiple vrfs none can resolve addr
 201   /* *INDENT-OFF* */
 202   vec_foreach (outside_fib, sm->outside_fibs)
 203     {
 204       fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 205       if (FIB_NODE_INDEX_INVALID != fei)
 206         {
 207           if (fib_entry_get_resolving_interface (fei) != ~0)
 208             {
 209               return outside_fib->fib_index;
 210             }
 211         }
 212     }
 213   /* *INDENT-ON* */
 214   return ~0;
 215 }
 216
 217 static_always_inline int
 218 nat44_ed_external_sm_lookup (snat_main_t *sm, ip4_address_t match_addr,
 219                              u16 match_port, nat_protocol_t match_protocol,
 220                              u32 match_fib_index, ip4_address_t *daddr,
 221                              u16 *dport)
 222 {
 223   clib_bihash_kv_8_8_t kv, value;
 224   init_nat_k (&kv, match_addr, match_port, match_fib_index, match_protocol);
 225   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 226     {
 227       /* Try address only mapping */
 228       init_nat_k (&kv, match_addr, 0, 0, 0);
 229       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv,
 230                                   &value))
 231         return 0;
 232     }
 233
 234   snat_static_mapping_t *m =
 235     pool_elt_at_index (sm->static_mappings, value.value);
 236   *daddr = m->local_addr;
 237   if (dport)
 238     {
 239       /* Address only mapping doesn't change port */
 240       *dport = is_addr_only_static_mapping (m) ? match_port : m->local_port;
 241     }
 242   return 1;
 243 }
 244
 245 static u32
 246 slow_path_ed (snat_main_t *sm, vlib_buffer_t *b, ip4_address_t l_addr,
 247               ip4_address_t r_addr, u16 l_port, u16 r_port, u8 proto,
 248               u32 rx_fib_index, snat_session_t **sessionp,
 249               vlib_node_runtime_t *node, u32 next, u32 thread_index, f64 now)
 250 {
 251   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 252   ip4_address_t outside_addr;
 253   u16 outside_port;
 254   u32 outside_fib_index;
 255   u8 is_identity_nat;
 256
 257   u32 nat_proto = ip_proto_to_nat_proto (proto);
 258   snat_session_t *s = NULL;
 259   lb_nat_type_t lb = 0;
 260   ip4_address_t daddr = r_addr;
 261   u16 dport = r_port;
 262
 263   if (PREDICT_TRUE (nat_proto == NAT_PROTOCOL_TCP))
 264     {
 265       if (PREDICT_FALSE
 266           (!tcp_flags_is_init
 267            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 268         {
 269           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 270           return NAT_NEXT_DROP;
 271         }
 272     }
 273
 274   if (PREDICT_FALSE
 275       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 276     {
 277       if (!nat_lru_free_one (sm, thread_index, now))
 278         {
 279           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 280           nat_ipfix_logging_max_sessions (thread_index,
 281                                           sm->max_translations_per_thread);
 282           nat_elog_notice ("maximum sessions exceeded");
 283           return NAT_NEXT_DROP;
 284         }
 285     }
 286
 287   outside_fib_index = sm->outside_fib_index;
 288
 289   switch (vec_len (sm->outside_fibs))
 290     {
 291     case 0:
 292       outside_fib_index = sm->outside_fib_index;
 293       break;
 294     case 1:
 295       outside_fib_index = sm->outside_fibs[0].fib_index;
 296       break;
 297     default:
 298       outside_fib_index = nat_outside_fib_index_lookup (sm, r_addr);
 299       break;
 300     }
 301
 302   ip4_address_t sm_addr;
 303   u16 sm_port;
 304   u32 sm_fib_index;
 305   /* First try to match static mapping by local address and port */
 306   int is_sm;
 307   if (snat_static_mapping_match (sm, l_addr, l_port, rx_fib_index, nat_proto,
 308                                  &sm_addr, &sm_port, &sm_fib_index, 0, 0, 0,
 309                                  &lb, 0, &is_identity_nat, 0))
 310     {
 311       is_sm = 0;
 312     }
 313   else
 314     {
 315       is_sm = 1;
 316     }
 317
 318   if (PREDICT_FALSE (is_sm && is_identity_nat))
 319     {
 320       *sessionp = NULL;
 321       return next;
 322     }
 323
 324   s = nat_ed_session_alloc (sm, thread_index, now, proto);
 325   ASSERT (s);
 326
 327   if (!is_sm)
 328     {
 329       s->in2out.addr = l_addr;
 330       s->in2out.port = l_port;
 331       s->nat_proto = nat_proto;
 332       s->in2out.fib_index = rx_fib_index;
 333       s->out2in.fib_index = outside_fib_index;
 334
 335       // suggest using local port to allocation function
 336       outside_port = l_port;
 337
 338       // hairpinning?
 339       int is_hairpinning = nat44_ed_external_sm_lookup (
 340         sm, r_addr, r_port, nat_proto, outside_fib_index, &daddr, &dport);
 341       s->flags |= is_hairpinning * SNAT_SESSION_FLAG_HAIRPINNING;
 342
 343       // destination addr/port updated with real values in
 344       // nat_ed_alloc_addr_and_port
 345       nat_6t_o2i_flow_init (sm, thread_index, s, daddr, dport, daddr, 0,
 346                             s->out2in.fib_index, proto);
 347       nat_6t_flow_daddr_rewrite_set (&s->o2i, l_addr.as_u32);
 348       if (NAT_PROTOCOL_ICMP == nat_proto)
 349         {
 350           nat_6t_flow_icmp_id_rewrite_set (&s->o2i, l_port);
 351         }
 352       else
 353         {
 354           nat_6t_flow_dport_rewrite_set (&s->o2i, l_port);
 355         }
 356       nat_6t_flow_txfib_rewrite_set (&s->o2i, rx_fib_index);
 357
 358       if (nat_ed_alloc_addr_and_port (
 359             sm, rx_fib_index, nat_proto, thread_index, daddr, dport, proto,
 360             sm->port_per_thread, tsm->snat_thread_index, s, &outside_addr,
 361             &outside_port))
 362         {
 363           nat_elog_notice ("addresses exhausted");
 364           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 365           nat_ed_session_delete (sm, s, thread_index, 1);
 366           return NAT_NEXT_DROP;
 367         }
 368       s->out2in.addr = outside_addr;
 369       s->out2in.port = outside_port;
 370     }
 371   else
 372     {
 373       // static mapping
 374       s->out2in.addr = outside_addr = sm_addr;
 375       s->out2in.port = outside_port = sm_port;
 376       s->in2out.addr = l_addr;
 377       s->in2out.port = l_port;
 378       s->nat_proto = nat_proto;
 379       s->in2out.fib_index = rx_fib_index;
 380       s->out2in.fib_index = outside_fib_index;
 381       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 382
 383       // hairpinning?
 384       int is_hairpinning = nat44_ed_external_sm_lookup (
 385         sm, r_addr, r_port, nat_proto, outside_fib_index, &daddr, &dport);
 386       s->flags |= is_hairpinning * SNAT_SESSION_FLAG_HAIRPINNING;
 387
 388       if (NAT_PROTOCOL_ICMP == nat_proto)
 389         {
 390           nat_6t_o2i_flow_init (sm, thread_index, s, daddr, sm_port, sm_addr,
 391                                 sm_port, s->out2in.fib_index, proto);
 392           nat_6t_flow_icmp_id_rewrite_set (&s->o2i, l_port);
 393         }
 394       else
 395         {
 396           nat_6t_o2i_flow_init (sm, thread_index, s, daddr, dport, sm_addr,
 397                                 sm_port, s->out2in.fib_index, proto);
 398           nat_6t_flow_dport_rewrite_set (&s->o2i, l_port);
 399         }
 400       nat_6t_flow_daddr_rewrite_set (&s->o2i, l_addr.as_u32);
 401       nat_6t_flow_txfib_rewrite_set (&s->o2i, rx_fib_index);
 402       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 2))
 403         {
 404           nat_elog_notice ("out2in key add failed");
 405           goto error;
 406         }
 407     }
 408
 409   if (lb)
 410     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 411   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 412   s->ext_host_addr = r_addr;
 413   s->ext_host_port = r_port;
 414
 415   nat_6t_i2o_flow_init (sm, thread_index, s, l_addr, l_port, r_addr, r_port,
 416                         rx_fib_index, proto);
 417   nat_6t_flow_saddr_rewrite_set (&s->i2o, outside_addr.as_u32);
 418   nat_6t_flow_daddr_rewrite_set (&s->i2o, daddr.as_u32);
 419   if (NAT_PROTOCOL_ICMP == nat_proto)
 420     {
 421       nat_6t_flow_icmp_id_rewrite_set (&s->i2o, outside_port);
 422     }
 423   else
 424     {
 425       nat_6t_flow_sport_rewrite_set (&s->i2o, outside_port);
 426       nat_6t_flow_dport_rewrite_set (&s->i2o, dport);
 427     }
 428   nat_6t_flow_txfib_rewrite_set (&s->i2o, outside_fib_index);
 429
 430   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 431     {
 432       nat_elog_notice ("in2out key add failed");
 433       goto error;
 434     }
 435
 436   /* log NAT event */
 437   nat_ipfix_logging_nat44_ses_create (thread_index,
 438                                       s->in2out.addr.as_u32,
 439                                       s->out2in.addr.as_u32,
 440                                       s->nat_proto,
 441                                       s->in2out.port,
 442                                       s->out2in.port, s->in2out.fib_index);
 443
 444   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 445                          &s->in2out.addr, s->in2out.port,
 446                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 447                          &s->out2in.addr, s->out2in.port,
 448                          &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 449                          0);
 450
 451   per_vrf_sessions_register_session (s, thread_index);
 452
 453   *sessionp = s;
 454   return next;
 455 error:
 456   if (s)
 457     {
 458       if (!is_sm)
 459         {
 460           snat_free_outside_address_and_port (sm->addresses, thread_index,
 461                                               &outside_addr, outside_port,
 462                                               nat_proto);
 463         }
 464       nat_ed_session_delete (sm, s, thread_index, 1);
 465     }
 466   *sessionp = s = NULL;
 467   return NAT_NEXT_DROP;
 468 }
 469
 470 static_always_inline int
 471 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 472                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 473                         u32 rx_fib_index, u32 thread_index)
 474 {
 475   udp_header_t *udp = ip4_next_header (ip);
 476   clib_bihash_kv_16_8_t kv, value;
 477
 478   init_ed_k (&kv, ip->dst_address, udp->dst_port, ip->src_address,
 479              udp->src_port, sm->outside_fib_index, ip->protocol);
 480
 481   /* NAT packet aimed at external address if has active sessions */
 482   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 483     {
 484       /* or is static mappings */
 485       ip4_address_t placeholder_addr;
 486       u16 placeholder_port;
 487       u32 placeholder_fib_index;
 488       if (!snat_static_mapping_match
 489           (sm, ip->dst_address, udp->dst_port, sm->outside_fib_index, proto,
 490            &placeholder_addr, &placeholder_port, &placeholder_fib_index, 1, 0,
 491            0, 0, 0, 0, 0))
 492         return 0;
 493     }
 494   else
 495     return 0;
 496
 497   if (sm->forwarding_enabled)
 498     return 1;
 499
 500   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 501                                   rx_fib_index);
 502 }
 503
 504 static_always_inline int
 505 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 506                                       u32 thread_index, f64 now,
 507                                       vlib_main_t * vm, vlib_buffer_t * b)
 508 {
 509   clib_bihash_kv_16_8_t kv, value;
 510   snat_session_t *s = 0;
 511   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 512
 513   if (!sm->forwarding_enabled)
 514     return 0;
 515
 516   if (ip->protocol == IP_PROTOCOL_ICMP)
 517     {
 518       ip4_address_t lookup_saddr, lookup_daddr;
 519       u16 lookup_sport, lookup_dport;
 520       u8 lookup_protocol;
 521       if (nat_get_icmp_session_lookup_values (b, ip, &lookup_saddr,
 522                                               &lookup_sport, &lookup_daddr,
 523                                               &lookup_dport, &lookup_protocol))
 524         return 0;
 525       init_ed_k (&kv, lookup_saddr, lookup_sport, lookup_daddr, lookup_dport,
 526                  0, lookup_protocol);
 527     }
 528   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 529     {
 530       init_ed_k (&kv, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
 531                  ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port, 0,
 532                  ip->protocol);
 533     }
 534   else
 535     {
 536       init_ed_k (&kv, ip->src_address, 0, ip->dst_address, 0, 0,
 537                  ip->protocol);
 538     }
 539
 540   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 541     {
 542       ASSERT (thread_index == ed_value_get_thread_index (&value));
 543       s =
 544         pool_elt_at_index (tsm->sessions,
 545                            ed_value_get_session_index (&value));
 546
 547       if (is_fwd_bypass_session (s))
 548         {
 549           if (ip->protocol == IP_PROTOCOL_TCP)
 550             {
 551               nat44_set_tcp_session_state_i2o (sm, now, s, b, thread_index);
 552             }
 553           /* Accounting */
 554           nat44_session_update_counters (s, now,
 555                                          vlib_buffer_length_in_chain (vm, b),
 556                                          thread_index);
 557           /* Per-user LRU list maintenance */
 558           nat44_session_update_lru (sm, s, thread_index);
 559           return 1;
 560         }
 561       else
 562         return 0;
 563     }
 564
 565   return 0;
 566 }
 567
 568 static_always_inline int
 569 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 570                                        u16 src_port, u16 dst_port,
 571                                        u32 thread_index, u32 rx_sw_if_index,
 572                                        u32 tx_sw_if_index, f64 now)
 573 {
 574   clib_bihash_kv_16_8_t kv, value;
 575   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 576   snat_interface_t *i;
 577   snat_session_t *s;
 578   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 579   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 580
 581   /* src NAT check */
 582   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 583              tx_fib_index, ip->protocol);
 584   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 585     {
 586       ASSERT (thread_index == ed_value_get_thread_index (&value));
 587       s =
 588         pool_elt_at_index (tsm->sessions,
 589                            ed_value_get_session_index (&value));
 590       if (nat44_is_ses_closed (s)
 591           && (!s->tcp_closed_timestamp || now >= s->tcp_closed_timestamp))
 592         {
 593           nat_free_session_data (sm, s, thread_index, 0);
 594           nat_ed_session_delete (sm, s, thread_index, 1);
 595         }
 596       return 1;
 597     }
 598
 599   /* dst NAT check */
 600   init_ed_k (&kv, ip->dst_address, dst_port, ip->src_address, src_port,
 601              rx_fib_index, ip->protocol);
 602   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 603     {
 604       ASSERT (thread_index == ed_value_get_thread_index (&value));
 605       s =
 606         pool_elt_at_index (tsm->sessions,
 607                            ed_value_get_session_index (&value));
 608
 609       if (is_fwd_bypass_session (s))
 610         return 0;
 611
 612       /* hairpinning */
 613       /* *INDENT-OFF* */
 614       pool_foreach (i, sm->output_feature_interfaces)
 615        {
 616         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 617            return 0;
 618       }
 619       /* *INDENT-ON* */
 620       return 1;
 621     }
 622
 623   return 0;
 624 }
 625
 626 static inline u32
 627 icmp_in2out_ed_slow_path (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
 628                           icmp46_header_t *icmp, u32 sw_if_index,
 629                           u32 rx_fib_index, vlib_node_runtime_t *node,
 630                           u32 next, f64 now, u32 thread_index,
 631                           nat_protocol_t nat_proto, snat_session_t **s_p)
 632 {
 633   vlib_main_t *vm = vlib_get_main ();
 634   u16 checksum;
 635   int err;
 636   snat_session_t *s = NULL;
 637   u8 lookup_protocol = ip->protocol;
 638   u16 lookup_sport, lookup_dport;
 639   ip4_address_t lookup_saddr, lookup_daddr;
 640
 641   err = nat_get_icmp_session_lookup_values (b, ip, &lookup_saddr,
 642                                             &lookup_sport, &lookup_daddr,
 643                                             &lookup_dport, &lookup_protocol);
 644   if (err != 0)
 645     {
 646       b->error = node->errors[err];
 647       return NAT_NEXT_DROP;
 648     }
 649
 650   if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 651     {
 652       if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (
 653             sm, ip, lookup_sport, lookup_dport, thread_index, sw_if_index,
 654             vnet_buffer (b)->sw_if_index[VLIB_TX], now)))
 655         {
 656           return next;
 657         }
 658     }
 659   else
 660     {
 661       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index, ip,
 662                                                  NAT_PROTOCOL_ICMP,
 663                                                  rx_fib_index, thread_index)))
 664         {
 665           return next;
 666         }
 667     }
 668
 669   if (PREDICT_FALSE (icmp_type_is_error_message (
 670         vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 671     {
 672       b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 673       return NAT_NEXT_DROP;
 674     }
 675
 676   next = slow_path_ed (sm, b, ip->src_address, ip->dst_address, lookup_sport,
 677                        lookup_dport, ip->protocol, rx_fib_index, &s, node,
 678                        next, thread_index, vlib_time_now (vm));
 679
 680   if (NAT_NEXT_DROP == next)
 681     goto out;
 682
 683   if (PREDICT_TRUE (!ip4_is_fragment (ip)))
 684     {
 685       ip_csum_t sum = ip_incremental_checksum_buffer (
 686         vm, b, (u8 *) icmp - (u8 *) vlib_buffer_get_current (b),
 687         ntohs (ip->length) - ip4_header_bytes (ip), 0);
 688       checksum = ~ip_csum_fold (sum);
 689       if (PREDICT_FALSE (checksum != 0 && checksum != 0xffff))
 690         {
 691           next = NAT_NEXT_DROP;
 692           goto out;
 693         }
 694     }
 695
 696 out:
 697   if (PREDICT_TRUE (next != NAT_NEXT_DROP && s))
 698     {
 699       /* Accounting */
 700       nat44_session_update_counters (
 701         s, now, vlib_buffer_length_in_chain (vm, b), thread_index);
 702       /* Per-user LRU list maintenance */
 703       nat44_session_update_lru (sm, s, thread_index);
 704     }
 705   *s_p = s;
 706   return next;
 707 }
 708
 709 static snat_session_t *
 710 nat44_ed_in2out_slowpath_unknown_proto (snat_main_t *sm, vlib_buffer_t *b,
 711                                         ip4_header_t *ip, u32 rx_fib_index,
 712                                         u32 thread_index, f64 now,
 713                                         vlib_main_t *vm,
 714                                         vlib_node_runtime_t *node)
 715 {
 716   clib_bihash_kv_8_8_t kv, value;
 717   clib_bihash_kv_16_8_t s_kv, s_value;
 718   snat_static_mapping_t *m = NULL;
 719   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 720   snat_session_t *s = NULL;
 721   u32 outside_fib_index = sm->outside_fib_index;
 722   int i;
 723   ip4_address_t new_src_addr = { 0 };
 724   ip4_address_t new_dst_addr = ip->dst_address;
 725
 726   if (PREDICT_FALSE (
 727         nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 728     {
 729       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 730       nat_ipfix_logging_max_sessions (thread_index,
 731                                       sm->max_translations_per_thread);
 732       nat_elog_notice ("maximum sessions exceeded");
 733       return 0;
 734     }
 735
 736   switch (vec_len (sm->outside_fibs))
 737     {
 738     case 0:
 739       outside_fib_index = sm->outside_fib_index;
 740       break;
 741     case 1:
 742       outside_fib_index = sm->outside_fibs[0].fib_index;
 743       break;
 744     default:
 745       outside_fib_index = nat_outside_fib_index_lookup (sm, ip->dst_address);
 746       break;
 747     }
 748
 749   init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 750
 751   /* Try to find static mapping first */
 752   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 753     {
 754       m = pool_elt_at_index (sm->static_mappings, value.value);
 755       new_src_addr = m->external_addr;
 756     }
 757   else
 758     {
 759       pool_foreach (s, tsm->sessions)
 760         {
 761           if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 762             {
 763               init_ed_k (&s_kv, s->out2in.addr, 0, ip->dst_address, 0,
 764                          outside_fib_index, ip->protocol);
 765               if (clib_bihash_search_16_8 (&sm->flow_hash, &s_kv, &s_value))
 766                 {
 767                   new_src_addr = s->out2in.addr;
 768                 }
 769               break;
 770             }
 771         }
 772
 773       if (!new_src_addr.as_u32)
 774         {
 775           for (i = 0; i < vec_len (sm->addresses); i++)
 776             {
 777               init_ed_k (&s_kv, sm->addresses[i].addr, 0, ip->dst_address, 0,
 778                          outside_fib_index, ip->protocol);
 779               if (clib_bihash_search_16_8 (&sm->flow_hash, &s_kv, &s_value))
 780                 {
 781                   new_src_addr = sm->addresses[i].addr;
 782                 }
 783             }
 784         }
 785     }
 786
 787   if (!new_src_addr.as_u32)
 788     {
 789       // could not allocate address for translation ...
 790       return 0;
 791     }
 792
 793   s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 794   if (!s)
 795     {
 796       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 797       nat_elog_warn ("create NAT session failed");
 798       return 0;
 799     }
 800
 801   nat_6t_i2o_flow_init (sm, thread_index, s, ip->src_address, 0,
 802                         ip->dst_address, 0, rx_fib_index, ip->protocol);
 803   nat_6t_flow_saddr_rewrite_set (&s->i2o, new_src_addr.as_u32);
 804   nat_6t_flow_txfib_rewrite_set (&s->i2o, outside_fib_index);
 805
 806   // hairpinning?
 807   int is_hairpinning =
 808     nat44_ed_external_sm_lookup (sm, ip->dst_address, 0, NAT_PROTOCOL_OTHER,
 809                                  outside_fib_index, &new_dst_addr, NULL);
 810   s->flags |= is_hairpinning * SNAT_SESSION_FLAG_HAIRPINNING;
 811
 812   nat_6t_flow_daddr_rewrite_set (&s->i2o, new_dst_addr.as_u32);
 813   nat_6t_flow_txfib_rewrite_set (&s->i2o, outside_fib_index);
 814
 815   nat_6t_o2i_flow_init (sm, thread_index, s, new_dst_addr, 0, new_src_addr, 0,
 816                         outside_fib_index, ip->protocol);
 817   nat_6t_flow_saddr_rewrite_set (&s->o2i, ip->dst_address.as_u32);
 818   nat_6t_flow_daddr_rewrite_set (&s->o2i, ip->src_address.as_u32);
 819   nat_6t_flow_txfib_rewrite_set (&s->o2i, rx_fib_index);
 820
 821   s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 822   s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 823   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 824   s->out2in.addr.as_u32 = new_src_addr.as_u32;
 825   s->out2in.fib_index = outside_fib_index;
 826   s->in2out.addr.as_u32 = ip->src_address.as_u32;
 827   s->in2out.fib_index = rx_fib_index;
 828   s->in2out.port = s->out2in.port = ip->protocol;
 829   if (m)
 830     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 831
 832   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 833     {
 834       nat_elog_notice ("in2out flow hash add failed");
 835       nat_ed_session_delete (sm, s, thread_index, 1);
 836       return NULL;
 837     }
 838
 839   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 840     {
 841       nat_elog_notice ("out2in flow hash add failed");
 842       nat_ed_session_delete (sm, s, thread_index, 1);
 843       return NULL;
 844     }
 845
 846   per_vrf_sessions_register_session (s, thread_index);
 847
 848   /* Accounting */
 849   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 850                                  thread_index);
 851   /* Per-user LRU list maintenance */
 852   nat44_session_update_lru (sm, s, thread_index);
 853
 854   return s;
 855 }
 856
 857 static inline uword
 858 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 859                                           vlib_node_runtime_t * node,
 860                                           vlib_frame_t * frame,
 861                                           int is_output_feature)
 862 {
 863   u32 n_left_from, *from;
 864   snat_main_t *sm = &snat_main;
 865   f64 now = vlib_time_now (vm);
 866   u32 thread_index = vm->thread_index;
 867   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 868   u32 def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH
 869     : NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 870
 871   from = vlib_frame_vector_args (frame);
 872   n_left_from = frame->n_vectors;
 873
 874   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 875   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 876   vlib_get_buffers (vm, from, b, n_left_from);
 877
 878   while (n_left_from > 0)
 879     {
 880       vlib_buffer_t *b0;
 881       u32 sw_if_index0, rx_fib_index0, iph_offset0 = 0;
 882       nat_protocol_t proto0;
 883       ip4_header_t *ip0;
 884       snat_session_t *s0 = 0;
 885       clib_bihash_kv_16_8_t kv0, value0;
 886       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
 887       nat_6t_flow_t *f = 0;
 888       ip4_address_t lookup_saddr, lookup_daddr;
 889       u16 lookup_sport, lookup_dport;
 890       u8 lookup_protocol;
 891       int lookup_skipped = 0;
 892
 893       b0 = *b;
 894       b++;
 895
 896       /* Prefetch next iteration. */
 897       if (PREDICT_TRUE (n_left_from >= 2))
 898         {
 899           vlib_buffer_t *p2;
 900
 901           p2 = *b;
 902
 903           vlib_prefetch_buffer_header (p2, LOAD);
 904
 905           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 906         }
 907
 908       if (is_output_feature)
 909         {
 910           iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 911         }
 912
 913       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 914
 915       ip0 =
 916         (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
 917
 918       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 919       rx_fib_index0 =
 920         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 921
 922       if (PREDICT_FALSE (ip0->ttl == 1))
 923         {
 924           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 925           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 926                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 927                                        0);
 928           next[0] = NAT_NEXT_ICMP_ERROR;
 929           goto trace0;
 930         }
 931
 932       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 933
 934       if (is_output_feature)
 935         {
 936           if (PREDICT_FALSE
 937               (nat_not_translate_output_feature_fwd
 938                (sm, ip0, thread_index, now, vm, b0)))
 939             goto trace0;
 940         }
 941
 942       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 943         {
 944           if (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 945                 ICMP4_echo_request &&
 946               vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 947                 ICMP4_echo_reply &&
 948               !icmp_type_is_error_message (
 949                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 950             {
 951               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 952               next[0] = NAT_NEXT_DROP;
 953               goto trace0;
 954             }
 955           int err = nat_get_icmp_session_lookup_values (
 956             b0, ip0, &lookup_saddr, &lookup_sport, &lookup_daddr,
 957             &lookup_dport, &lookup_protocol);
 958           if (err != 0)
 959             {
 960               b0->error = node->errors[err];
 961               next[0] = NAT_NEXT_DROP;
 962               goto trace0;
 963             }
 964         }
 965       else
 966         {
 967           lookup_protocol = ip0->protocol;
 968           lookup_saddr = ip0->src_address;
 969           lookup_daddr = ip0->dst_address;
 970           lookup_sport = vnet_buffer (b0)->ip.reass.l4_src_port;
 971           lookup_dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
 972         }
 973
 974       /* there might be a stashed index in vnet_buffer2 from handoff or
 975        * classify node, see if it can be used */
 976       if (!pool_is_free_index (tsm->sessions,
 977                                vnet_buffer2 (b0)->nat.cached_session_index))
 978         {
 979           s0 = pool_elt_at_index (tsm->sessions,
 980                                   vnet_buffer2 (b0)->nat.cached_session_index);
 981           if (PREDICT_TRUE (
 982                 nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
 983                                    lookup_daddr, lookup_dport, lookup_protocol,
 984                                    rx_fib_index0)
 985                 // for some hairpinning cases there are two "i2i" flows instead
 986                 // of i2o and o2i as both hosts are on inside
 987                 || (s0->flags & SNAT_SESSION_FLAG_HAIRPINNING &&
 988                     nat_6t_flow_match (
 989                       &s0->o2i, b0, lookup_saddr, lookup_sport, lookup_daddr,
 990                       lookup_dport, lookup_protocol, rx_fib_index0))))
 991             {
 992               /* yes, this is the droid we're looking for */
 993               lookup_skipped = 1;
 994               goto skip_lookup;
 995             }
 996           s0 = NULL;
 997         }
 998
 999       init_ed_k (&kv0, ip0->src_address, lookup_sport, ip0->dst_address,
1000                  lookup_dport, rx_fib_index0, lookup_protocol);
1001
1002       // lookup flow
1003       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
1004         {
1005           // flow does not exist go slow path
1006           next[0] = def_slow;
1007           goto trace0;
1008         }
1009
1010       ASSERT (thread_index == ed_value_get_thread_index (&value0));
1011       s0 =
1012         pool_elt_at_index (tsm->sessions,
1013                            ed_value_get_session_index (&value0));
1014
1015     skip_lookup:
1016
1017       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
1018         {
1019           // session is closed, go slow path
1020           nat_free_session_data (sm, s0, thread_index, 0);
1021           nat_ed_session_delete (sm, s0, thread_index, 1);
1022           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1023           goto trace0;
1024         }
1025
1026       if (s0->tcp_closed_timestamp)
1027         {
1028           if (now >= s0->tcp_closed_timestamp)
1029             {
1030               // session is closed, go slow path, freed in slow path
1031               next[0] = def_slow;
1032             }
1033           else
1034             {
1035               // session in transitory timeout, drop
1036               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
1037               next[0] = NAT_NEXT_DROP;
1038             }
1039           goto trace0;
1040         }
1041
1042       // drop if session expired
1043       u64 sess_timeout_time;
1044       sess_timeout_time =
1045         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
1046       if (now >= sess_timeout_time)
1047         {
1048           nat_free_session_data (sm, s0, thread_index, 0);
1049           nat_ed_session_delete (sm, s0, thread_index, 1);
1050           // session is closed, go slow path
1051           next[0] = def_slow;
1052           goto trace0;
1053         }
1054
1055       b0->flags |= VNET_BUFFER_F_IS_NATED;
1056
1057       if (nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
1058                              lookup_daddr, lookup_dport, lookup_protocol,
1059                              rx_fib_index0))
1060         {
1061           f = &s0->i2o;
1062         }
1063       else if (s0->flags & SNAT_SESSION_FLAG_HAIRPINNING &&
1064                nat_6t_flow_match (&s0->o2i, b0, lookup_saddr, lookup_sport,
1065                                   lookup_daddr, lookup_dport, lookup_protocol,
1066                                   rx_fib_index0))
1067         {
1068           f = &s0->o2i;
1069         }
1070       else
1071         {
1072           translation_error = NAT_ED_TRNSL_ERR_FLOW_MISMATCH;
1073           nat_free_session_data (sm, s0, thread_index, 0);
1074           nat_ed_session_delete (sm, s0, thread_index, 1);
1075           next[0] = NAT_NEXT_DROP;
1076           goto trace0;
1077         }
1078
1079       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1080           (translation_error = nat_6t_flow_buf_translate (
1081              sm, b0, ip0, f, proto0, is_output_feature)))
1082         {
1083           nat_free_session_data (sm, s0, thread_index, 0);
1084           nat_ed_session_delete (sm, s0, thread_index, 1);
1085           next[0] = NAT_NEXT_DROP;
1086           goto trace0;
1087         }
1088
1089       switch (proto0)
1090         {
1091         case NAT_PROTOCOL_TCP:
1092           vlib_increment_simple_counter (&sm->counters.fastpath.in2out_ed.tcp,
1093                                          thread_index, sw_if_index0, 1);
1094           nat44_set_tcp_session_state_i2o (sm, now, s0, b0, thread_index);
1095           break;
1096         case NAT_PROTOCOL_UDP:
1097           vlib_increment_simple_counter (&sm->counters.fastpath.in2out_ed.udp,
1098                                          thread_index, sw_if_index0, 1);
1099           break;
1100         case NAT_PROTOCOL_ICMP:
1101           vlib_increment_simple_counter (&sm->counters.fastpath.in2out_ed.icmp,
1102                                          thread_index, sw_if_index0, 1);
1103           break;
1104         case NAT_PROTOCOL_OTHER:
1105           vlib_increment_simple_counter (
1106             &sm->counters.fastpath.in2out_ed.other, thread_index, sw_if_index0,
1107             1);
1108           break;
1109         }
1110
1111       /* Accounting */
1112       nat44_session_update_counters (s0, now,
1113                                      vlib_buffer_length_in_chain (vm, b0),
1114                                      thread_index);
1115       /* Per-user LRU list maintenance */
1116       nat44_session_update_lru (sm, s0, thread_index);
1117
1118     trace0:
1119       if (PREDICT_FALSE
1120           ((node->flags & VLIB_NODE_FLAG_TRACE)
1121            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1122         {
1123           nat_in2out_ed_trace_t *t =
1124             vlib_add_trace (vm, node, b0, sizeof (*t));
1125           t->sw_if_index = sw_if_index0;
1126           t->next_index = next[0];
1127           t->is_slow_path = 0;
1128           t->translation_error = translation_error;
1129           t->lookup_skipped = lookup_skipped;
1130           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1131
1132           if (s0)
1133             {
1134               t->session_index = s0 - tsm->sessions;
1135               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1136               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1137               t->translation_via_i2of = (&s0->i2o == f);
1138             }
1139           else
1140             {
1141               t->session_index = ~0;
1142             }
1143         }
1144
1145       if (next[0] == NAT_NEXT_DROP)
1146         {
1147           vlib_increment_simple_counter (&sm->counters.fastpath.
1148                                          in2out_ed.drops, thread_index,
1149                                          sw_if_index0, 1);
1150         }
1151
1152       n_left_from--;
1153       next++;
1154     }
1155
1156   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1157                                frame->n_vectors);
1158   return frame->n_vectors;
1159 }
1160
1161 static inline uword
1162 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1163                                           vlib_node_runtime_t * node,
1164                                           vlib_frame_t * frame,
1165                                           int is_output_feature)
1166 {
1167   u32 n_left_from, *from;
1168   snat_main_t *sm = &snat_main;
1169   f64 now = vlib_time_now (vm);
1170   u32 thread_index = vm->thread_index;
1171   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1172
1173   from = vlib_frame_vector_args (frame);
1174   n_left_from = frame->n_vectors;
1175
1176   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1177   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1178   vlib_get_buffers (vm, from, b, n_left_from);
1179
1180   while (n_left_from > 0)
1181     {
1182       vlib_buffer_t *b0;
1183       u32 sw_if_index0, rx_fib_index0, iph_offset0 = 0;
1184       nat_protocol_t proto0;
1185       ip4_header_t *ip0;
1186       udp_header_t *udp0;
1187       icmp46_header_t *icmp0;
1188       snat_session_t *s0 = 0;
1189       clib_bihash_kv_16_8_t kv0, value0;
1190       int translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
1191
1192       b0 = *b;
1193
1194       if (is_output_feature)
1195         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1196
1197       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1198
1199       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1200                               iph_offset0);
1201
1202       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1203       rx_fib_index0 =
1204         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1205
1206       if (PREDICT_FALSE (ip0->ttl == 1))
1207         {
1208           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1209           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1210                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1211                                        0);
1212           next[0] = NAT_NEXT_ICMP_ERROR;
1213           goto trace0;
1214         }
1215
1216       udp0 = ip4_next_header (ip0);
1217       icmp0 = (icmp46_header_t *) udp0;
1218       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1219
1220       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1221         {
1222           s0 = nat44_ed_in2out_slowpath_unknown_proto (
1223             sm, b0, ip0, rx_fib_index0, thread_index, now, vm, node);
1224           if (!s0)
1225             next[0] = NAT_NEXT_DROP;
1226
1227           if (NAT_ED_TRNSL_ERR_SUCCESS !=
1228               (translation_error = nat_6t_flow_buf_translate (
1229                  sm, b0, ip0, &s0->i2o, proto0, is_output_feature)))
1230             {
1231               goto trace0;
1232             }
1233
1234           vlib_increment_simple_counter (&sm->counters.slowpath.
1235                                          in2out_ed.other, thread_index,
1236                                          sw_if_index0, 1);
1237           goto trace0;
1238         }
1239
1240       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1241         {
1242           next[0] = icmp_in2out_ed_slow_path (sm, b0, ip0, icmp0, sw_if_index0,
1243                                               rx_fib_index0, node, next[0],
1244                                               now, thread_index, proto0, &s0);
1245           if (NAT_NEXT_DROP != next[0] && s0 &&
1246               NAT_ED_TRNSL_ERR_SUCCESS !=
1247                 (translation_error = nat_6t_flow_buf_translate (
1248                    sm, b0, ip0, &s0->i2o, proto0, is_output_feature)))
1249             {
1250               goto trace0;
1251             }
1252
1253           vlib_increment_simple_counter (&sm->counters.slowpath.
1254                                          in2out_ed.icmp, thread_index,
1255                                          sw_if_index0, 1);
1256           goto trace0;
1257         }
1258
1259       init_ed_k (&kv0, ip0->src_address,
1260                  vnet_buffer (b0)->ip.reass.l4_src_port, ip0->dst_address,
1261                  vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1262                  ip0->protocol);
1263       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
1264         {
1265           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1266           s0 =
1267             pool_elt_at_index (tsm->sessions,
1268                                ed_value_get_session_index (&value0));
1269
1270           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1271             {
1272               nat_free_session_data (sm, s0, thread_index, 0);
1273               nat_ed_session_delete (sm, s0, thread_index, 1);
1274               s0 = NULL;
1275             }
1276         }
1277
1278       if (!s0)
1279         {
1280           if (is_output_feature)
1281             {
1282               if (PREDICT_FALSE
1283                   (nat44_ed_not_translate_output_feature
1284                    (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1285                     vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1286                     sw_if_index0, vnet_buffer (b0)->sw_if_index[VLIB_TX],
1287                     now)))
1288                 goto trace0;
1289
1290               /*
1291                * Send DHCP packets to the ipv4 stack, or we won't
1292                * be able to use dhcp client on the outside interface
1293                */
1294               if (PREDICT_FALSE
1295                   (proto0 == NAT_PROTOCOL_UDP
1296                    && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1297                        clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1298                    && ip0->dst_address.as_u32 == 0xffffffff))
1299                 goto trace0;
1300             }
1301           else
1302             {
1303               if (PREDICT_FALSE
1304                   (nat44_ed_not_translate
1305                    (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1306                     thread_index)))
1307                 goto trace0;
1308             }
1309
1310           next[0] =
1311             slow_path_ed (sm, b0, ip0->src_address, ip0->dst_address,
1312                           vnet_buffer (b0)->ip.reass.l4_src_port,
1313                           vnet_buffer (b0)->ip.reass.l4_dst_port,
1314                           ip0->protocol, rx_fib_index0, &s0, node, next[0],
1315                           thread_index, now);
1316
1317           if (PREDICT_FALSE (next[0] == NAT_NEXT_DROP))
1318             goto trace0;
1319
1320           if (PREDICT_FALSE (!s0))
1321             goto trace0;
1322
1323         }
1324
1325       b0->flags |= VNET_BUFFER_F_IS_NATED;
1326
1327       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1328           (translation_error = nat_6t_flow_buf_translate (
1329              sm, b0, ip0, &s0->i2o, proto0, is_output_feature)))
1330         {
1331           nat_free_session_data (sm, s0, thread_index, 0);
1332           nat_ed_session_delete (sm, s0, thread_index, 1);
1333           s0 = NULL;
1334           goto trace0;
1335         }
1336
1337       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1338         {
1339           vlib_increment_simple_counter (&sm->counters.slowpath.in2out_ed.tcp,
1340                                          thread_index, sw_if_index0, 1);
1341           nat44_set_tcp_session_state_i2o (sm, now, s0, b0, thread_index);
1342         }
1343       else
1344         {
1345           vlib_increment_simple_counter (&sm->counters.slowpath.in2out_ed.udp,
1346                                          thread_index, sw_if_index0, 1);
1347         }
1348
1349       /* Accounting */
1350       nat44_session_update_counters (s0, now,
1351                                      vlib_buffer_length_in_chain
1352                                      (vm, b0), thread_index);
1353       /* Per-user LRU list maintenance */
1354       nat44_session_update_lru (sm, s0, thread_index);
1355
1356     trace0:
1357       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1358                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1359         {
1360           nat_in2out_ed_trace_t *t =
1361             vlib_add_trace (vm, node, b0, sizeof (*t));
1362           t->sw_if_index = sw_if_index0;
1363           t->next_index = next[0];
1364           t->is_slow_path = 1;
1365           t->translation_error = translation_error;
1366           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1367
1368           if (s0)
1369             {
1370               t->session_index = s0 - tsm->sessions;
1371               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1372               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1373               t->translation_via_i2of = 1;
1374             }
1375
1376           else
1377             {
1378               t->session_index = ~0;
1379             }
1380         }
1381
1382       if (next[0] == NAT_NEXT_DROP)
1383         {
1384           vlib_increment_simple_counter (&sm->counters.slowpath.
1385                                          in2out_ed.drops, thread_index,
1386                                          sw_if_index0, 1);
1387         }
1388
1389       n_left_from--;
1390       next++;
1391       b++;
1392     }
1393
1394   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1395                                frame->n_vectors);
1396
1397   return frame->n_vectors;
1398 }
1399
1400 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1401                                      vlib_node_runtime_t * node,
1402                                      vlib_frame_t * frame)
1403 {
1404   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1405 }
1406
1407 /* *INDENT-OFF* */
1408 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1409   .name = "nat44-ed-in2out",
1410   .vector_size = sizeof (u32),
1411   .sibling_of = "nat-default",
1412   .format_trace = format_nat_in2out_ed_trace,
1413   .type = VLIB_NODE_TYPE_INTERNAL,
1414   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1415   .error_strings = nat_in2out_ed_error_strings,
1416   .runtime_data_bytes = sizeof (snat_runtime_t),
1417 };
1418 /* *INDENT-ON* */
1419
1420 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1421                                             vlib_node_runtime_t * node,
1422                                             vlib_frame_t * frame)
1423 {
1424   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1425 }
1426
1427 /* *INDENT-OFF* */
1428 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1429   .name = "nat44-ed-in2out-output",
1430   .vector_size = sizeof (u32),
1431   .sibling_of = "nat-default",
1432   .format_trace = format_nat_in2out_ed_trace,
1433   .type = VLIB_NODE_TYPE_INTERNAL,
1434   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1435   .error_strings = nat_in2out_ed_error_strings,
1436   .runtime_data_bytes = sizeof (snat_runtime_t),
1437 };
1438 /* *INDENT-ON* */
1439
1440 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1441                                               vlib_node_runtime_t *
1442                                               node, vlib_frame_t * frame)
1443 {
1444   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1445 }
1446
1447 /* *INDENT-OFF* */
1448 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1449   .name = "nat44-ed-in2out-slowpath",
1450   .vector_size = sizeof (u32),
1451   .sibling_of = "nat-default",
1452   .format_trace = format_nat_in2out_ed_trace,
1453   .type = VLIB_NODE_TYPE_INTERNAL,
1454   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1455   .error_strings = nat_in2out_ed_error_strings,
1456   .runtime_data_bytes = sizeof (snat_runtime_t),
1457 };
1458 /* *INDENT-ON* */
1459
1460 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1461                                                      vlib_node_runtime_t
1462                                                      * node,
1463                                                      vlib_frame_t * frame)
1464 {
1465   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1466 }
1467
1468 /* *INDENT-OFF* */
1469 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1470   .name = "nat44-ed-in2out-output-slowpath",
1471   .vector_size = sizeof (u32),
1472   .sibling_of = "nat-default",
1473   .format_trace = format_nat_in2out_ed_trace,
1474   .type = VLIB_NODE_TYPE_INTERNAL,
1475   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1476   .error_strings = nat_in2out_ed_error_strings,
1477   .runtime_data_bytes = sizeof (snat_runtime_t),
1478 };
1479 /* *INDENT-ON* */
1480
1481 static u8 *
1482 format_nat_pre_trace (u8 * s, va_list * args)
1483 {
1484   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1485   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1486   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1487   return format (s, "in2out next_index %d arc_next_index %d", t->next_index,
1488                  t->arc_next_index);
1489 }
1490
1491 VLIB_NODE_FN (nat_pre_in2out_node)
1492   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1493 {
1494   return nat_pre_node_fn_inline (vm, node, frame,
1495                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1496 }
1497
1498 VLIB_NODE_FN (nat_pre_in2out_output_node)
1499   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1500 {
1501   return nat_pre_node_fn_inline (vm, node, frame,
1502                                  NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH);
1503 }
1504
1505 /* *INDENT-OFF* */
1506 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1507   .name = "nat-pre-in2out",
1508   .vector_size = sizeof (u32),
1509   .sibling_of = "nat-default",
1510   .format_trace = format_nat_pre_trace,
1511   .type = VLIB_NODE_TYPE_INTERNAL,
1512   .n_errors = 0,
1513 };
1514
1515 VLIB_REGISTER_NODE (nat_pre_in2out_output_node) = {
1516   .name = "nat-pre-in2out-output",
1517   .vector_size = sizeof (u32),
1518   .sibling_of = "nat-default",
1519   .format_trace = format_nat_pre_trace,
1520   .type = VLIB_NODE_TYPE_INTERNAL,
1521   .n_errors = 0,
1522 };
1523 /* *INDENT-ON* */
1524
1525 /*
1526  * fd.io coding-style-patch-verification: ON
1527  *
1528  * Local Variables:
1529  * eval: (c-set-style "gnu")
1530  * End:
1531  */