src/plugins/nat/nat44-ed/nat44_ed_out2in.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27
  28 #include <nat/lib/nat_syslog.h>
  29 #include <nat/lib/ipfix_logging.h>
  30
  31 #include <nat/nat44-ed/nat44_ed.h>
  32 #include <nat/nat44-ed/nat44_ed_inlines.h>
  33
  34 static char *nat_out2in_ed_error_strings[] = {
  35 #define _(sym,string) string,
  36   foreach_nat_out2in_ed_error
  37 #undef _
  38 };
  39
  40 typedef enum
  41 {
  42   NAT_ED_SP_REASON_NO_REASON,
  43   NAT_ED_SP_REASON_LOOKUP_FAILED,
  44   NAT_ED_SP_REASON_VRF_EXPIRED,
  45   NAT_ED_SP_TCP_CLOSED,
  46   NAT_ED_SP_SESS_EXPIRED,
  47 } nat_slow_path_reason_e;
  48
  49 typedef struct
  50 {
  51   u32 sw_if_index;
  52   u32 next_index;
  53   u32 session_index;
  54   nat_translation_error_e translation_error;
  55   nat_6t_flow_t i2of;
  56   nat_6t_flow_t o2if;
  57   clib_bihash_kv_16_8_t search_key;
  58   u8 is_slow_path;
  59   u8 translation_via_i2of;
  60   u8 lookup_skipped;
  61   nat_slow_path_reason_e slow_path_reason;
  62 } nat44_ed_out2in_trace_t;
  63
  64 static u8 *
  65 format_slow_path_reason (u8 *s, va_list *args)
  66 {
  67   nat_slow_path_reason_e reason = va_arg (*args, nat_slow_path_reason_e);
  68   switch (reason)
  69     {
  70     case NAT_ED_SP_REASON_NO_REASON:
  71       return format (s, "no reason for slow path");
  72     case NAT_ED_SP_REASON_LOOKUP_FAILED:
  73       return format (s, "slow path because lookup failed");
  74     case NAT_ED_SP_REASON_VRF_EXPIRED:
  75       return format (s, "slow path because vrf expired");
  76     case NAT_ED_SP_TCP_CLOSED:
  77       return format (s, "slow path because tcp closed");
  78     case NAT_ED_SP_SESS_EXPIRED:
  79       return format (s, "slow path because session expired");
  80     }
  81   return format (s, "invalid reason value");
  82 }
  83
  84 static u8 *
  85 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  86 {
  87   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  88   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  89   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  90   char *tag;
  91
  92   tag =
  93     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  94     "NAT44_OUT2IN_ED_FAST_PATH";
  95
  96   s = format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
  97               t->next_index);
  98   if (~0 != t->session_index)
  99     {
 100       s = format (s, ", session %d, translation result '%U' via %s",
 101                   t->session_index, format_nat_ed_translation_error,
 102                   t->translation_error,
 103                   t->translation_via_i2of ? "i2of" : "o2if");
 104       s = format (s, "\n  i2of %U", format_nat_6t_flow, &t->i2of);
 105       s = format (s, "\n  o2if %U", format_nat_6t_flow, &t->o2if);
 106     }
 107   if (!t->is_slow_path)
 108     {
 109       if (t->lookup_skipped)
 110         {
 111           s = format (s, "\n lookup skipped - cached session index used");
 112         }
 113       else
 114         {
 115           s = format (s, "\n  search key %U", format_ed_session_kvp,
 116                       &t->search_key);
 117         }
 118       s = format (s, "\n %U", format_slow_path_reason, t->slow_path_reason);
 119     }
 120
 121   return s;
 122 }
 123
 124 static int
 125 next_src_nat (snat_main_t *sm, ip4_header_t *ip, u16 src_port, u16 dst_port,
 126               u32 rx_fib_index)
 127 {
 128   clib_bihash_kv_16_8_t kv, value;
 129
 130   init_ed_k (&kv, ip->src_address, src_port, ip->dst_address, dst_port,
 131              rx_fib_index, ip->protocol);
 132   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 133     return 1;
 134
 135   return 0;
 136 }
 137
 138 static void create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b,
 139                                    snat_session_t *s, ip4_header_t *ip,
 140                                    u32 rx_fib_index, u32 thread_index);
 141
 142 static snat_session_t *create_session_for_static_mapping_ed (
 143   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 144   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 145   nat_protocol_t nat_proto, vlib_node_runtime_t *node, u32 rx_fib_index,
 146   u32 thread_index, twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 147   snat_static_mapping_t *mapping);
 148
 149 static inline u32
 150 icmp_out2in_ed_slow_path (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
 151                           icmp46_header_t *icmp, u32 sw_if_index,
 152                           u32 rx_fib_index, vlib_node_runtime_t *node,
 153                           u32 next, f64 now, u32 thread_index,
 154                           snat_session_t **s_p)
 155 {
 156   vlib_main_t *vm = vlib_get_main ();
 157
 158   ip_csum_t sum;
 159   u16 checksum;
 160
 161   snat_session_t *s = 0;
 162   u8 is_addr_only, identity_nat;
 163   ip4_address_t sm_addr;
 164   u16 sm_port;
 165   u32 sm_fib_index;
 166   snat_static_mapping_t *m;
 167   u8 lookup_protocol;
 168   ip4_address_t lookup_saddr, lookup_daddr;
 169   u16 lookup_sport, lookup_dport;
 170
 171   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 172   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 173
 174   if (nat_get_icmp_session_lookup_values (b, ip, &lookup_saddr, &lookup_sport,
 175                                           &lookup_daddr, &lookup_dport,
 176                                           &lookup_protocol))
 177     {
 178       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 179       next = NAT_NEXT_DROP;
 180       goto out;
 181     }
 182
 183   if (snat_static_mapping_match (
 184         vm, sm, ip->dst_address, lookup_sport, rx_fib_index,
 185         ip_proto_to_nat_proto (ip->protocol), &sm_addr, &sm_port,
 186         &sm_fib_index, 1, &is_addr_only, 0, 0, 0, &identity_nat, &m))
 187     {
 188       // static mapping not matched
 189       if (!sm->forwarding_enabled)
 190         {
 191           /* Don't NAT packet aimed at the intfc address */
 192           if (!is_interface_addr (sm, node, sw_if_index,
 193                                   ip->dst_address.as_u32))
 194             {
 195               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 196               next = NAT_NEXT_DROP;
 197             }
 198         }
 199       else
 200         {
 201           if (next_src_nat (sm, ip, lookup_sport, lookup_dport, rx_fib_index))
 202             {
 203               next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 204             }
 205           else
 206             {
 207               create_bypass_for_fwd (sm, b, s, ip, rx_fib_index, thread_index);
 208             }
 209         }
 210       goto out;
 211     }
 212
 213   if (PREDICT_FALSE (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 214                        ICMP4_echo_reply &&
 215                      (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 216                         ICMP4_echo_request ||
 217                       !is_addr_only)))
 218     {
 219       b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 220       next = NAT_NEXT_DROP;
 221       goto out;
 222     }
 223
 224   if (PREDICT_FALSE (identity_nat))
 225     {
 226       goto out;
 227     }
 228
 229   /* Create session initiated by host from external network */
 230   s = create_session_for_static_mapping_ed (
 231     sm, b, sm_addr, sm_port, sm_fib_index, ip->dst_address, lookup_sport,
 232     rx_fib_index, ip_proto_to_nat_proto (lookup_protocol), node, rx_fib_index,
 233     thread_index, 0, 0, vlib_time_now (vm), m);
 234   if (!s)
 235     next = NAT_NEXT_DROP;
 236
 237   if (PREDICT_TRUE (!ip4_is_fragment (ip)))
 238     {
 239       sum = ip_incremental_checksum_buffer (
 240         vm, b, (u8 *) icmp - (u8 *) vlib_buffer_get_current (b),
 241         ntohs (ip->length) - ip4_header_bytes (ip), 0);
 242       checksum = ~ip_csum_fold (sum);
 243       if (checksum != 0 && checksum != 0xffff)
 244         {
 245           next = NAT_NEXT_DROP;
 246           goto out;
 247         }
 248     }
 249
 250   if (PREDICT_TRUE (next != NAT_NEXT_DROP && s))
 251     {
 252       /* Accounting */
 253       nat44_session_update_counters (
 254         s, now, vlib_buffer_length_in_chain (vm, b), thread_index);
 255       /* Per-user LRU list maintenance */
 256       nat44_session_update_lru (sm, s, thread_index);
 257     }
 258 out:
 259   if (NAT_NEXT_DROP == next && s)
 260     {
 261       nat_ed_session_delete (sm, s, thread_index, 1);
 262       s = 0;
 263     }
 264   *s_p = s;
 265   return next;
 266 }
 267
 268 // allocate exact address based on preference
 269 static_always_inline int
 270 nat_alloc_addr_and_port_exact (snat_address_t * a,
 271                                u32 thread_index,
 272                                nat_protocol_t proto,
 273                                ip4_address_t * addr,
 274                                u16 * port,
 275                                u16 port_per_thread, u32 snat_thread_index)
 276 {
 277   snat_main_t *sm = &snat_main;
 278   u32 portnum;
 279
 280   switch (proto)
 281     {
 282 #define _(N, j, n, s) \
 283     case NAT_PROTOCOL_##N: \
 284       if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
 285         { \
 286           while (1) \
 287             { \
 288               portnum = (port_per_thread * \
 289                 snat_thread_index) + \
 290                 snat_random_port(0, port_per_thread - 1) + 1024; \
 291               if (a->busy_##n##_port_refcounts[portnum]) \
 292                 continue; \
 293               --a->busy_##n##_port_refcounts[portnum]; \
 294               a->busy_##n##_ports_per_thread[thread_index]++; \
 295               a->busy_##n##_ports++; \
 296               *addr = a->addr; \
 297               *port = clib_host_to_net_u16(portnum); \
 298               return 0; \
 299             } \
 300         } \
 301       break;
 302       foreach_nat_protocol
 303 #undef _
 304         default : nat_elog_info (sm, "unknown protocol");
 305       return 1;
 306     }
 307
 308   /* Totally out of translations to use... */
 309   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 310   return 1;
 311 }
 312
 313 static_always_inline int
 314 nat44_ed_alloc_outside_addr_and_port (snat_address_t *addresses, u32 fib_index,
 315                                       u32 thread_index, nat_protocol_t proto,
 316                                       ip4_address_t *addr, u16 *port,
 317                                       u16 port_per_thread,
 318                                       u32 snat_thread_index)
 319 {
 320   snat_main_t *sm = &snat_main;
 321   snat_address_t *a, *ga = 0;
 322   u32 portnum;
 323   int i;
 324
 325   for (i = 0; i < vec_len (addresses); i++)
 326     {
 327       a = addresses + i;
 328       switch (proto)
 329         {
 330 #define _(N, j, n, s)                                                         \
 331   case NAT_PROTOCOL_##N:                                                      \
 332     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
 333       {                                                                       \
 334         if (a->fib_index == fib_index)                                        \
 335           {                                                                   \
 336             while (1)                                                         \
 337               {                                                               \
 338                 portnum = (port_per_thread * snat_thread_index) +             \
 339                           snat_random_port (0, port_per_thread - 1) + 1024;   \
 340                 if (a->busy_##n##_port_refcounts[portnum])                    \
 341                   continue;                                                   \
 342                 --a->busy_##n##_port_refcounts[portnum];                      \
 343                 a->busy_##n##_ports_per_thread[thread_index]++;               \
 344                 a->busy_##n##_ports++;                                        \
 345                 *addr = a->addr;                                              \
 346                 *port = clib_host_to_net_u16 (portnum);                       \
 347                 return 0;                                                     \
 348               }                                                               \
 349           }                                                                   \
 350         else if (a->fib_index == ~0)                                          \
 351           {                                                                   \
 352             ga = a;                                                           \
 353           }                                                                   \
 354       }                                                                       \
 355     break;
 356           foreach_nat_protocol
 357 #undef _
 358             default : nat_elog_info (sm, "unknown protocol");
 359           return 1;
 360         }
 361     }
 362
 363   if (ga)
 364     {
 365       a = ga;
 366       switch (proto)
 367         {
 368 #define _(N, j, n, s)                                                         \
 369   case NAT_PROTOCOL_##N:                                                      \
 370     while (1)                                                                 \
 371       {                                                                       \
 372         portnum = (port_per_thread * snat_thread_index) +                     \
 373                   snat_random_port (0, port_per_thread - 1) + 1024;           \
 374         if (a->busy_##n##_port_refcounts[portnum])                            \
 375           continue;                                                           \
 376         ++a->busy_##n##_port_refcounts[portnum];                              \
 377         a->busy_##n##_ports_per_thread[thread_index]++;                       \
 378         a->busy_##n##_ports++;                                                \
 379         *addr = a->addr;                                                      \
 380         *port = clib_host_to_net_u16 (portnum);                               \
 381         return 0;                                                             \
 382       }
 383           break;
 384           foreach_nat_protocol
 385 #undef _
 386             default : nat_elog_info (sm, "unknown protocol");
 387           return 1;
 388         }
 389     }
 390
 391   /* Totally out of translations to use... */
 392   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 393   return 1;
 394 }
 395
 396 static snat_session_t *
 397 create_session_for_static_mapping_ed (
 398   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 399   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 400   nat_protocol_t nat_proto, vlib_node_runtime_t *node, u32 rx_fib_index,
 401   u32 thread_index, twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 402   snat_static_mapping_t *mapping)
 403 {
 404   snat_session_t *s;
 405   ip4_header_t *ip;
 406   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 407
 408   if (PREDICT_FALSE
 409       (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 410     {
 411       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 412       nat_elog_notice (sm, "maximum sessions exceeded");
 413       return 0;
 414     }
 415
 416   s = nat_ed_session_alloc (sm, thread_index, now, nat_proto);
 417   if (!s)
 418     {
 419       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 420       nat_elog_warn (sm, "create NAT session failed");
 421       return 0;
 422     }
 423
 424   ip = vlib_buffer_get_current (b);
 425
 426   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 427   s->ext_host_port =
 428     nat_proto == NAT_PROTOCOL_ICMP ? 0 : vnet_buffer (b)->ip.reass.l4_src_port;
 429   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 430   if (lb_nat)
 431     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 432   if (lb_nat == AFFINITY_LB_NAT)
 433     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 434   s->out2in.addr = o2i_addr;
 435   s->out2in.port = o2i_port;
 436   s->out2in.fib_index = o2i_fib_index;
 437   s->in2out.addr = i2o_addr;
 438   s->in2out.port = i2o_port;
 439   s->in2out.fib_index = i2o_fib_index;
 440   s->nat_proto = nat_proto;
 441
 442   if (NAT_PROTOCOL_ICMP == nat_proto)
 443     {
 444       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr, o2i_port,
 445                             o2i_addr, o2i_port, o2i_fib_index, ip->protocol);
 446       nat_6t_flow_icmp_id_rewrite_set (&s->o2i, i2o_port);
 447     }
 448   else
 449     {
 450       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr,
 451                             s->ext_host_port, o2i_addr, o2i_port,
 452                             o2i_fib_index, ip->protocol);
 453       nat_6t_flow_dport_rewrite_set (&s->o2i, i2o_port);
 454     }
 455   nat_6t_flow_daddr_rewrite_set (&s->o2i, i2o_addr.as_u32);
 456   nat_6t_flow_txfib_rewrite_set (&s->o2i, i2o_fib_index);
 457
 458   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 459     {
 460       b->error = node->errors[NAT_OUT2IN_ED_ERROR_HASH_ADD_FAILED];
 461       nat_ed_session_delete (sm, s, thread_index, 1);
 462       nat_elog_warn (sm, "out2in flow hash add failed");
 463       return 0;
 464     }
 465
 466   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 467                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 468     {
 469       int rc = 0;
 470       snat_address_t *filter = 0;
 471
 472       // if exact address is specified use this address
 473       if (is_sm_exact_address (mapping->flags))
 474         {
 475           snat_address_t *ap;
 476           vec_foreach (ap, sm->twice_nat_addresses)
 477           {
 478             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 479               {
 480                 filter = ap;
 481                 break;
 482               }
 483           }
 484         }
 485
 486       if (filter)
 487         {
 488           rc = nat_alloc_addr_and_port_exact (filter,
 489                                               thread_index,
 490                                               nat_proto,
 491                                               &s->ext_host_nat_addr,
 492                                               &s->ext_host_nat_port,
 493                                               sm->port_per_thread,
 494                                               tsm->snat_thread_index);
 495           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 496         }
 497       else
 498         {
 499           rc = nat44_ed_alloc_outside_addr_and_port (
 500             sm->twice_nat_addresses, 0, thread_index, nat_proto,
 501             &s->ext_host_nat_addr, &s->ext_host_nat_port, sm->port_per_thread,
 502             tsm->snat_thread_index);
 503         }
 504
 505       if (rc)
 506         {
 507           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 508           if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 509             {
 510               nat_elog_warn (sm, "out2in flow hash del failed");
 511             }
 512           snat_free_outside_address_and_port (
 513             sm->twice_nat_addresses, thread_index, &s->ext_host_nat_addr,
 514             s->ext_host_nat_port, s->nat_proto);
 515           nat_ed_session_delete (sm, s, thread_index, 1);
 516           return 0;
 517         }
 518
 519       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 520
 521       nat_6t_flow_saddr_rewrite_set (&s->o2i, s->ext_host_nat_addr.as_u32);
 522       if (NAT_PROTOCOL_ICMP == nat_proto)
 523         {
 524           nat_6t_flow_icmp_id_rewrite_set (&s->o2i, s->ext_host_nat_port);
 525         }
 526       else
 527         {
 528           nat_6t_flow_sport_rewrite_set (&s->o2i, s->ext_host_nat_port);
 529         }
 530
 531       nat_6t_l3_l4_csum_calc (&s->o2i);
 532
 533       nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 534                             s->ext_host_nat_addr, s->ext_host_nat_port,
 535                             i2o_fib_index, ip->protocol);
 536       nat_6t_flow_daddr_rewrite_set (&s->i2o, s->ext_host_addr.as_u32);
 537       if (NAT_PROTOCOL_ICMP == nat_proto)
 538         {
 539           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, s->ext_host_port);
 540         }
 541       else
 542         {
 543           nat_6t_flow_dport_rewrite_set (&s->i2o, s->ext_host_port);
 544         }
 545     }
 546   else
 547     {
 548       if (NAT_PROTOCOL_ICMP == nat_proto)
 549         {
 550           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 551                                 s->ext_host_addr, i2o_port, i2o_fib_index,
 552                                 ip->protocol);
 553         }
 554       else
 555         {
 556           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 557                                 s->ext_host_addr, s->ext_host_port,
 558                                 i2o_fib_index, ip->protocol);
 559         }
 560     }
 561
 562   nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 563   if (NAT_PROTOCOL_ICMP == nat_proto)
 564     {
 565       nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 566     }
 567   else
 568     {
 569       nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 570     }
 571
 572   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 573     {
 574       nat_elog_notice (sm, "in2out flow hash add failed");
 575       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 576         {
 577           nat_elog_warn (sm, "out2in flow hash del failed");
 578         }
 579       nat_ed_session_delete (sm, s, thread_index, 1);
 580       return 0;
 581     }
 582
 583   nat_ipfix_logging_nat44_ses_create (thread_index,
 584                                       s->in2out.addr.as_u32,
 585                                       s->out2in.addr.as_u32,
 586                                       s->nat_proto,
 587                                       s->in2out.port,
 588                                       s->out2in.port, s->in2out.fib_index);
 589
 590   nat_syslog_nat44_sadd (0, s->in2out.fib_index, &s->in2out.addr,
 591                          s->in2out.port, &s->ext_host_nat_addr,
 592                          s->ext_host_nat_port, &s->out2in.addr, s->out2in.port,
 593                          &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 594                          nat44_ed_is_twice_nat_session (s));
 595
 596   per_vrf_sessions_register_session (s, thread_index);
 597
 598   return s;
 599 }
 600
 601 static void
 602 create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b, snat_session_t *s,
 603                        ip4_header_t *ip, u32 rx_fib_index, u32 thread_index)
 604 {
 605   clib_bihash_kv_16_8_t kv, value;
 606   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 607   vlib_main_t *vm = vlib_get_main ();
 608   f64 now = vlib_time_now (vm);
 609   u16 lookup_sport, lookup_dport;
 610   u8 lookup_protocol;
 611   ip4_address_t lookup_saddr, lookup_daddr;
 612
 613   if (ip->protocol == IP_PROTOCOL_ICMP)
 614     {
 615       if (nat_get_icmp_session_lookup_values (b, ip, &lookup_daddr,
 616                                               &lookup_sport, &lookup_saddr,
 617                                               &lookup_dport, &lookup_protocol))
 618         return;
 619     }
 620   else
 621     {
 622       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 623         {
 624           lookup_sport = vnet_buffer (b)->ip.reass.l4_dst_port;
 625           lookup_dport = vnet_buffer (b)->ip.reass.l4_src_port;
 626         }
 627       else
 628         {
 629           lookup_sport = 0;
 630           lookup_dport = 0;
 631         }
 632       lookup_saddr.as_u32 = ip->dst_address.as_u32;
 633       lookup_daddr.as_u32 = ip->src_address.as_u32;
 634       lookup_protocol = ip->protocol;
 635     }
 636
 637   init_ed_k (&kv, lookup_saddr, lookup_sport, lookup_daddr, lookup_dport,
 638              rx_fib_index, lookup_protocol);
 639
 640   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 641     {
 642       ASSERT (thread_index == ed_value_get_thread_index (&value));
 643       s =
 644         pool_elt_at_index (tsm->sessions,
 645                            ed_value_get_session_index (&value));
 646     }
 647   else if (ip->protocol == IP_PROTOCOL_ICMP &&
 648            icmp_type_is_error_message
 649            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 650     {
 651       return;
 652     }
 653   else
 654     {
 655       u32 proto;
 656
 657       if (PREDICT_FALSE
 658           (nat44_ed_maximum_sessions_exceeded
 659            (sm, rx_fib_index, thread_index)))
 660         return;
 661
 662       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 663       if (!s)
 664         {
 665           nat_elog_warn (sm, "create NAT session failed");
 666           return;
 667         }
 668
 669       proto = ip_proto_to_nat_proto (ip->protocol);
 670
 671       s->ext_host_addr = ip->src_address;
 672       s->ext_host_port = lookup_dport;
 673       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 674       s->out2in.addr = ip->dst_address;
 675       s->out2in.port = lookup_sport;
 676       s->nat_proto = proto;
 677       if (proto == NAT_PROTOCOL_OTHER)
 678         {
 679           s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 680           s->out2in.port = ip->protocol;
 681         }
 682       s->out2in.fib_index = rx_fib_index;
 683       s->in2out.addr = s->out2in.addr;
 684       s->in2out.port = s->out2in.port;
 685       s->in2out.fib_index = s->out2in.fib_index;
 686
 687       nat_6t_i2o_flow_init (sm, thread_index, s, ip->dst_address, lookup_sport,
 688                             ip->src_address, lookup_dport, rx_fib_index,
 689                             ip->protocol);
 690       nat_6t_flow_txfib_rewrite_set (&s->i2o, rx_fib_index);
 691       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 692         {
 693           nat_elog_notice (sm, "in2out flow add failed");
 694           nat_ed_session_delete (sm, s, thread_index, 1);
 695           return;
 696         }
 697
 698       per_vrf_sessions_register_session (s, thread_index);
 699     }
 700
 701   if (ip->protocol == IP_PROTOCOL_TCP)
 702     {
 703       tcp_header_t *tcp = ip4_next_header (ip);
 704       nat44_set_tcp_session_state_o2i (sm, now, s, tcp->flags,
 705                                        tcp->ack_number, tcp->seq_number,
 706                                        thread_index);
 707     }
 708
 709   /* Accounting */
 710   nat44_session_update_counters (s, now, 0, thread_index);
 711   /* Per-user LRU list maintenance */
 712   nat44_session_update_lru (sm, s, thread_index);
 713 }
 714
 715 static snat_session_t *
 716 nat44_ed_out2in_slowpath_unknown_proto (snat_main_t *sm, vlib_buffer_t *b,
 717                                         ip4_header_t *ip, u32 rx_fib_index,
 718                                         u32 thread_index, f64 now,
 719                                         vlib_main_t *vm,
 720                                         vlib_node_runtime_t *node)
 721 {
 722   clib_bihash_kv_8_8_t kv, value;
 723   snat_static_mapping_t *m;
 724   snat_session_t *s;
 725
 726   if (PREDICT_FALSE (
 727         nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 728     {
 729       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 730       nat_elog_notice (sm, "maximum sessions exceeded");
 731       return 0;
 732     }
 733
 734   init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 735   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 736     {
 737       b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 738       return 0;
 739     }
 740
 741   m = pool_elt_at_index (sm->static_mappings, value.value);
 742
 743   /* Create a new session */
 744   s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 745   if (!s)
 746     {
 747       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 748       nat_elog_warn (sm, "create NAT session failed");
 749       return 0;
 750     }
 751
 752   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 753   s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 754   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 755   s->out2in.addr.as_u32 = ip->dst_address.as_u32;
 756   s->out2in.fib_index = rx_fib_index;
 757   s->in2out.addr.as_u32 = m->local_addr.as_u32;
 758   s->in2out.fib_index = m->fib_index;
 759   s->in2out.port = s->out2in.port = ip->protocol;
 760
 761   nat_6t_o2i_flow_init (sm, thread_index, s, ip->dst_address, 0,
 762                         ip->src_address, 0, m->fib_index, ip->protocol);
 763   nat_6t_flow_saddr_rewrite_set (&s->i2o, ip->dst_address.as_u32);
 764   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 765     {
 766       nat_elog_notice (sm, "in2out key add failed");
 767       nat_ed_session_delete (sm, s, thread_index, 1);
 768       return NULL;
 769     }
 770
 771   nat_6t_o2i_flow_init (sm, thread_index, s, ip->src_address, 0,
 772                         ip->dst_address, 0, rx_fib_index, ip->protocol);
 773   nat_6t_flow_daddr_rewrite_set (&s->o2i, m->local_addr.as_u32);
 774   nat_6t_flow_txfib_rewrite_set (&s->o2i, m->fib_index);
 775   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 776     {
 777       nat_elog_notice (sm, "out2in flow hash add failed");
 778       nat_ed_session_delete (sm, s, thread_index, 1);
 779       return NULL;
 780     }
 781
 782   per_vrf_sessions_register_session (s, thread_index);
 783
 784   /* Accounting */
 785   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 786                                  thread_index);
 787   /* Per-user LRU list maintenance */
 788   nat44_session_update_lru (sm, s, thread_index);
 789
 790   return s;
 791 }
 792
 793 static inline uword
 794 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 795                                           vlib_node_runtime_t * node,
 796                                           vlib_frame_t * frame,
 797                                           int is_multi_worker)
 798 {
 799   u32 n_left_from, *from;
 800   snat_main_t *sm = &snat_main;
 801   f64 now = vlib_time_now (vm);
 802   u32 thread_index = vm->thread_index;
 803   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 804
 805   from = vlib_frame_vector_args (frame);
 806   n_left_from = frame->n_vectors;
 807
 808   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 809   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 810   vlib_get_buffers (vm, from, b, n_left_from);
 811
 812   while (n_left_from > 0)
 813     {
 814       vlib_buffer_t *b0;
 815       u32 sw_if_index0, rx_fib_index0;
 816       nat_protocol_t proto0;
 817       ip4_header_t *ip0;
 818       snat_session_t *s0 = 0;
 819       clib_bihash_kv_16_8_t kv0, value0;
 820       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
 821       nat_slow_path_reason_e slow_path_reason = NAT_ED_SP_REASON_NO_REASON;
 822       nat_6t_flow_t *f = 0;
 823       nat_6t_t lookup;
 824       int lookup_skipped = 0;
 825
 826       b0 = *b;
 827       b++;
 828
 829       /* Prefetch next iteration. */
 830       if (PREDICT_TRUE (n_left_from >= 2))
 831         {
 832           vlib_buffer_t *p2;
 833
 834           p2 = *b;
 835
 836           vlib_prefetch_buffer_header (p2, LOAD);
 837
 838           clib_prefetch_load (p2->data);
 839         }
 840
 841       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 842
 843       lookup.sport = vnet_buffer (b0)->ip.reass.l4_src_port;
 844       lookup.dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
 845
 846       vnet_buffer (b0)->snat.flags = 0;
 847       ip0 = vlib_buffer_get_current (b0);
 848
 849       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 850       rx_fib_index0 =
 851         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 852
 853       lookup.fib_index = rx_fib_index0;
 854
 855       if (PREDICT_FALSE (ip0->ttl == 1))
 856         {
 857           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 858           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 859                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 860                                        0);
 861           next[0] = NAT_NEXT_ICMP_ERROR;
 862           goto trace0;
 863         }
 864
 865       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 866
 867       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 868         {
 869           if (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 870                 ICMP4_echo_request &&
 871               vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 872                 ICMP4_echo_reply &&
 873               !icmp_type_is_error_message (
 874                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 875             {
 876               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 877               next[0] = NAT_NEXT_DROP;
 878               goto trace0;
 879             }
 880           int err = nat_get_icmp_session_lookup_values (
 881             b0, ip0, &lookup.saddr, &lookup.sport, &lookup.daddr,
 882             &lookup.dport, &lookup.proto);
 883           if (err != 0)
 884             {
 885               b0->error = node->errors[err];
 886               next[0] = NAT_NEXT_DROP;
 887               goto trace0;
 888             }
 889         }
 890       else
 891         {
 892           lookup.saddr.as_u32 = ip0->src_address.as_u32;
 893           lookup.daddr.as_u32 = ip0->dst_address.as_u32;
 894           lookup.proto = ip0->protocol;
 895         }
 896
 897       /* there might be a stashed index in vnet_buffer2 from handoff or
 898        * classify node, see if it can be used */
 899       if (is_multi_worker &&
 900           !pool_is_free_index (tsm->sessions,
 901                                vnet_buffer2 (b0)->nat.cached_session_index))
 902         {
 903           s0 = pool_elt_at_index (tsm->sessions,
 904                                   vnet_buffer2 (b0)->nat.cached_session_index);
 905           if (PREDICT_TRUE (nat_6t_t_eq (&s0->o2i.match, &lookup)) ||
 906               (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 907                nat_6t_t_eq (&s0->i2o.match, &lookup)))
 908             {
 909               /* yes, this is the droid we're looking for */
 910               lookup_skipped = 1;
 911               goto skip_lookup;
 912             }
 913           s0 = NULL;
 914         }
 915
 916       init_ed_k (&kv0, lookup.saddr, lookup.sport, lookup.daddr, lookup.dport,
 917                  lookup.fib_index, lookup.proto);
 918
 919       // lookup flow
 920       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
 921         {
 922           // flow does not exist go slow path
 923           slow_path_reason = NAT_ED_SP_REASON_LOOKUP_FAILED;
 924           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 925           goto trace0;
 926         }
 927       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 928       s0 =
 929         pool_elt_at_index (tsm->sessions,
 930                            ed_value_get_session_index (&value0));
 931     skip_lookup:
 932
 933       ASSERT (thread_index == s0->thread_index);
 934
 935       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 936         {
 937           // session is closed, go slow path
 938           nat_free_session_data (sm, s0, thread_index, 0);
 939           nat_ed_session_delete (sm, s0, thread_index, 1);
 940           slow_path_reason = NAT_ED_SP_REASON_VRF_EXPIRED;
 941           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 942           goto trace0;
 943         }
 944
 945       if (s0->tcp_closed_timestamp)
 946         {
 947           if (now >= s0->tcp_closed_timestamp)
 948             {
 949               // session is closed, go slow path, freed in slow path
 950               slow_path_reason = NAT_ED_SP_TCP_CLOSED;
 951               next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 952             }
 953           else
 954             {
 955               // session in transitory timeout, drop
 956               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 957               next[0] = NAT_NEXT_DROP;
 958             }
 959           goto trace0;
 960         }
 961
 962       // drop if session expired
 963       u64 sess_timeout_time;
 964       sess_timeout_time =
 965         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 966       if (now >= sess_timeout_time)
 967         {
 968           // session is closed, go slow path
 969           nat_free_session_data (sm, s0, thread_index, 0);
 970           nat_ed_session_delete (sm, s0, thread_index, 1);
 971           slow_path_reason = NAT_ED_SP_SESS_EXPIRED;
 972           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 973           goto trace0;
 974         }
 975
 976       if (nat_6t_t_eq (&s0->o2i.match, &lookup))
 977         {
 978           f = &s0->o2i;
 979         }
 980       else if (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 981                nat_6t_t_eq (&s0->i2o.match, &lookup))
 982         {
 983           f = &s0->i2o;
 984         }
 985       else
 986         {
 987           /*
 988            * Send DHCP packets to the ipv4 stack, or we won't
 989            * be able to use dhcp client on the outside interface
 990            */
 991           if (PREDICT_FALSE (
 992                 proto0 == NAT_PROTOCOL_UDP &&
 993                 (vnet_buffer (b0)->ip.reass.l4_dst_port ==
 994                  clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
 995             {
 996               goto trace0;
 997             }
 998
 999           if (!sm->forwarding_enabled)
1000             {
1001               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1002               next[0] = NAT_NEXT_DROP;
1003               goto trace0;
1004             }
1005           else
1006             {
1007               if (nat_6t_t_eq (&s0->i2o.match, &lookup))
1008                 {
1009                   f = &s0->i2o;
1010                 }
1011               else
1012                 {
1013                   // FIXME TODO bypass ???
1014                   //  create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
1015                   //                       thread_index);
1016                   translation_error = NAT_ED_TRNSL_ERR_FLOW_MISMATCH;
1017                   nat_free_session_data (sm, s0, thread_index, 0);
1018                   nat_ed_session_delete (sm, s0, thread_index, 1);
1019                   next[0] = NAT_NEXT_DROP;
1020                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1021                   goto trace0;
1022                 }
1023             }
1024         }
1025
1026       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1027           (translation_error = nat_6t_flow_buf_translate_o2i (
1028              vm, sm, b0, ip0, f, proto0, 0 /* is_output_feature */)))
1029         {
1030           next[0] = NAT_NEXT_DROP;
1031           b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1032           goto trace0;
1033         }
1034
1035       switch (proto0)
1036         {
1037         case NAT_PROTOCOL_TCP:
1038           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.tcp,
1039                                          thread_index, sw_if_index0, 1);
1040           nat44_set_tcp_session_state_o2i (sm, now, s0,
1041                                            vnet_buffer (b0)->ip.
1042                                            reass.icmp_type_or_tcp_flags,
1043                                            vnet_buffer (b0)->ip.
1044                                            reass.tcp_ack_number,
1045                                            vnet_buffer (b0)->ip.
1046                                            reass.tcp_seq_number,
1047                                            thread_index);
1048           break;
1049         case NAT_PROTOCOL_UDP:
1050           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.udp,
1051                                          thread_index, sw_if_index0, 1);
1052           break;
1053         case NAT_PROTOCOL_ICMP:
1054           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.icmp,
1055                                          thread_index, sw_if_index0, 1);
1056           break;
1057         case NAT_PROTOCOL_OTHER:
1058           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.other,
1059                                          thread_index, sw_if_index0, 1);
1060           break;
1061         }
1062
1063       /* Accounting */
1064       nat44_session_update_counters (s0, now,
1065                                      vlib_buffer_length_in_chain (vm, b0),
1066                                      thread_index);
1067       /* Per-user LRU list maintenance */
1068       nat44_session_update_lru (sm, s0, thread_index);
1069
1070     trace0:
1071       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1072                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1073         {
1074           nat44_ed_out2in_trace_t *t =
1075             vlib_add_trace (vm, node, b0, sizeof (*t));
1076           t->sw_if_index = sw_if_index0;
1077           t->next_index = next[0];
1078           t->is_slow_path = 0;
1079           t->translation_error = translation_error;
1080           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1081           t->lookup_skipped = lookup_skipped;
1082           t->slow_path_reason = slow_path_reason;
1083
1084           if (s0)
1085             {
1086               t->session_index = s0 - tsm->sessions;
1087               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1088               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1089               t->translation_via_i2of = (&s0->i2o == f);
1090             }
1091           else
1092             {
1093               t->session_index = ~0;
1094             }
1095         }
1096
1097       if (next[0] == NAT_NEXT_DROP)
1098         {
1099           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.drops,
1100                                          thread_index, sw_if_index0, 1);
1101         }
1102
1103       n_left_from--;
1104       next++;
1105     }
1106
1107   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1108                                frame->n_vectors);
1109   return frame->n_vectors;
1110 }
1111
1112 static inline uword
1113 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1114                                           vlib_node_runtime_t * node,
1115                                           vlib_frame_t * frame)
1116 {
1117   u32 n_left_from, *from;
1118   snat_main_t *sm = &snat_main;
1119   f64 now = vlib_time_now (vm);
1120   u32 thread_index = vm->thread_index;
1121   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1122   snat_static_mapping_t *m;
1123
1124   from = vlib_frame_vector_args (frame);
1125   n_left_from = frame->n_vectors;
1126
1127   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1128   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1129   vlib_get_buffers (vm, from, b, n_left_from);
1130
1131   while (n_left_from > 0)
1132     {
1133       vlib_buffer_t *b0;
1134       u32 sw_if_index0, rx_fib_index0;
1135       nat_protocol_t proto0;
1136       ip4_header_t *ip0;
1137       udp_header_t *udp0;
1138       icmp46_header_t *icmp0;
1139       snat_session_t *s0 = 0;
1140       clib_bihash_kv_16_8_t kv0, value0;
1141       lb_nat_type_t lb_nat0;
1142       twice_nat_type_t twice_nat0;
1143       u8 identity_nat0;
1144       ip4_address_t sm_addr;
1145       u16 sm_port;
1146       u32 sm_fib_index;
1147       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
1148
1149       b0 = *b;
1150       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1151
1152       vnet_buffer (b0)->snat.flags = 0;
1153       ip0 = vlib_buffer_get_current (b0);
1154
1155       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1156       rx_fib_index0 =
1157         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1158
1159       if (PREDICT_FALSE (ip0->ttl == 1))
1160         {
1161           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1162           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1163                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1164                                        0);
1165           next[0] = NAT_NEXT_ICMP_ERROR;
1166           goto trace0;
1167         }
1168
1169       udp0 = ip4_next_header (ip0);
1170       icmp0 = (icmp46_header_t *) udp0;
1171       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1172
1173       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1174         {
1175           s0 = nat44_ed_out2in_slowpath_unknown_proto (
1176             sm, b0, ip0, rx_fib_index0, thread_index, now, vm, node);
1177           if (!sm->forwarding_enabled)
1178             {
1179               if (!s0)
1180                 next[0] = NAT_NEXT_DROP;
1181             }
1182           if (NAT_NEXT_DROP != next[0] && s0 &&
1183               NAT_ED_TRNSL_ERR_SUCCESS !=
1184                 (translation_error = nat_6t_flow_buf_translate_o2i (
1185                    vm, sm, b0, ip0, &s0->o2i, proto0,
1186                    0 /* is_output_feature */)))
1187             {
1188               next[0] = NAT_NEXT_DROP;
1189               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1190               goto trace0;
1191             }
1192
1193           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.other,
1194                                          thread_index, sw_if_index0, 1);
1195           goto trace0;
1196         }
1197
1198       if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1199         {
1200           next[0] = icmp_out2in_ed_slow_path
1201             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1202              next[0], now, thread_index, &s0);
1203
1204           if (NAT_NEXT_DROP != next[0] && s0 &&
1205               NAT_ED_TRNSL_ERR_SUCCESS !=
1206                 (translation_error = nat_6t_flow_buf_translate_o2i (
1207                    vm, sm, b0, ip0, &s0->o2i, proto0,
1208                    0 /* is_output_feature */)))
1209             {
1210               next[0] = NAT_NEXT_DROP;
1211               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1212               goto trace0;
1213             }
1214
1215           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.icmp,
1216                                          thread_index, sw_if_index0, 1);
1217           goto trace0;
1218         }
1219
1220       init_ed_k (&kv0, ip0->src_address,
1221                  vnet_buffer (b0)->ip.reass.l4_src_port, ip0->dst_address,
1222                  vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1223                  ip0->protocol);
1224
1225       s0 = NULL;
1226       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
1227         {
1228           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1229           s0 =
1230             pool_elt_at_index (tsm->sessions,
1231                                ed_value_get_session_index (&value0));
1232
1233           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1234             {
1235               nat_free_session_data (sm, s0, thread_index, 0);
1236               nat_ed_session_delete (sm, s0, thread_index, 1);
1237               s0 = NULL;
1238             }
1239         }
1240
1241       if (!s0)
1242         {
1243           /* Try to match static mapping by external address and port,
1244              destination address and port in packet */
1245
1246           if (snat_static_mapping_match (
1247                 vm, sm, ip0->dst_address,
1248                 vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0, proto0,
1249                 &sm_addr, &sm_port, &sm_fib_index, 1, 0, &twice_nat0, &lb_nat0,
1250                 &ip0->src_address, &identity_nat0, &m))
1251             {
1252               /*
1253                * Send DHCP packets to the ipv4 stack, or we won't
1254                * be able to use dhcp client on the outside interface
1255                */
1256               if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_UDP
1257                                  && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1258                                      clib_host_to_net_u16
1259                                      (UDP_DST_PORT_dhcp_to_client))))
1260                 {
1261                   goto trace0;
1262                 }
1263
1264               if (!sm->forwarding_enabled)
1265                 {
1266                   b0->error =
1267                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1268                   next[0] = NAT_NEXT_DROP;
1269                 }
1270               else
1271                 {
1272                   if (next_src_nat (
1273                         sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1274                         vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0))
1275                     {
1276                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1277                     }
1278                   else
1279                     {
1280                       create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
1281                                              thread_index);
1282                     }
1283                 }
1284               goto trace0;
1285             }
1286
1287           if (PREDICT_FALSE (identity_nat0))
1288             goto trace0;
1289
1290           if ((proto0 == NAT_PROTOCOL_TCP)
1291               && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1292                                      reass.icmp_type_or_tcp_flags))
1293             {
1294               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1295               next[0] = NAT_NEXT_DROP;
1296               goto trace0;
1297             }
1298
1299           /* Create session initiated by host from external network */
1300           s0 = create_session_for_static_mapping_ed (sm, b0,
1301                                                      sm_addr, sm_port,
1302                                                      sm_fib_index,
1303                                                      ip0->dst_address,
1304                                                      vnet_buffer (b0)->
1305                                                      ip.reass.l4_dst_port,
1306                                                      rx_fib_index0, proto0,
1307                                                      node, rx_fib_index0,
1308                                                      thread_index, twice_nat0,
1309                                                      lb_nat0, now, m);
1310           if (!s0)
1311             {
1312               next[0] = NAT_NEXT_DROP;
1313               goto trace0;
1314             }
1315         }
1316
1317       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1318           (translation_error = nat_6t_flow_buf_translate_o2i (
1319              vm, sm, b0, ip0, &s0->o2i, proto0, 0 /* is_output_feature */)))
1320         {
1321           next[0] = NAT_NEXT_DROP;
1322           goto trace0;
1323         }
1324
1325       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1326         {
1327           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.tcp,
1328                                          thread_index, sw_if_index0, 1);
1329           nat44_set_tcp_session_state_o2i (sm, now, s0,
1330                                            vnet_buffer (b0)->ip.
1331                                            reass.icmp_type_or_tcp_flags,
1332                                            vnet_buffer (b0)->ip.
1333                                            reass.tcp_ack_number,
1334                                            vnet_buffer (b0)->ip.
1335                                            reass.tcp_seq_number,
1336                                            thread_index);
1337         }
1338       else
1339         {
1340           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.udp,
1341                                          thread_index, sw_if_index0, 1);
1342         }
1343
1344       /* Accounting */
1345       nat44_session_update_counters (s0, now,
1346                                      vlib_buffer_length_in_chain (vm, b0),
1347                                      thread_index);
1348       /* Per-user LRU list maintenance */
1349       nat44_session_update_lru (sm, s0, thread_index);
1350
1351     trace0:
1352       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1353                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1354         {
1355           nat44_ed_out2in_trace_t *t =
1356             vlib_add_trace (vm, node, b0, sizeof (*t));
1357           t->sw_if_index = sw_if_index0;
1358           t->next_index = next[0];
1359           t->is_slow_path = 1;
1360           t->translation_error = translation_error;
1361           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1362
1363           if (s0)
1364             {
1365               t->session_index = s0 - tsm->sessions;
1366               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1367               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1368             }
1369           else
1370             {
1371               t->session_index = ~0;
1372             }
1373         }
1374
1375       if (next[0] == NAT_NEXT_DROP)
1376         {
1377           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.drops,
1378                                          thread_index, sw_if_index0, 1);
1379         }
1380
1381       n_left_from--;
1382       next++;
1383       b++;
1384     }
1385
1386   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1387                                frame->n_vectors);
1388
1389   return frame->n_vectors;
1390 }
1391
1392 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1393                                      vlib_node_runtime_t * node,
1394                                      vlib_frame_t * frame)
1395 {
1396   if (snat_main.num_workers > 1)
1397     {
1398       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1399     }
1400   else
1401     {
1402       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1403     }
1404 }
1405
1406 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1407   .name = "nat44-ed-out2in",
1408   .vector_size = sizeof (u32),
1409   .sibling_of = "nat-default",
1410   .format_trace = format_nat44_ed_out2in_trace,
1411   .type = VLIB_NODE_TYPE_INTERNAL,
1412   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1413   .error_strings = nat_out2in_ed_error_strings,
1414   .runtime_data_bytes = sizeof (snat_runtime_t),
1415 };
1416
1417 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1418                                               vlib_node_runtime_t * node,
1419                                               vlib_frame_t * frame)
1420 {
1421   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1422 }
1423
1424 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1425   .name = "nat44-ed-out2in-slowpath",
1426   .vector_size = sizeof (u32),
1427   .sibling_of = "nat-default",
1428   .format_trace = format_nat44_ed_out2in_trace,
1429   .type = VLIB_NODE_TYPE_INTERNAL,
1430   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1431   .error_strings = nat_out2in_ed_error_strings,
1432   .runtime_data_bytes = sizeof (snat_runtime_t),
1433 };
1434
1435 static u8 *
1436 format_nat_pre_trace (u8 * s, va_list * args)
1437 {
1438   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1439   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1440   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1441   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1442                  t->arc_next_index);
1443 }
1444
1445 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1446                                     vlib_node_runtime_t * node,
1447                                     vlib_frame_t * frame)
1448 {
1449   return nat_pre_node_fn_inline (vm, node, frame,
1450                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1451 }
1452
1453 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1454   .name = "nat-pre-out2in",
1455   .vector_size = sizeof (u32),
1456   .sibling_of = "nat-default",
1457   .format_trace = format_nat_pre_trace,
1458   .type = VLIB_NODE_TYPE_INTERNAL,
1459   .n_errors = 0,
1460  };
1461
1462 /*
1463  * fd.io coding-style-patch-verification: ON
1464  *
1465  * Local Variables:
1466  * eval: (c-set-style "gnu")
1467  * End:
1468  */