src/plugins/nat/nat44-ed/nat44_ed_out2in.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27
  28 #include <nat/lib/ipfix_logging.h>
  29
  30 #include <nat/nat44-ed/nat44_ed.h>
  31 #include <nat/nat44-ed/nat44_ed_inlines.h>
  32
  33 static char *nat_out2in_ed_error_strings[] = {
  34 #define _(sym,string) string,
  35   foreach_nat_out2in_ed_error
  36 #undef _
  37 };
  38
  39 typedef enum
  40 {
  41   NAT_ED_SP_REASON_NO_REASON,
  42   NAT_ED_SP_REASON_LOOKUP_FAILED,
  43   NAT_ED_SP_REASON_VRF_EXPIRED,
  44   NAT_ED_SP_TCP_CLOSED,
  45   NAT_ED_SP_SESS_EXPIRED,
  46 } nat_slow_path_reason_e;
  47
  48 typedef struct
  49 {
  50   u32 sw_if_index;
  51   u32 next_index;
  52   u32 session_index;
  53   nat_translation_error_e translation_error;
  54   nat_6t_flow_t i2of;
  55   nat_6t_flow_t o2if;
  56   clib_bihash_kv_16_8_t search_key;
  57   u8 is_slow_path;
  58   u8 translation_via_i2of;
  59   u8 lookup_skipped;
  60   nat_slow_path_reason_e slow_path_reason;
  61 } nat44_ed_out2in_trace_t;
  62
  63 static u8 *
  64 format_slow_path_reason (u8 *s, va_list *args)
  65 {
  66   nat_slow_path_reason_e reason = va_arg (*args, nat_slow_path_reason_e);
  67   switch (reason)
  68     {
  69     case NAT_ED_SP_REASON_NO_REASON:
  70       return format (s, "no reason for slow path");
  71     case NAT_ED_SP_REASON_LOOKUP_FAILED:
  72       return format (s, "slow path because lookup failed");
  73     case NAT_ED_SP_REASON_VRF_EXPIRED:
  74       return format (s, "slow path because vrf expired");
  75     case NAT_ED_SP_TCP_CLOSED:
  76       return format (s, "slow path because tcp closed");
  77     case NAT_ED_SP_SESS_EXPIRED:
  78       return format (s, "slow path because session expired");
  79     }
  80   return format (s, "invalid reason value");
  81 }
  82
  83 static u8 *
  84 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  85 {
  86   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  87   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  88   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  89   char *tag;
  90
  91   tag =
  92     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  93     "NAT44_OUT2IN_ED_FAST_PATH";
  94
  95   s = format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
  96               t->next_index);
  97   if (~0 != t->session_index)
  98     {
  99       s = format (s, ", session %d, translation result '%U' via %s",
 100                   t->session_index, format_nat_ed_translation_error,
 101                   t->translation_error,
 102                   t->translation_via_i2of ? "i2of" : "o2if");
 103       s = format (s, "\n  i2of %U", format_nat_6t_flow, &t->i2of);
 104       s = format (s, "\n  o2if %U", format_nat_6t_flow, &t->o2if);
 105     }
 106   if (!t->is_slow_path)
 107     {
 108       if (t->lookup_skipped)
 109         {
 110           s = format (s, "\n lookup skipped - cached session index used");
 111         }
 112       else
 113         {
 114           s = format (s, "\n  search key %U", format_ed_session_kvp,
 115                       &t->search_key);
 116         }
 117       s = format (s, "\n %U", format_slow_path_reason, t->slow_path_reason);
 118     }
 119
 120   return s;
 121 }
 122
 123 static int
 124 next_src_nat (snat_main_t *sm, ip4_header_t *ip, u16 src_port, u16 dst_port,
 125               u32 rx_fib_index)
 126 {
 127   clib_bihash_kv_16_8_t kv, value;
 128
 129   init_ed_k (&kv, ip->src_address.as_u32, src_port, ip->dst_address.as_u32,
 130              dst_port, rx_fib_index, ip->protocol);
 131   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 132     return 1;
 133
 134   return 0;
 135 }
 136
 137 static void create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b,
 138                                    snat_session_t *s, ip4_header_t *ip,
 139                                    u32 rx_fib_index, u32 thread_index);
 140
 141 static snat_session_t *create_session_for_static_mapping_ed (
 142   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 143   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 144   ip_protocol_t proto, vlib_node_runtime_t *node, u32 thread_index,
 145   twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 146   snat_static_mapping_t *mapping);
 147
 148 static inline u32
 149 icmp_out2in_ed_slow_path (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
 150                           icmp46_header_t *icmp, u32 sw_if_index,
 151                           u32 rx_fib_index, vlib_node_runtime_t *node,
 152                           u32 next, f64 now, u32 thread_index,
 153                           snat_session_t **s_p)
 154 {
 155   vlib_main_t *vm = vlib_get_main ();
 156
 157   ip_csum_t sum;
 158   u16 checksum;
 159
 160   snat_session_t *s = 0;
 161   u8 is_addr_only, identity_nat;
 162   ip4_address_t sm_addr;
 163   u16 sm_port;
 164   u32 sm_fib_index;
 165   snat_static_mapping_t *m;
 166   u8 lookup_protocol;
 167   ip4_address_t lookup_saddr, lookup_daddr;
 168   u16 lookup_sport, lookup_dport;
 169
 170   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 171   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 172
 173   if (nat_get_icmp_session_lookup_values (b, ip, &lookup_saddr, &lookup_sport,
 174                                           &lookup_daddr, &lookup_dport,
 175                                           &lookup_protocol))
 176     {
 177       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 178       next = NAT_NEXT_DROP;
 179       goto out;
 180     }
 181
 182   if (snat_static_mapping_match (vm, sm, ip->dst_address, lookup_sport,
 183                                  rx_fib_index, ip->protocol, &sm_addr,
 184                                  &sm_port, &sm_fib_index, 1, &is_addr_only, 0,
 185                                  0, 0, &identity_nat, &m))
 186     {
 187       // static mapping not matched
 188       if (!sm->forwarding_enabled)
 189         {
 190           /* Don't NAT packet aimed at the intfc address */
 191           if (!is_interface_addr (sm, node, sw_if_index,
 192                                   ip->dst_address.as_u32))
 193             {
 194               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 195               next = NAT_NEXT_DROP;
 196             }
 197         }
 198       else
 199         {
 200           if (next_src_nat (sm, ip, lookup_sport, lookup_dport, rx_fib_index))
 201             {
 202               next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 203             }
 204           else
 205             {
 206               create_bypass_for_fwd (sm, b, s, ip, rx_fib_index, thread_index);
 207             }
 208         }
 209       goto out;
 210     }
 211
 212   if (PREDICT_FALSE (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 213                        ICMP4_echo_reply &&
 214                      (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 215                         ICMP4_echo_request ||
 216                       !is_addr_only)))
 217     {
 218       b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 219       next = NAT_NEXT_DROP;
 220       goto out;
 221     }
 222
 223   if (PREDICT_FALSE (identity_nat))
 224     {
 225       goto out;
 226     }
 227
 228   /* Create session initiated by host from external network */
 229   s = create_session_for_static_mapping_ed (
 230     sm, b, sm_addr, sm_port, sm_fib_index, ip->dst_address, lookup_sport,
 231     rx_fib_index, lookup_protocol, node, thread_index, 0, 0,
 232     vlib_time_now (vm), m);
 233   if (!s)
 234     next = NAT_NEXT_DROP;
 235
 236   if (PREDICT_TRUE (!ip4_is_fragment (ip)))
 237     {
 238       sum = ip_incremental_checksum_buffer (
 239         vm, b, (u8 *) icmp - (u8 *) vlib_buffer_get_current (b),
 240         ntohs (ip->length) - ip4_header_bytes (ip), 0);
 241       checksum = ~ip_csum_fold (sum);
 242       if (checksum != 0 && checksum != 0xffff)
 243         {
 244           next = NAT_NEXT_DROP;
 245           goto out;
 246         }
 247     }
 248
 249   if (PREDICT_TRUE (next != NAT_NEXT_DROP && s))
 250     {
 251       /* Accounting */
 252       nat44_session_update_counters (
 253         s, now, vlib_buffer_length_in_chain (vm, b), thread_index);
 254       /* Per-user LRU list maintenance */
 255       nat44_session_update_lru (sm, s, thread_index);
 256     }
 257 out:
 258   if (NAT_NEXT_DROP == next && s)
 259     {
 260       nat_ed_session_delete (sm, s, thread_index, 1);
 261       s = 0;
 262     }
 263   *s_p = s;
 264   return next;
 265 }
 266
 267 static_always_inline int
 268 nat44_ed_alloc_i2o_port (snat_main_t *sm, snat_address_t *a, snat_session_t *s,
 269                          ip4_address_t i2o_addr, u16 i2o_port,
 270                          u32 i2o_fib_index, ip_protocol_t proto,
 271                          u32 thread_index, u32 snat_thread_index,
 272                          ip4_address_t *outside_addr, u16 *outside_port)
 273 {
 274   u32 portnum;
 275
 276   for (int i = 0; i < ED_PORT_ALLOC_ATTEMPTS; ++i)
 277     {
 278       portnum = (sm->port_per_thread * snat_thread_index) +
 279                 snat_random_port (0, sm->port_per_thread - 1) + 1024;
 280       portnum = clib_host_to_net_u16 (portnum);
 281       nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port, a->addr,
 282                             portnum, i2o_fib_index, proto);
 283       if (!nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s,
 284                                              1 /* is_add */))
 285         {
 286           *outside_addr = a->addr;
 287           *outside_port = portnum;
 288           return 0;
 289         }
 290     }
 291
 292   /* Totally out of translations to use... */
 293   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 294   return 1;
 295 }
 296
 297 static_always_inline int
 298 nat44_ed_alloc_i2o_addr_and_port (snat_main_t *sm, snat_address_t *addresses,
 299                                   snat_session_t *s, ip4_address_t i2o_addr,
 300                                   u16 i2o_port, u32 i2o_fib_index,
 301                                   ip_protocol_t proto, u32 thread_index,
 302                                   u32 snat_thread_index,
 303                                   ip4_address_t *outside_addr,
 304                                   u16 *outside_port)
 305 {
 306   snat_address_t *a, *ga = 0;
 307   int i;
 308
 309   if (vec_len (addresses) > 0)
 310     {
 311       int s_addr_offset = i2o_addr.as_u32 % vec_len (addresses);
 312
 313       for (i = s_addr_offset; i < vec_len (addresses); ++i)
 314         {
 315           a = addresses + i;
 316           if (a->fib_index == i2o_fib_index)
 317             {
 318               return nat44_ed_alloc_i2o_port (
 319                 sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 320                 thread_index, snat_thread_index, outside_addr, outside_port);
 321             }
 322           else if (a->fib_index == ~0)
 323             {
 324               ga = a;
 325             }
 326         }
 327
 328       for (i = 0; i < s_addr_offset; ++i)
 329         {
 330           a = addresses + i;
 331           if (a->fib_index == i2o_fib_index)
 332             {
 333               return nat44_ed_alloc_i2o_port (
 334                 sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 335                 thread_index, snat_thread_index, outside_addr, outside_port);
 336             }
 337           else if (a->fib_index == ~0)
 338             {
 339               ga = a;
 340             }
 341         }
 342
 343       if (ga)
 344         {
 345           return nat44_ed_alloc_i2o_port (
 346             sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto, thread_index,
 347             snat_thread_index, outside_addr, outside_port);
 348         }
 349     }
 350
 351   /* Totally out of translations to use... */
 352   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 353   return 1;
 354 }
 355
 356 static snat_session_t *
 357 create_session_for_static_mapping_ed (
 358   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 359   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 360   ip_protocol_t proto, vlib_node_runtime_t *node, u32 thread_index,
 361   twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 362   snat_static_mapping_t *mapping)
 363 {
 364   snat_session_t *s;
 365   ip4_header_t *ip;
 366   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 367
 368   if (PREDICT_FALSE (
 369         nat44_ed_maximum_sessions_exceeded (sm, o2i_fib_index, thread_index)))
 370     {
 371       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 372       nat_elog_notice (sm, "maximum sessions exceeded");
 373       return 0;
 374     }
 375
 376   s = nat_ed_session_alloc (sm, thread_index, now, proto);
 377   if (!s)
 378     {
 379       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 380       nat_elog_warn (sm, "create NAT session failed");
 381       return 0;
 382     }
 383
 384   ip = vlib_buffer_get_current (b);
 385
 386   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 387   s->ext_host_port =
 388     proto == IP_PROTOCOL_ICMP ? 0 : vnet_buffer (b)->ip.reass.l4_src_port;
 389   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 390   if (lb_nat)
 391     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 392   if (lb_nat == AFFINITY_LB_NAT)
 393     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 394   s->out2in.addr = o2i_addr;
 395   s->out2in.port = o2i_port;
 396   s->out2in.fib_index = o2i_fib_index;
 397   s->in2out.addr = i2o_addr;
 398   s->in2out.port = i2o_port;
 399   s->in2out.fib_index = i2o_fib_index;
 400   s->proto = proto;
 401
 402   if (IP_PROTOCOL_ICMP == proto)
 403     {
 404       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr, o2i_port,
 405                             o2i_addr, o2i_port, o2i_fib_index, ip->protocol);
 406       nat_6t_flow_icmp_id_rewrite_set (&s->o2i, i2o_port);
 407     }
 408   else
 409     {
 410       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr,
 411                             s->ext_host_port, o2i_addr, o2i_port,
 412                             o2i_fib_index, ip->protocol);
 413       nat_6t_flow_dport_rewrite_set (&s->o2i, i2o_port);
 414     }
 415   nat_6t_flow_daddr_rewrite_set (&s->o2i, i2o_addr.as_u32);
 416   nat_6t_flow_txfib_rewrite_set (&s->o2i, i2o_fib_index);
 417
 418   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 419     {
 420       b->error = node->errors[NAT_OUT2IN_ED_ERROR_HASH_ADD_FAILED];
 421       nat_ed_session_delete (sm, s, thread_index, 1);
 422       nat_elog_warn (sm, "out2in flow hash add failed");
 423       return 0;
 424     }
 425
 426   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 427                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 428     {
 429       int rc = 0;
 430       snat_address_t *filter = 0;
 431
 432       // if exact address is specified use this address
 433       if (is_sm_exact_address (mapping->flags))
 434         {
 435           snat_address_t *ap;
 436           vec_foreach (ap, sm->twice_nat_addresses)
 437           {
 438             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 439               {
 440                 filter = ap;
 441                 break;
 442               }
 443           }
 444         }
 445
 446       if (filter)
 447         {
 448           rc = nat44_ed_alloc_i2o_port (
 449             sm, filter, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 450             thread_index, tsm->snat_thread_index, &s->ext_host_nat_addr,
 451             &s->ext_host_nat_port);
 452           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 453         }
 454       else
 455         {
 456           rc = nat44_ed_alloc_i2o_addr_and_port (
 457             sm, sm->twice_nat_addresses, s, i2o_addr, i2o_port, i2o_fib_index,
 458             proto, thread_index, tsm->snat_thread_index, &s->ext_host_nat_addr,
 459             &s->ext_host_nat_port);
 460         }
 461
 462       if (rc)
 463         {
 464           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 465           nat_ed_session_delete (sm, s, thread_index, 1);
 466           return 0;
 467         }
 468
 469       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 470
 471       nat_6t_flow_saddr_rewrite_set (&s->o2i, s->ext_host_nat_addr.as_u32);
 472       if (IP_PROTOCOL_ICMP == proto)
 473         {
 474           nat_6t_flow_icmp_id_rewrite_set (&s->o2i, s->ext_host_nat_port);
 475         }
 476       else
 477         {
 478           nat_6t_flow_sport_rewrite_set (&s->o2i, s->ext_host_nat_port);
 479         }
 480
 481       nat_6t_l3_l4_csum_calc (&s->o2i);
 482
 483       nat_6t_flow_daddr_rewrite_set (&s->i2o, s->ext_host_addr.as_u32);
 484       if (IP_PROTOCOL_ICMP == proto)
 485         {
 486           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, s->ext_host_port);
 487         }
 488       else
 489         {
 490           nat_6t_flow_dport_rewrite_set (&s->i2o, s->ext_host_port);
 491         }
 492
 493       nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 494       if (IP_PROTOCOL_ICMP == proto)
 495         {
 496           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 497         }
 498       else
 499         {
 500           nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 501         }
 502       nat_6t_l3_l4_csum_calc (&s->i2o);
 503     }
 504   else
 505     {
 506       if (IP_PROTOCOL_ICMP == proto)
 507         {
 508           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 509                                 s->ext_host_addr, i2o_port, i2o_fib_index,
 510                                 ip->protocol);
 511         }
 512       else
 513         {
 514           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 515                                 s->ext_host_addr, s->ext_host_port,
 516                                 i2o_fib_index, ip->protocol);
 517         }
 518
 519   nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 520   if (IP_PROTOCOL_ICMP == proto)
 521     {
 522       nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 523     }
 524   else
 525     {
 526       nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 527     }
 528
 529   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 530     {
 531       nat_elog_notice (sm, "in2out flow hash add failed");
 532       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 533         {
 534           nat_elog_warn (sm, "out2in flow hash del failed");
 535         }
 536       nat_ed_session_delete (sm, s, thread_index, 1);
 537       return 0;
 538     }
 539     }
 540   nat_ipfix_logging_nat44_ses_create (
 541     thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 542     s->in2out.port, s->out2in.port, s->in2out.fib_index);
 543
 544   nat_syslog_nat44_sadd (0, s->in2out.fib_index, &s->in2out.addr,
 545                          s->in2out.port, &s->ext_host_nat_addr,
 546                          s->ext_host_nat_port, &s->out2in.addr, s->out2in.port,
 547                          &s->ext_host_addr, s->ext_host_port, s->proto,
 548                          nat44_ed_is_twice_nat_session (s));
 549
 550   per_vrf_sessions_register_session (s, thread_index);
 551
 552   return s;
 553 }
 554
 555 static void
 556 create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b, snat_session_t *s,
 557                        ip4_header_t *ip, u32 rx_fib_index, u32 thread_index)
 558 {
 559   clib_bihash_kv_16_8_t kv, value;
 560   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 561   vlib_main_t *vm = vlib_get_main ();
 562   f64 now = vlib_time_now (vm);
 563   u16 lookup_sport, lookup_dport;
 564   u8 lookup_protocol;
 565   ip4_address_t lookup_saddr, lookup_daddr;
 566
 567   if (ip->protocol == IP_PROTOCOL_ICMP)
 568     {
 569       if (nat_get_icmp_session_lookup_values (b, ip, &lookup_daddr,
 570                                               &lookup_sport, &lookup_saddr,
 571                                               &lookup_dport, &lookup_protocol))
 572         return;
 573     }
 574   else
 575     {
 576       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 577         {
 578           lookup_sport = vnet_buffer (b)->ip.reass.l4_dst_port;
 579           lookup_dport = vnet_buffer (b)->ip.reass.l4_src_port;
 580         }
 581       else
 582         {
 583           lookup_sport = 0;
 584           lookup_dport = 0;
 585         }
 586       lookup_saddr.as_u32 = ip->dst_address.as_u32;
 587       lookup_daddr.as_u32 = ip->src_address.as_u32;
 588       lookup_protocol = ip->protocol;
 589     }
 590
 591   init_ed_k (&kv, lookup_saddr.as_u32, lookup_sport, lookup_daddr.as_u32,
 592              lookup_dport, rx_fib_index, lookup_protocol);
 593
 594   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 595     {
 596       ASSERT (thread_index == ed_value_get_thread_index (&value));
 597       s =
 598         pool_elt_at_index (tsm->sessions,
 599                            ed_value_get_session_index (&value));
 600     }
 601   else if (ip->protocol == IP_PROTOCOL_ICMP &&
 602            icmp_type_is_error_message
 603            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 604     {
 605       return;
 606     }
 607   else
 608     {
 609       if (PREDICT_FALSE
 610           (nat44_ed_maximum_sessions_exceeded
 611            (sm, rx_fib_index, thread_index)))
 612         return;
 613
 614       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 615       if (!s)
 616         {
 617           nat_elog_warn (sm, "create NAT session failed");
 618           return;
 619         }
 620
 621       s->ext_host_addr = ip->src_address;
 622       s->ext_host_port = lookup_dport;
 623       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 624       s->out2in.addr = ip->dst_address;
 625       s->out2in.port = lookup_sport;
 626       s->proto = ip->protocol;
 627       s->out2in.fib_index = rx_fib_index;
 628       s->in2out.addr = s->out2in.addr;
 629       s->in2out.port = s->out2in.port;
 630       s->in2out.fib_index = s->out2in.fib_index;
 631
 632       nat_6t_i2o_flow_init (sm, thread_index, s, ip->dst_address, lookup_sport,
 633                             ip->src_address, lookup_dport, rx_fib_index,
 634                             ip->protocol);
 635       nat_6t_flow_txfib_rewrite_set (&s->i2o, rx_fib_index);
 636       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 637         {
 638           nat_elog_notice (sm, "in2out flow add failed");
 639           nat_ed_session_delete (sm, s, thread_index, 1);
 640           return;
 641         }
 642
 643       per_vrf_sessions_register_session (s, thread_index);
 644     }
 645
 646   if (ip->protocol == IP_PROTOCOL_TCP)
 647     {
 648       tcp_header_t *tcp = ip4_next_header (ip);
 649       nat44_set_tcp_session_state_o2i (sm, now, s, tcp->flags,
 650                                        tcp->ack_number, tcp->seq_number,
 651                                        thread_index);
 652     }
 653
 654   /* Accounting */
 655   nat44_session_update_counters (s, now, 0, thread_index);
 656   /* Per-user LRU list maintenance */
 657   nat44_session_update_lru (sm, s, thread_index);
 658 }
 659
 660 static snat_session_t *
 661 nat44_ed_out2in_slowpath_unknown_proto (snat_main_t *sm, vlib_buffer_t *b,
 662                                         ip4_header_t *ip, u32 rx_fib_index,
 663                                         u32 thread_index, f64 now,
 664                                         vlib_main_t *vm,
 665                                         vlib_node_runtime_t *node)
 666 {
 667   snat_static_mapping_t *m;
 668   snat_session_t *s;
 669
 670   if (PREDICT_FALSE (
 671         nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 672     {
 673       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 674       nat_elog_notice (sm, "maximum sessions exceeded");
 675       return 0;
 676     }
 677
 678   m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, ip->protocol);
 679   if (!m)
 680     {
 681       b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 682       return 0;
 683     }
 684
 685   /* Create a new session */
 686   s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 687   if (!s)
 688     {
 689       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 690       nat_elog_warn (sm, "create NAT session failed");
 691       return 0;
 692     }
 693
 694   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 695   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 696   s->out2in.addr.as_u32 = ip->dst_address.as_u32;
 697   s->out2in.fib_index = rx_fib_index;
 698   s->in2out.addr.as_u32 = m->local_addr.as_u32;
 699   s->in2out.fib_index = m->fib_index;
 700   s->in2out.port = s->out2in.port = ip->protocol;
 701
 702   nat_6t_o2i_flow_init (sm, thread_index, s, ip->dst_address, 0,
 703                         ip->src_address, 0, m->fib_index, ip->protocol);
 704   nat_6t_flow_saddr_rewrite_set (&s->i2o, ip->dst_address.as_u32);
 705   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 706     {
 707       nat_elog_notice (sm, "in2out key add failed");
 708       nat_ed_session_delete (sm, s, thread_index, 1);
 709       return NULL;
 710     }
 711
 712   nat_6t_o2i_flow_init (sm, thread_index, s, ip->src_address, 0,
 713                         ip->dst_address, 0, rx_fib_index, ip->protocol);
 714   nat_6t_flow_daddr_rewrite_set (&s->o2i, m->local_addr.as_u32);
 715   nat_6t_flow_txfib_rewrite_set (&s->o2i, m->fib_index);
 716   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 717     {
 718       nat_elog_notice (sm, "out2in flow hash add failed");
 719       nat_ed_session_delete (sm, s, thread_index, 1);
 720       return NULL;
 721     }
 722
 723   per_vrf_sessions_register_session (s, thread_index);
 724
 725   /* Accounting */
 726   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 727                                  thread_index);
 728   /* Per-user LRU list maintenance */
 729   nat44_session_update_lru (sm, s, thread_index);
 730
 731   return s;
 732 }
 733
 734 static inline uword
 735 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 736                                           vlib_node_runtime_t * node,
 737                                           vlib_frame_t * frame,
 738                                           int is_multi_worker)
 739 {
 740   u32 n_left_from, *from;
 741   snat_main_t *sm = &snat_main;
 742   f64 now = vlib_time_now (vm);
 743   u32 thread_index = vm->thread_index;
 744   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 745
 746   from = vlib_frame_vector_args (frame);
 747   n_left_from = frame->n_vectors;
 748
 749   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 750   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 751   vlib_get_buffers (vm, from, b, n_left_from);
 752
 753   while (n_left_from > 0)
 754     {
 755       vlib_buffer_t *b0;
 756       u32 sw_if_index0, rx_fib_index0;
 757       ip_protocol_t proto0;
 758       ip4_header_t *ip0;
 759       snat_session_t *s0 = 0;
 760       clib_bihash_kv_16_8_t kv0, value0;
 761       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
 762       nat_slow_path_reason_e slow_path_reason = NAT_ED_SP_REASON_NO_REASON;
 763       nat_6t_flow_t *f = 0;
 764       nat_6t_t lookup;
 765       int lookup_skipped = 0;
 766
 767       b0 = *b;
 768       b++;
 769
 770       /* Prefetch next iteration. */
 771       if (PREDICT_TRUE (n_left_from >= 2))
 772         {
 773           vlib_buffer_t *p2;
 774
 775           p2 = *b;
 776
 777           vlib_prefetch_buffer_header (p2, LOAD);
 778
 779           clib_prefetch_load (p2->data);
 780         }
 781
 782       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 783
 784       lookup.sport = vnet_buffer (b0)->ip.reass.l4_src_port;
 785       lookup.dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
 786
 787       vnet_buffer (b0)->snat.flags = 0;
 788       ip0 = vlib_buffer_get_current (b0);
 789
 790       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 791       rx_fib_index0 =
 792         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 793
 794       lookup.fib_index = rx_fib_index0;
 795
 796       if (PREDICT_FALSE (ip0->ttl == 1))
 797         {
 798           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 799           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 800                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 801                                        0);
 802           next[0] = NAT_NEXT_ICMP_ERROR;
 803           goto trace0;
 804         }
 805
 806       proto0 = ip0->protocol;
 807
 808       if (PREDICT_FALSE (proto0 == IP_PROTOCOL_ICMP))
 809         {
 810           if (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 811                 ICMP4_echo_request &&
 812               vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 813                 ICMP4_echo_reply &&
 814               !icmp_type_is_error_message (
 815                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 816             {
 817               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 818               next[0] = NAT_NEXT_DROP;
 819               goto trace0;
 820             }
 821           int err = nat_get_icmp_session_lookup_values (
 822             b0, ip0, &lookup.saddr, &lookup.sport, &lookup.daddr,
 823             &lookup.dport, &lookup.proto);
 824           if (err != 0)
 825             {
 826               b0->error = node->errors[err];
 827               next[0] = NAT_NEXT_DROP;
 828               goto trace0;
 829             }
 830         }
 831       else
 832         {
 833           lookup.saddr.as_u32 = ip0->src_address.as_u32;
 834           lookup.daddr.as_u32 = ip0->dst_address.as_u32;
 835           lookup.proto = ip0->protocol;
 836         }
 837
 838       /* there might be a stashed index in vnet_buffer2 from handoff or
 839        * classify node, see if it can be used */
 840       if (is_multi_worker &&
 841           !pool_is_free_index (tsm->sessions,
 842                                vnet_buffer2 (b0)->nat.cached_session_index))
 843         {
 844           s0 = pool_elt_at_index (tsm->sessions,
 845                                   vnet_buffer2 (b0)->nat.cached_session_index);
 846           if (PREDICT_TRUE (nat_6t_t_eq (&s0->o2i.match, &lookup)) ||
 847               (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 848                nat_6t_t_eq (&s0->i2o.match, &lookup)))
 849             {
 850               /* yes, this is the droid we're looking for */
 851               lookup_skipped = 1;
 852               goto skip_lookup;
 853             }
 854           s0 = NULL;
 855         }
 856
 857       init_ed_k (&kv0, lookup.saddr.as_u32, lookup.sport, lookup.daddr.as_u32,
 858                  lookup.dport, lookup.fib_index, lookup.proto);
 859
 860       // lookup flow
 861       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
 862         {
 863           // flow does not exist go slow path
 864           slow_path_reason = NAT_ED_SP_REASON_LOOKUP_FAILED;
 865           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 866           goto trace0;
 867         }
 868       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 869       s0 =
 870         pool_elt_at_index (tsm->sessions,
 871                            ed_value_get_session_index (&value0));
 872     skip_lookup:
 873
 874       ASSERT (thread_index == s0->thread_index);
 875
 876       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 877         {
 878           // session is closed, go slow path
 879           nat44_ed_free_session_data (sm, s0, thread_index, 0);
 880           nat_ed_session_delete (sm, s0, thread_index, 1);
 881           slow_path_reason = NAT_ED_SP_REASON_VRF_EXPIRED;
 882           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 883           goto trace0;
 884         }
 885
 886       if (s0->tcp_closed_timestamp)
 887         {
 888           if (now >= s0->tcp_closed_timestamp)
 889             {
 890               // session is closed, go slow path, freed in slow path
 891               slow_path_reason = NAT_ED_SP_TCP_CLOSED;
 892               next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 893             }
 894           else
 895             {
 896               // session in transitory timeout, drop
 897               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
 898               next[0] = NAT_NEXT_DROP;
 899             }
 900           goto trace0;
 901         }
 902
 903       // drop if session expired
 904       u64 sess_timeout_time;
 905       sess_timeout_time =
 906         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 907       if (now >= sess_timeout_time)
 908         {
 909           // session is closed, go slow path
 910           nat44_ed_free_session_data (sm, s0, thread_index, 0);
 911           nat_ed_session_delete (sm, s0, thread_index, 1);
 912           slow_path_reason = NAT_ED_SP_SESS_EXPIRED;
 913           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 914           goto trace0;
 915         }
 916
 917       if (nat_6t_t_eq (&s0->o2i.match, &lookup))
 918         {
 919           f = &s0->o2i;
 920         }
 921       else if (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 922                nat_6t_t_eq (&s0->i2o.match, &lookup))
 923         {
 924           f = &s0->i2o;
 925         }
 926       else
 927         {
 928           /*
 929            * Send DHCP packets to the ipv4 stack, or we won't
 930            * be able to use dhcp client on the outside interface
 931            */
 932           if (PREDICT_FALSE (
 933                 proto0 == IP_PROTOCOL_UDP &&
 934                 (vnet_buffer (b0)->ip.reass.l4_dst_port ==
 935                  clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
 936             {
 937               goto trace0;
 938             }
 939
 940           if (!sm->forwarding_enabled)
 941             {
 942               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 943               next[0] = NAT_NEXT_DROP;
 944               goto trace0;
 945             }
 946           else
 947             {
 948               if (nat_6t_t_eq (&s0->i2o.match, &lookup))
 949                 {
 950                   f = &s0->i2o;
 951                 }
 952               else
 953                 {
 954                   // FIXME TODO bypass ???
 955                   //  create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
 956                   //                       thread_index);
 957                   translation_error = NAT_ED_TRNSL_ERR_FLOW_MISMATCH;
 958                   nat44_ed_free_session_data (sm, s0, thread_index, 0);
 959                   nat_ed_session_delete (sm, s0, thread_index, 1);
 960                   next[0] = NAT_NEXT_DROP;
 961                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
 962                   goto trace0;
 963                 }
 964             }
 965         }
 966
 967       if (NAT_ED_TRNSL_ERR_SUCCESS !=
 968           (translation_error = nat_6t_flow_buf_translate_o2i (
 969              vm, sm, b0, ip0, f, proto0, 0 /* is_output_feature */)))
 970         {
 971           next[0] = NAT_NEXT_DROP;
 972           b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
 973           goto trace0;
 974         }
 975
 976       switch (proto0)
 977         {
 978         case IP_PROTOCOL_TCP:
 979           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.tcp,
 980                                          thread_index, sw_if_index0, 1);
 981           nat44_set_tcp_session_state_o2i (sm, now, s0,
 982                                            vnet_buffer (b0)->ip.
 983                                            reass.icmp_type_or_tcp_flags,
 984                                            vnet_buffer (b0)->ip.
 985                                            reass.tcp_ack_number,
 986                                            vnet_buffer (b0)->ip.
 987                                            reass.tcp_seq_number,
 988                                            thread_index);
 989           break;
 990         case IP_PROTOCOL_UDP:
 991           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.udp,
 992                                          thread_index, sw_if_index0, 1);
 993           break;
 994         case IP_PROTOCOL_ICMP:
 995           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.icmp,
 996                                          thread_index, sw_if_index0, 1);
 997           break;
 998         default:
 999           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.other,
1000                                          thread_index, sw_if_index0, 1);
1001           break;
1002         }
1003
1004       /* Accounting */
1005       nat44_session_update_counters (s0, now,
1006                                      vlib_buffer_length_in_chain (vm, b0),
1007                                      thread_index);
1008       /* Per-user LRU list maintenance */
1009       nat44_session_update_lru (sm, s0, thread_index);
1010
1011     trace0:
1012       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1013                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1014         {
1015           nat44_ed_out2in_trace_t *t =
1016             vlib_add_trace (vm, node, b0, sizeof (*t));
1017           t->sw_if_index = sw_if_index0;
1018           t->next_index = next[0];
1019           t->is_slow_path = 0;
1020           t->translation_error = translation_error;
1021           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1022           t->lookup_skipped = lookup_skipped;
1023           t->slow_path_reason = slow_path_reason;
1024
1025           if (s0)
1026             {
1027               t->session_index = s0 - tsm->sessions;
1028               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1029               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1030               t->translation_via_i2of = (&s0->i2o == f);
1031             }
1032           else
1033             {
1034               t->session_index = ~0;
1035             }
1036         }
1037
1038       if (next[0] == NAT_NEXT_DROP)
1039         {
1040           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.drops,
1041                                          thread_index, sw_if_index0, 1);
1042         }
1043
1044       n_left_from--;
1045       next++;
1046     }
1047
1048   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1049                                frame->n_vectors);
1050   return frame->n_vectors;
1051 }
1052
1053 static inline uword
1054 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1055                                           vlib_node_runtime_t * node,
1056                                           vlib_frame_t * frame)
1057 {
1058   u32 n_left_from, *from;
1059   snat_main_t *sm = &snat_main;
1060   f64 now = vlib_time_now (vm);
1061   u32 thread_index = vm->thread_index;
1062   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1063   snat_static_mapping_t *m;
1064
1065   from = vlib_frame_vector_args (frame);
1066   n_left_from = frame->n_vectors;
1067
1068   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1069   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1070   vlib_get_buffers (vm, from, b, n_left_from);
1071
1072   while (n_left_from > 0)
1073     {
1074       vlib_buffer_t *b0;
1075       u32 sw_if_index0, rx_fib_index0;
1076       ip_protocol_t proto0;
1077       ip4_header_t *ip0;
1078       udp_header_t *udp0;
1079       icmp46_header_t *icmp0;
1080       snat_session_t *s0 = 0;
1081       clib_bihash_kv_16_8_t kv0, value0;
1082       lb_nat_type_t lb_nat0;
1083       twice_nat_type_t twice_nat0;
1084       u8 identity_nat0;
1085       ip4_address_t sm_addr;
1086       u16 sm_port;
1087       u32 sm_fib_index;
1088       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
1089
1090       b0 = *b;
1091       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1092
1093       vnet_buffer (b0)->snat.flags = 0;
1094       ip0 = vlib_buffer_get_current (b0);
1095
1096       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1097       rx_fib_index0 =
1098         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1099
1100       if (PREDICT_FALSE (ip0->ttl == 1))
1101         {
1102           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1103           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1104                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1105                                        0);
1106           next[0] = NAT_NEXT_ICMP_ERROR;
1107           goto trace0;
1108         }
1109
1110       udp0 = ip4_next_header (ip0);
1111       icmp0 = (icmp46_header_t *) udp0;
1112       proto0 = ip0->protocol;
1113
1114       if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto0)))
1115         {
1116           s0 = nat44_ed_out2in_slowpath_unknown_proto (
1117             sm, b0, ip0, rx_fib_index0, thread_index, now, vm, node);
1118           if (!sm->forwarding_enabled)
1119             {
1120               if (!s0)
1121                 next[0] = NAT_NEXT_DROP;
1122             }
1123           if (NAT_NEXT_DROP != next[0] && s0 &&
1124               NAT_ED_TRNSL_ERR_SUCCESS !=
1125                 (translation_error = nat_6t_flow_buf_translate_o2i (
1126                    vm, sm, b0, ip0, &s0->o2i, proto0,
1127                    0 /* is_output_feature */)))
1128             {
1129               next[0] = NAT_NEXT_DROP;
1130               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1131               goto trace0;
1132             }
1133
1134           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.other,
1135                                          thread_index, sw_if_index0, 1);
1136           goto trace0;
1137         }
1138
1139       if (PREDICT_FALSE (proto0 == IP_PROTOCOL_ICMP))
1140         {
1141           next[0] = icmp_out2in_ed_slow_path
1142             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1143              next[0], now, thread_index, &s0);
1144
1145           if (NAT_NEXT_DROP != next[0] && s0 &&
1146               NAT_ED_TRNSL_ERR_SUCCESS !=
1147                 (translation_error = nat_6t_flow_buf_translate_o2i (
1148                    vm, sm, b0, ip0, &s0->o2i, proto0,
1149                    0 /* is_output_feature */)))
1150             {
1151               next[0] = NAT_NEXT_DROP;
1152               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1153               goto trace0;
1154             }
1155
1156           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.icmp,
1157                                          thread_index, sw_if_index0, 1);
1158           goto trace0;
1159         }
1160
1161       init_ed_k (
1162         &kv0, ip0->src_address.as_u32, vnet_buffer (b0)->ip.reass.l4_src_port,
1163         ip0->dst_address.as_u32, vnet_buffer (b0)->ip.reass.l4_dst_port,
1164         rx_fib_index0, ip0->protocol);
1165
1166       s0 = NULL;
1167       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
1168         {
1169           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1170           s0 =
1171             pool_elt_at_index (tsm->sessions,
1172                                ed_value_get_session_index (&value0));
1173
1174           if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1175             {
1176               nat44_ed_free_session_data (sm, s0, thread_index, 0);
1177               nat_ed_session_delete (sm, s0, thread_index, 1);
1178               s0 = NULL;
1179             }
1180         }
1181
1182       if (!s0)
1183         {
1184           /* Try to match static mapping by external address and port,
1185              destination address and port in packet */
1186
1187           if (snat_static_mapping_match (
1188                 vm, sm, ip0->dst_address,
1189                 vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0, proto0,
1190                 &sm_addr, &sm_port, &sm_fib_index, 1, 0, &twice_nat0, &lb_nat0,
1191                 &ip0->src_address, &identity_nat0, &m))
1192             {
1193               /*
1194                * Send DHCP packets to the ipv4 stack, or we won't
1195                * be able to use dhcp client on the outside interface
1196                */
1197               if (PREDICT_FALSE (
1198                     proto0 == IP_PROTOCOL_UDP &&
1199                     (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1200                      clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
1201                 {
1202                   goto trace0;
1203                 }
1204
1205               if (!sm->forwarding_enabled)
1206                 {
1207                   b0->error =
1208                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1209                   next[0] = NAT_NEXT_DROP;
1210                 }
1211               else
1212                 {
1213                   if (next_src_nat (
1214                         sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1215                         vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0))
1216                     {
1217                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1218                     }
1219                   else
1220                     {
1221                       create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
1222                                              thread_index);
1223                     }
1224                 }
1225               goto trace0;
1226             }
1227
1228           if (PREDICT_FALSE (identity_nat0))
1229             goto trace0;
1230
1231           if ((proto0 == IP_PROTOCOL_TCP) &&
1232               !tcp_flags_is_init (
1233                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
1234             {
1235               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1236               next[0] = NAT_NEXT_DROP;
1237               goto trace0;
1238             }
1239
1240           /* Create session initiated by host from external network */
1241           s0 = create_session_for_static_mapping_ed (
1242             sm, b0, sm_addr, sm_port, sm_fib_index, ip0->dst_address,
1243             vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0, proto0,
1244             node, thread_index, twice_nat0, lb_nat0, now, m);
1245           if (!s0)
1246             {
1247               next[0] = NAT_NEXT_DROP;
1248               goto trace0;
1249             }
1250         }
1251
1252       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1253           (translation_error = nat_6t_flow_buf_translate_o2i (
1254              vm, sm, b0, ip0, &s0->o2i, proto0, 0 /* is_output_feature */)))
1255         {
1256           next[0] = NAT_NEXT_DROP;
1257           goto trace0;
1258         }
1259
1260       if (PREDICT_TRUE (proto0 == IP_PROTOCOL_TCP))
1261         {
1262           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.tcp,
1263                                          thread_index, sw_if_index0, 1);
1264           nat44_set_tcp_session_state_o2i (sm, now, s0,
1265                                            vnet_buffer (b0)->ip.
1266                                            reass.icmp_type_or_tcp_flags,
1267                                            vnet_buffer (b0)->ip.
1268                                            reass.tcp_ack_number,
1269                                            vnet_buffer (b0)->ip.
1270                                            reass.tcp_seq_number,
1271                                            thread_index);
1272         }
1273       else
1274         {
1275           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.udp,
1276                                          thread_index, sw_if_index0, 1);
1277         }
1278
1279       /* Accounting */
1280       nat44_session_update_counters (s0, now,
1281                                      vlib_buffer_length_in_chain (vm, b0),
1282                                      thread_index);
1283       /* Per-user LRU list maintenance */
1284       nat44_session_update_lru (sm, s0, thread_index);
1285
1286     trace0:
1287       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1288                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1289         {
1290           nat44_ed_out2in_trace_t *t =
1291             vlib_add_trace (vm, node, b0, sizeof (*t));
1292           t->sw_if_index = sw_if_index0;
1293           t->next_index = next[0];
1294           t->is_slow_path = 1;
1295           t->translation_error = translation_error;
1296           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1297
1298           if (s0)
1299             {
1300               t->session_index = s0 - tsm->sessions;
1301               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1302               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1303             }
1304           else
1305             {
1306               t->session_index = ~0;
1307             }
1308         }
1309
1310       if (next[0] == NAT_NEXT_DROP)
1311         {
1312           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.drops,
1313                                          thread_index, sw_if_index0, 1);
1314         }
1315
1316       n_left_from--;
1317       next++;
1318       b++;
1319     }
1320
1321   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1322                                frame->n_vectors);
1323
1324   return frame->n_vectors;
1325 }
1326
1327 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1328                                      vlib_node_runtime_t * node,
1329                                      vlib_frame_t * frame)
1330 {
1331   if (snat_main.num_workers > 1)
1332     {
1333       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1334     }
1335   else
1336     {
1337       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1338     }
1339 }
1340
1341 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1342   .name = "nat44-ed-out2in",
1343   .vector_size = sizeof (u32),
1344   .sibling_of = "nat-default",
1345   .format_trace = format_nat44_ed_out2in_trace,
1346   .type = VLIB_NODE_TYPE_INTERNAL,
1347   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1348   .error_strings = nat_out2in_ed_error_strings,
1349   .runtime_data_bytes = sizeof (snat_runtime_t),
1350 };
1351
1352 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1353                                               vlib_node_runtime_t * node,
1354                                               vlib_frame_t * frame)
1355 {
1356   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1357 }
1358
1359 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1360   .name = "nat44-ed-out2in-slowpath",
1361   .vector_size = sizeof (u32),
1362   .sibling_of = "nat-default",
1363   .format_trace = format_nat44_ed_out2in_trace,
1364   .type = VLIB_NODE_TYPE_INTERNAL,
1365   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1366   .error_strings = nat_out2in_ed_error_strings,
1367   .runtime_data_bytes = sizeof (snat_runtime_t),
1368 };
1369
1370 static u8 *
1371 format_nat_pre_trace (u8 * s, va_list * args)
1372 {
1373   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1374   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1375   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1376   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1377                  t->arc_next_index);
1378 }
1379
1380 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1381                                     vlib_node_runtime_t * node,
1382                                     vlib_frame_t * frame)
1383 {
1384   return nat_pre_node_fn_inline (vm, node, frame,
1385                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1386 }
1387
1388 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1389   .name = "nat-pre-out2in",
1390   .vector_size = sizeof (u32),
1391   .sibling_of = "nat-default",
1392   .format_trace = format_nat_pre_trace,
1393   .type = VLIB_NODE_TYPE_INTERNAL,
1394   .n_errors = 0,
1395  };
1396
1397 /*
1398  * fd.io coding-style-patch-verification: ON
1399  *
1400  * Local Variables:
1401  * eval: (c-set-style "gnu")
1402  * End:
1403  */