src/plugins/nat/nat44-ed/nat44_ed_out2in.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27
  28 #include <nat/lib/ipfix_logging.h>
  29
  30 #include <nat/nat44-ed/nat44_ed.h>
  31 #include <nat/nat44-ed/nat44_ed_inlines.h>
  32
  33 static char *nat_out2in_ed_error_strings[] = {
  34 #define _(sym,string) string,
  35   foreach_nat_out2in_ed_error
  36 #undef _
  37 };
  38
  39 typedef enum
  40 {
  41   NAT_ED_SP_REASON_NO_REASON,
  42   NAT_ED_SP_REASON_LOOKUP_FAILED,
  43   NAT_ED_SP_REASON_VRF_EXPIRED,
  44   NAT_ED_SP_SESS_EXPIRED,
  45 } nat_slow_path_reason_e;
  46
  47 typedef struct
  48 {
  49   u32 sw_if_index;
  50   u32 next_index;
  51   u32 session_index;
  52   nat_translation_error_e translation_error;
  53   nat_6t_flow_t i2of;
  54   nat_6t_flow_t o2if;
  55   clib_bihash_kv_16_8_t search_key;
  56   u8 is_slow_path;
  57   u8 translation_via_i2of;
  58   u8 lookup_skipped;
  59   u8 tcp_state;
  60   nat_slow_path_reason_e slow_path_reason;
  61 } nat44_ed_out2in_trace_t;
  62
  63 static u8 *
  64 format_slow_path_reason (u8 *s, va_list *args)
  65 {
  66   nat_slow_path_reason_e reason = va_arg (*args, nat_slow_path_reason_e);
  67   switch (reason)
  68     {
  69     case NAT_ED_SP_REASON_NO_REASON:
  70       return format (s, "no reason for slow path");
  71     case NAT_ED_SP_REASON_LOOKUP_FAILED:
  72       return format (s, "slow path because lookup failed");
  73     case NAT_ED_SP_REASON_VRF_EXPIRED:
  74       return format (s, "slow path because vrf expired");
  75     case NAT_ED_SP_SESS_EXPIRED:
  76       return format (s, "slow path because session expired");
  77     }
  78   return format (s, "invalid reason value");
  79 }
  80
  81 static u8 *
  82 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  83 {
  84   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  85   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  86   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  87   char *tag;
  88
  89   tag =
  90     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  91     "NAT44_OUT2IN_ED_FAST_PATH";
  92
  93   s = format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
  94               t->next_index);
  95   if (~0 != t->session_index)
  96     {
  97       s = format (s, ", session %d, translation result '%U' via %s",
  98                   t->session_index, format_nat_ed_translation_error,
  99                   t->translation_error,
 100                   t->translation_via_i2of ? "i2of" : "o2if");
 101       s = format (s, "\n  i2of %U", format_nat_6t_flow, &t->i2of);
 102       s = format (s, "\n  o2if %U", format_nat_6t_flow, &t->o2if);
 103     }
 104   if (!t->is_slow_path)
 105     {
 106       if (t->lookup_skipped)
 107         {
 108           s = format (s, "\n  lookup skipped - cached session index used");
 109         }
 110       else
 111         {
 112           s = format (s, "\n  search key %U", format_ed_session_kvp,
 113                       &t->search_key);
 114         }
 115       s = format (s, "\n  %U", format_slow_path_reason, t->slow_path_reason);
 116     }
 117   if (IP_PROTOCOL_TCP == t->i2of.match.proto)
 118     {
 119       s = format (s, "\n  TCP state: %U", format_nat44_ed_tcp_state,
 120                   t->tcp_state);
 121     }
 122
 123   return s;
 124 }
 125
 126 static int
 127 next_src_nat (snat_main_t *sm, ip4_header_t *ip, u16 src_port, u16 dst_port,
 128               u32 rx_fib_index)
 129 {
 130   clib_bihash_kv_16_8_t kv, value;
 131
 132   init_ed_k (&kv, ip->src_address.as_u32, src_port, ip->dst_address.as_u32,
 133              dst_port, rx_fib_index, ip->protocol);
 134   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 135     return 1;
 136
 137   return 0;
 138 }
 139
 140 static void create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b,
 141                                    snat_session_t *s, ip4_header_t *ip,
 142                                    u32 rx_fib_index, u32 thread_index);
 143
 144 static snat_session_t *create_session_for_static_mapping_ed (
 145   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 146   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 147   ip_protocol_t proto, vlib_node_runtime_t *node, u32 thread_index,
 148   twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 149   snat_static_mapping_t *mapping);
 150
 151 static inline u32
 152 icmp_out2in_ed_slow_path (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
 153                           icmp46_header_t *icmp, u32 sw_if_index,
 154                           u32 rx_fib_index, vlib_node_runtime_t *node,
 155                           u32 next, f64 now, u32 thread_index,
 156                           snat_session_t **s_p)
 157 {
 158   vlib_main_t *vm = vlib_get_main ();
 159
 160   ip_csum_t sum;
 161   u16 checksum;
 162
 163   snat_session_t *s = 0;
 164   u8 is_addr_only, identity_nat;
 165   ip4_address_t sm_addr;
 166   u16 sm_port;
 167   u32 sm_fib_index;
 168   snat_static_mapping_t *m;
 169   u8 lookup_protocol;
 170   ip4_address_t lookup_saddr, lookup_daddr;
 171   u16 lookup_sport, lookup_dport;
 172
 173   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 174   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 175
 176   if (nat_get_icmp_session_lookup_values (b, ip, &lookup_saddr, &lookup_sport,
 177                                           &lookup_daddr, &lookup_dport,
 178                                           &lookup_protocol))
 179     {
 180       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 181       next = NAT_NEXT_DROP;
 182       goto out;
 183     }
 184
 185   if (snat_static_mapping_match (vm, ip->dst_address, lookup_sport,
 186                                  rx_fib_index, ip->protocol, &sm_addr,
 187                                  &sm_port, &sm_fib_index, 1, &is_addr_only, 0,
 188                                  0, 0, &identity_nat, &m))
 189     {
 190       // static mapping not matched
 191       if (!sm->forwarding_enabled)
 192         {
 193           /* Don't NAT packet aimed at the intfc address */
 194           if (!is_interface_addr (sm, node, sw_if_index,
 195                                   ip->dst_address.as_u32))
 196             {
 197               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 198               next = NAT_NEXT_DROP;
 199             }
 200         }
 201       else
 202         {
 203           if (next_src_nat (sm, ip, lookup_sport, lookup_dport, rx_fib_index))
 204             {
 205               next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 206             }
 207           else
 208             {
 209               create_bypass_for_fwd (sm, b, s, ip, rx_fib_index, thread_index);
 210             }
 211         }
 212       goto out;
 213     }
 214
 215   if (PREDICT_FALSE (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 216                        ICMP4_echo_reply &&
 217                      (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 218                         ICMP4_echo_request ||
 219                       !is_addr_only)))
 220     {
 221       b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 222       next = NAT_NEXT_DROP;
 223       goto out;
 224     }
 225
 226   if (PREDICT_FALSE (identity_nat))
 227     {
 228       goto out;
 229     }
 230
 231   /* Create session initiated by host from external network */
 232   s = create_session_for_static_mapping_ed (
 233     sm, b, sm_addr, sm_port, sm_fib_index, ip->dst_address, lookup_sport,
 234     rx_fib_index, lookup_protocol, node, thread_index, 0, 0,
 235     vlib_time_now (vm), m);
 236   if (!s)
 237     next = NAT_NEXT_DROP;
 238
 239   if (PREDICT_TRUE (!ip4_is_fragment (ip)))
 240     {
 241       sum = ip_incremental_checksum_buffer (
 242         vm, b, (u8 *) icmp - (u8 *) vlib_buffer_get_current (b),
 243         ntohs (ip->length) - ip4_header_bytes (ip), 0);
 244       checksum = ~ip_csum_fold (sum);
 245       if (checksum != 0 && checksum != 0xffff)
 246         {
 247           next = NAT_NEXT_DROP;
 248           goto out;
 249         }
 250     }
 251
 252   if (PREDICT_TRUE (next != NAT_NEXT_DROP && s))
 253     {
 254       /* Accounting */
 255       nat44_session_update_counters (
 256         s, now, vlib_buffer_length_in_chain (vm, b), thread_index);
 257       /* Per-user LRU list maintenance */
 258       nat44_session_update_lru (sm, s, thread_index);
 259     }
 260 out:
 261   if (NAT_NEXT_DROP == next && s)
 262     {
 263       nat_ed_session_delete (sm, s, thread_index, 1);
 264       s = 0;
 265     }
 266   *s_p = s;
 267   return next;
 268 }
 269
 270 static_always_inline int
 271 nat44_ed_alloc_i2o_port (snat_main_t *sm, snat_address_t *a, snat_session_t *s,
 272                          ip4_address_t i2o_addr, u16 i2o_port,
 273                          u32 i2o_fib_index, ip_protocol_t proto,
 274                          u32 thread_index, u32 snat_thread_index,
 275                          ip4_address_t *outside_addr, u16 *outside_port)
 276 {
 277   u32 portnum;
 278
 279   for (int i = 0; i < ED_PORT_ALLOC_ATTEMPTS; ++i)
 280     {
 281       portnum = (sm->port_per_thread * snat_thread_index) +
 282                 snat_random_port (0, sm->port_per_thread - 1) + 1024;
 283       portnum = clib_host_to_net_u16 (portnum);
 284       nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port, a->addr,
 285                             portnum, i2o_fib_index, proto);
 286       if (!nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s,
 287                                              1 /* is_add */))
 288         {
 289           *outside_addr = a->addr;
 290           *outside_port = portnum;
 291           return 0;
 292         }
 293     }
 294
 295   /* Totally out of translations to use... */
 296   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 297   return 1;
 298 }
 299
 300 static_always_inline int
 301 nat44_ed_alloc_i2o_addr_and_port (snat_main_t *sm, snat_address_t *addresses,
 302                                   snat_session_t *s, ip4_address_t i2o_addr,
 303                                   u16 i2o_port, u32 i2o_fib_index,
 304                                   ip_protocol_t proto, u32 thread_index,
 305                                   u32 snat_thread_index,
 306                                   ip4_address_t *outside_addr,
 307                                   u16 *outside_port)
 308 {
 309   snat_address_t *a, *ga = 0;
 310   int i;
 311
 312   if (vec_len (addresses) > 0)
 313     {
 314       int s_addr_offset = i2o_addr.as_u32 % vec_len (addresses);
 315
 316       for (i = s_addr_offset; i < vec_len (addresses); ++i)
 317         {
 318           a = addresses + i;
 319           if (a->fib_index == i2o_fib_index)
 320             {
 321               return nat44_ed_alloc_i2o_port (
 322                 sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 323                 thread_index, snat_thread_index, outside_addr, outside_port);
 324             }
 325           else if (a->fib_index == ~0)
 326             {
 327               ga = a;
 328             }
 329         }
 330
 331       for (i = 0; i < s_addr_offset; ++i)
 332         {
 333           a = addresses + i;
 334           if (a->fib_index == i2o_fib_index)
 335             {
 336               return nat44_ed_alloc_i2o_port (
 337                 sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 338                 thread_index, snat_thread_index, outside_addr, outside_port);
 339             }
 340           else if (a->fib_index == ~0)
 341             {
 342               ga = a;
 343             }
 344         }
 345
 346       if (ga)
 347         {
 348           return nat44_ed_alloc_i2o_port (
 349             sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto, thread_index,
 350             snat_thread_index, outside_addr, outside_port);
 351         }
 352     }
 353
 354   /* Totally out of translations to use... */
 355   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 356   return 1;
 357 }
 358
 359 static snat_session_t *
 360 create_session_for_static_mapping_ed (
 361   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 362   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 363   ip_protocol_t proto, vlib_node_runtime_t *node, u32 thread_index,
 364   twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 365   snat_static_mapping_t *mapping)
 366 {
 367   snat_session_t *s;
 368   ip4_header_t *ip;
 369   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 370
 371   if (PREDICT_FALSE (
 372         nat44_ed_maximum_sessions_exceeded (sm, o2i_fib_index, thread_index)))
 373     {
 374       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 375       nat_elog_notice (sm, "maximum sessions exceeded");
 376       return 0;
 377     }
 378
 379   s = nat_ed_session_alloc (sm, thread_index, now, proto);
 380   if (!s)
 381     {
 382       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 383       nat_elog_warn (sm, "create NAT session failed");
 384       return 0;
 385     }
 386
 387   ip = vlib_buffer_get_current (b);
 388
 389   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 390   s->ext_host_port =
 391     proto == IP_PROTOCOL_ICMP ? 0 : vnet_buffer (b)->ip.reass.l4_src_port;
 392   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 393   if (lb_nat)
 394     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 395   if (lb_nat == AFFINITY_LB_NAT)
 396     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 397   s->out2in.addr = o2i_addr;
 398   s->out2in.port = o2i_port;
 399   s->out2in.fib_index = o2i_fib_index;
 400   s->in2out.addr = i2o_addr;
 401   s->in2out.port = i2o_port;
 402   s->in2out.fib_index = i2o_fib_index;
 403   s->proto = proto;
 404
 405   if (IP_PROTOCOL_ICMP == proto)
 406     {
 407       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr, o2i_port,
 408                             o2i_addr, o2i_port, o2i_fib_index, ip->protocol);
 409       nat_6t_flow_icmp_id_rewrite_set (&s->o2i, i2o_port);
 410     }
 411   else
 412     {
 413       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr,
 414                             s->ext_host_port, o2i_addr, o2i_port,
 415                             o2i_fib_index, ip->protocol);
 416       nat_6t_flow_dport_rewrite_set (&s->o2i, i2o_port);
 417     }
 418   nat_6t_flow_daddr_rewrite_set (&s->o2i, i2o_addr.as_u32);
 419   nat_6t_flow_txfib_rewrite_set (&s->o2i, i2o_fib_index);
 420
 421   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 422     {
 423       b->error = node->errors[NAT_OUT2IN_ED_ERROR_HASH_ADD_FAILED];
 424       nat_ed_session_delete (sm, s, thread_index, 1);
 425       nat_elog_warn (sm, "out2in flow hash add failed");
 426       return 0;
 427     }
 428
 429   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 430                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 431     {
 432       int rc = 0;
 433       snat_address_t *filter = 0;
 434
 435       // if exact address is specified use this address
 436       if (is_sm_exact_address (mapping->flags))
 437         {
 438           snat_address_t *ap;
 439           vec_foreach (ap, sm->twice_nat_addresses)
 440           {
 441             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 442               {
 443                 filter = ap;
 444                 break;
 445               }
 446           }
 447         }
 448
 449       if (filter)
 450         {
 451           rc = nat44_ed_alloc_i2o_port (
 452             sm, filter, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 453             thread_index, tsm->snat_thread_index, &s->ext_host_nat_addr,
 454             &s->ext_host_nat_port);
 455           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 456         }
 457       else
 458         {
 459           rc = nat44_ed_alloc_i2o_addr_and_port (
 460             sm, sm->twice_nat_addresses, s, i2o_addr, i2o_port, i2o_fib_index,
 461             proto, thread_index, tsm->snat_thread_index, &s->ext_host_nat_addr,
 462             &s->ext_host_nat_port);
 463         }
 464
 465       if (rc)
 466         {
 467           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 468           nat_ed_session_delete (sm, s, thread_index, 1);
 469           return 0;
 470         }
 471
 472       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 473
 474       nat_6t_flow_saddr_rewrite_set (&s->o2i, s->ext_host_nat_addr.as_u32);
 475       if (IP_PROTOCOL_ICMP == proto)
 476         {
 477           nat_6t_flow_icmp_id_rewrite_set (&s->o2i, s->ext_host_nat_port);
 478         }
 479       else
 480         {
 481           nat_6t_flow_sport_rewrite_set (&s->o2i, s->ext_host_nat_port);
 482         }
 483
 484       nat_6t_l3_l4_csum_calc (&s->o2i);
 485
 486       nat_6t_flow_daddr_rewrite_set (&s->i2o, s->ext_host_addr.as_u32);
 487       if (IP_PROTOCOL_ICMP == proto)
 488         {
 489           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, s->ext_host_port);
 490         }
 491       else
 492         {
 493           nat_6t_flow_dport_rewrite_set (&s->i2o, s->ext_host_port);
 494         }
 495
 496       nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 497       if (IP_PROTOCOL_ICMP == proto)
 498         {
 499           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 500         }
 501       else
 502         {
 503           nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 504         }
 505       nat_6t_l3_l4_csum_calc (&s->i2o);
 506     }
 507   else
 508     {
 509       if (IP_PROTOCOL_ICMP == proto)
 510         {
 511           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 512                                 s->ext_host_addr, i2o_port, i2o_fib_index,
 513                                 ip->protocol);
 514         }
 515       else
 516         {
 517           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 518                                 s->ext_host_addr, s->ext_host_port,
 519                                 i2o_fib_index, ip->protocol);
 520         }
 521
 522   nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 523   if (IP_PROTOCOL_ICMP == proto)
 524     {
 525       nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 526     }
 527   else
 528     {
 529       nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 530     }
 531
 532   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 533     {
 534       nat_elog_notice (sm, "in2out flow hash add failed");
 535       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 536         {
 537           nat_elog_warn (sm, "out2in flow hash del failed");
 538         }
 539       nat_ed_session_delete (sm, s, thread_index, 1);
 540       return 0;
 541     }
 542     }
 543   nat_ipfix_logging_nat44_ses_create (
 544     thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 545     s->in2out.port, s->out2in.port, s->in2out.fib_index);
 546
 547   nat_syslog_nat44_sadd (0, s->in2out.fib_index, &s->in2out.addr,
 548                          s->in2out.port, &s->ext_host_nat_addr,
 549                          s->ext_host_nat_port, &s->out2in.addr, s->out2in.port,
 550                          &s->ext_host_addr, s->ext_host_port, s->proto,
 551                          nat44_ed_is_twice_nat_session (s));
 552
 553   per_vrf_sessions_register_session (s, thread_index);
 554
 555   return s;
 556 }
 557
 558 static void
 559 create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b, snat_session_t *s,
 560                        ip4_header_t *ip, u32 rx_fib_index, u32 thread_index)
 561 {
 562   clib_bihash_kv_16_8_t kv, value;
 563   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 564   vlib_main_t *vm = vlib_get_main ();
 565   f64 now = vlib_time_now (vm);
 566   u16 lookup_sport, lookup_dport;
 567   u8 lookup_protocol;
 568   ip4_address_t lookup_saddr, lookup_daddr;
 569
 570   if (ip->protocol == IP_PROTOCOL_ICMP)
 571     {
 572       if (nat_get_icmp_session_lookup_values (b, ip, &lookup_daddr,
 573                                               &lookup_sport, &lookup_saddr,
 574                                               &lookup_dport, &lookup_protocol))
 575         return;
 576     }
 577   else
 578     {
 579       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 580         {
 581           lookup_sport = vnet_buffer (b)->ip.reass.l4_dst_port;
 582           lookup_dport = vnet_buffer (b)->ip.reass.l4_src_port;
 583         }
 584       else
 585         {
 586           lookup_sport = 0;
 587           lookup_dport = 0;
 588         }
 589       lookup_saddr.as_u32 = ip->dst_address.as_u32;
 590       lookup_daddr.as_u32 = ip->src_address.as_u32;
 591       lookup_protocol = ip->protocol;
 592     }
 593
 594   init_ed_k (&kv, lookup_saddr.as_u32, lookup_sport, lookup_daddr.as_u32,
 595              lookup_dport, rx_fib_index, lookup_protocol);
 596
 597   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 598     {
 599       ASSERT (thread_index == ed_value_get_thread_index (&value));
 600       s =
 601         pool_elt_at_index (tsm->sessions,
 602                            ed_value_get_session_index (&value));
 603     }
 604   else if (ip->protocol == IP_PROTOCOL_ICMP &&
 605            icmp_type_is_error_message
 606            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 607     {
 608       return;
 609     }
 610   else
 611     {
 612       if (PREDICT_FALSE
 613           (nat44_ed_maximum_sessions_exceeded
 614            (sm, rx_fib_index, thread_index)))
 615         return;
 616
 617       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 618       if (!s)
 619         {
 620           nat_elog_warn (sm, "create NAT session failed");
 621           return;
 622         }
 623
 624       s->ext_host_addr = ip->src_address;
 625       s->ext_host_port = lookup_dport;
 626       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 627       s->out2in.addr = ip->dst_address;
 628       s->out2in.port = lookup_sport;
 629       s->proto = ip->protocol;
 630       s->out2in.fib_index = rx_fib_index;
 631       s->in2out.addr = s->out2in.addr;
 632       s->in2out.port = s->out2in.port;
 633       s->in2out.fib_index = s->out2in.fib_index;
 634
 635       nat_6t_i2o_flow_init (sm, thread_index, s, ip->dst_address, lookup_sport,
 636                             ip->src_address, lookup_dport, rx_fib_index,
 637                             ip->protocol);
 638       nat_6t_flow_txfib_rewrite_set (&s->i2o, rx_fib_index);
 639       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 640         {
 641           nat_elog_notice (sm, "in2out flow add failed");
 642           nat_ed_session_delete (sm, s, thread_index, 1);
 643           return;
 644         }
 645
 646       per_vrf_sessions_register_session (s, thread_index);
 647     }
 648
 649   if (ip->protocol == IP_PROTOCOL_TCP)
 650     {
 651       nat44_set_tcp_session_state_o2i (
 652         sm, now, s, vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags,
 653         thread_index);
 654     }
 655
 656   /* Accounting */
 657   nat44_session_update_counters (s, now, 0, thread_index);
 658   /* Per-user LRU list maintenance */
 659   nat44_session_update_lru (sm, s, thread_index);
 660 }
 661
 662 static snat_session_t *
 663 nat44_ed_out2in_slowpath_unknown_proto (snat_main_t *sm, vlib_buffer_t *b,
 664                                         ip4_header_t *ip, u32 rx_fib_index,
 665                                         u32 thread_index, f64 now,
 666                                         vlib_main_t *vm,
 667                                         vlib_node_runtime_t *node)
 668 {
 669   snat_static_mapping_t *m;
 670   snat_session_t *s;
 671
 672   if (PREDICT_FALSE (
 673         nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 674     {
 675       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 676       nat_elog_notice (sm, "maximum sessions exceeded");
 677       return 0;
 678     }
 679
 680   m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, ip->protocol);
 681   if (!m)
 682     {
 683       b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 684       return 0;
 685     }
 686
 687   /* Create a new session */
 688   s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 689   if (!s)
 690     {
 691       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 692       nat_elog_warn (sm, "create NAT session failed");
 693       return 0;
 694     }
 695
 696   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 697   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 698   s->out2in.addr.as_u32 = ip->dst_address.as_u32;
 699   s->out2in.fib_index = rx_fib_index;
 700   s->in2out.addr.as_u32 = m->local_addr.as_u32;
 701   s->in2out.fib_index = m->fib_index;
 702   s->in2out.port = s->out2in.port = ip->protocol;
 703
 704   nat_6t_o2i_flow_init (sm, thread_index, s, ip->dst_address, 0,
 705                         ip->src_address, 0, m->fib_index, ip->protocol);
 706   nat_6t_flow_saddr_rewrite_set (&s->i2o, ip->dst_address.as_u32);
 707   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 708     {
 709       nat_elog_notice (sm, "in2out key add failed");
 710       nat_ed_session_delete (sm, s, thread_index, 1);
 711       return NULL;
 712     }
 713
 714   nat_6t_o2i_flow_init (sm, thread_index, s, ip->src_address, 0,
 715                         ip->dst_address, 0, rx_fib_index, ip->protocol);
 716   nat_6t_flow_daddr_rewrite_set (&s->o2i, m->local_addr.as_u32);
 717   nat_6t_flow_txfib_rewrite_set (&s->o2i, m->fib_index);
 718   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 719     {
 720       nat_elog_notice (sm, "out2in flow hash add failed");
 721       nat_ed_session_delete (sm, s, thread_index, 1);
 722       return NULL;
 723     }
 724
 725   per_vrf_sessions_register_session (s, thread_index);
 726
 727   /* Accounting */
 728   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 729                                  thread_index);
 730   /* Per-user LRU list maintenance */
 731   nat44_session_update_lru (sm, s, thread_index);
 732
 733   return s;
 734 }
 735
 736 static inline uword
 737 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 738                                           vlib_node_runtime_t * node,
 739                                           vlib_frame_t * frame,
 740                                           int is_multi_worker)
 741 {
 742   u32 n_left_from, *from;
 743   snat_main_t *sm = &snat_main;
 744   f64 now = vlib_time_now (vm);
 745   u32 thread_index = vm->thread_index;
 746   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 747
 748   from = vlib_frame_vector_args (frame);
 749   n_left_from = frame->n_vectors;
 750
 751   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 752   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 753   vlib_get_buffers (vm, from, b, n_left_from);
 754
 755   while (n_left_from > 0)
 756     {
 757       vlib_buffer_t *b0;
 758       u32 sw_if_index0, rx_fib_index0;
 759       ip_protocol_t proto0;
 760       ip4_header_t *ip0;
 761       snat_session_t *s0 = 0;
 762       clib_bihash_kv_16_8_t kv0, value0;
 763       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
 764       nat_slow_path_reason_e slow_path_reason = NAT_ED_SP_REASON_NO_REASON;
 765       nat_6t_flow_t *f = 0;
 766       nat_6t_t lookup;
 767       int lookup_skipped = 0;
 768
 769       b0 = *b;
 770       b++;
 771
 772       /* Prefetch next iteration. */
 773       if (PREDICT_TRUE (n_left_from >= 2))
 774         {
 775           vlib_buffer_t *p2;
 776
 777           p2 = *b;
 778
 779           vlib_prefetch_buffer_header (p2, LOAD);
 780
 781           clib_prefetch_load (p2->data);
 782         }
 783
 784       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 785
 786       lookup.sport = vnet_buffer (b0)->ip.reass.l4_src_port;
 787       lookup.dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
 788
 789       vnet_buffer (b0)->snat.flags = 0;
 790       ip0 = vlib_buffer_get_current (b0);
 791
 792       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 793       rx_fib_index0 =
 794         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 795
 796       lookup.fib_index = rx_fib_index0;
 797
 798       if (PREDICT_FALSE (ip0->ttl == 1))
 799         {
 800           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 801           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 802                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 803                                        0);
 804           next[0] = NAT_NEXT_ICMP_ERROR;
 805           goto trace0;
 806         }
 807
 808       proto0 = ip0->protocol;
 809
 810       if (PREDICT_FALSE (proto0 == IP_PROTOCOL_ICMP))
 811         {
 812           if (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 813                 ICMP4_echo_request &&
 814               vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 815                 ICMP4_echo_reply &&
 816               !icmp_type_is_error_message (
 817                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 818             {
 819               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 820               next[0] = NAT_NEXT_DROP;
 821               goto trace0;
 822             }
 823           int err = nat_get_icmp_session_lookup_values (
 824             b0, ip0, &lookup.saddr, &lookup.sport, &lookup.daddr,
 825             &lookup.dport, &lookup.proto);
 826           if (err != 0)
 827             {
 828               b0->error = node->errors[err];
 829               next[0] = NAT_NEXT_DROP;
 830               goto trace0;
 831             }
 832         }
 833       else
 834         {
 835           lookup.saddr.as_u32 = ip0->src_address.as_u32;
 836           lookup.daddr.as_u32 = ip0->dst_address.as_u32;
 837           lookup.proto = ip0->protocol;
 838         }
 839
 840       /* there might be a stashed index in vnet_buffer2 from handoff or
 841        * classify node, see if it can be used */
 842       if (is_multi_worker &&
 843           !pool_is_free_index (tsm->sessions,
 844                                vnet_buffer2 (b0)->nat.cached_session_index))
 845         {
 846           s0 = pool_elt_at_index (tsm->sessions,
 847                                   vnet_buffer2 (b0)->nat.cached_session_index);
 848           if (PREDICT_TRUE (nat_6t_t_eq (&s0->o2i.match, &lookup)) ||
 849               (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 850                nat_6t_t_eq (&s0->i2o.match, &lookup)))
 851             {
 852               /* yes, this is the droid we're looking for */
 853               lookup_skipped = 1;
 854               goto skip_lookup;
 855             }
 856           s0 = NULL;
 857         }
 858
 859       init_ed_k (&kv0, lookup.saddr.as_u32, lookup.sport, lookup.daddr.as_u32,
 860                  lookup.dport, lookup.fib_index, lookup.proto);
 861
 862       // lookup flow
 863       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
 864         {
 865           // flow does not exist go slow path
 866           slow_path_reason = NAT_ED_SP_REASON_LOOKUP_FAILED;
 867           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 868           goto trace0;
 869         }
 870       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 871       s0 =
 872         pool_elt_at_index (tsm->sessions,
 873                            ed_value_get_session_index (&value0));
 874     skip_lookup:
 875
 876       ASSERT (thread_index == s0->thread_index);
 877
 878       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 879         {
 880           // session is closed, go slow path
 881           nat44_ed_free_session_data (sm, s0, thread_index, 0);
 882           nat_ed_session_delete (sm, s0, thread_index, 1);
 883           slow_path_reason = NAT_ED_SP_REASON_VRF_EXPIRED;
 884           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 885           goto trace0;
 886         }
 887
 888       // drop if session expired
 889       u64 sess_timeout_time;
 890       sess_timeout_time =
 891         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 892       if (now >= sess_timeout_time)
 893         {
 894           // session is closed, go slow path
 895           nat44_ed_free_session_data (sm, s0, thread_index, 0);
 896           nat_ed_session_delete (sm, s0, thread_index, 1);
 897           slow_path_reason = NAT_ED_SP_SESS_EXPIRED;
 898           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 899           goto trace0;
 900         }
 901
 902       if (nat_6t_t_eq (&s0->o2i.match, &lookup))
 903         {
 904           f = &s0->o2i;
 905         }
 906       else if (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 907                nat_6t_t_eq (&s0->i2o.match, &lookup))
 908         {
 909           f = &s0->i2o;
 910         }
 911       else
 912         {
 913           /*
 914            * Send DHCP packets to the ipv4 stack, or we won't
 915            * be able to use dhcp client on the outside interface
 916            */
 917           if (PREDICT_FALSE (
 918                 proto0 == IP_PROTOCOL_UDP &&
 919                 (vnet_buffer (b0)->ip.reass.l4_dst_port ==
 920                  clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
 921             {
 922               goto trace0;
 923             }
 924
 925           if (!sm->forwarding_enabled)
 926             {
 927               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 928               next[0] = NAT_NEXT_DROP;
 929               goto trace0;
 930             }
 931           else
 932             {
 933               if (nat_6t_t_eq (&s0->i2o.match, &lookup))
 934                 {
 935                   f = &s0->i2o;
 936                 }
 937               else
 938                 {
 939                   // FIXME TODO bypass ???
 940                   //  create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
 941                   //                       thread_index);
 942                   translation_error = NAT_ED_TRNSL_ERR_FLOW_MISMATCH;
 943                   nat44_ed_free_session_data (sm, s0, thread_index, 0);
 944                   nat_ed_session_delete (sm, s0, thread_index, 1);
 945                   next[0] = NAT_NEXT_DROP;
 946                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
 947                   goto trace0;
 948                 }
 949             }
 950         }
 951
 952       if (NAT_ED_TRNSL_ERR_SUCCESS !=
 953           (translation_error = nat_6t_flow_buf_translate_o2i (
 954              vm, sm, b0, ip0, f, proto0, 0 /* is_output_feature */)))
 955         {
 956           next[0] = NAT_NEXT_DROP;
 957           b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
 958           goto trace0;
 959         }
 960
 961       switch (proto0)
 962         {
 963         case IP_PROTOCOL_TCP:
 964           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.tcp,
 965                                          thread_index, sw_if_index0, 1);
 966           nat44_set_tcp_session_state_o2i (sm, now, s0,
 967                                            vnet_buffer (b0)->ip.
 968                                            reass.icmp_type_or_tcp_flags,
 969                                            thread_index);
 970           break;
 971         case IP_PROTOCOL_UDP:
 972           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.udp,
 973                                          thread_index, sw_if_index0, 1);
 974           break;
 975         case IP_PROTOCOL_ICMP:
 976           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.icmp,
 977                                          thread_index, sw_if_index0, 1);
 978           break;
 979         default:
 980           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.other,
 981                                          thread_index, sw_if_index0, 1);
 982           break;
 983         }
 984
 985       /* Accounting */
 986       nat44_session_update_counters (s0, now,
 987                                      vlib_buffer_length_in_chain (vm, b0),
 988                                      thread_index);
 989       /* Per-user LRU list maintenance */
 990       nat44_session_update_lru (sm, s0, thread_index);
 991
 992     trace0:
 993       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 994                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 995         {
 996           nat44_ed_out2in_trace_t *t =
 997             vlib_add_trace (vm, node, b0, sizeof (*t));
 998           t->sw_if_index = sw_if_index0;
 999           t->next_index = next[0];
1000           t->is_slow_path = 0;
1001           t->translation_error = translation_error;
1002           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1003           t->lookup_skipped = lookup_skipped;
1004           t->slow_path_reason = slow_path_reason;
1005
1006           if (s0)
1007             {
1008               t->session_index = s0 - tsm->sessions;
1009               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1010               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1011               t->translation_via_i2of = (&s0->i2o == f);
1012               t->tcp_state = s0->tcp_state;
1013             }
1014           else
1015             {
1016               t->session_index = ~0;
1017             }
1018         }
1019
1020       if (next[0] == NAT_NEXT_DROP)
1021         {
1022           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.drops,
1023                                          thread_index, sw_if_index0, 1);
1024         }
1025
1026       n_left_from--;
1027       next++;
1028     }
1029
1030   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1031                                frame->n_vectors);
1032   return frame->n_vectors;
1033 }
1034
1035 static inline uword
1036 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1037                                           vlib_node_runtime_t * node,
1038                                           vlib_frame_t * frame)
1039 {
1040   u32 n_left_from, *from;
1041   snat_main_t *sm = &snat_main;
1042   f64 now = vlib_time_now (vm);
1043   u32 thread_index = vm->thread_index;
1044   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1045   snat_static_mapping_t *m;
1046
1047   from = vlib_frame_vector_args (frame);
1048   n_left_from = frame->n_vectors;
1049
1050   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1051   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1052   vlib_get_buffers (vm, from, b, n_left_from);
1053
1054   while (n_left_from > 0)
1055     {
1056       vlib_buffer_t *b0;
1057       u32 sw_if_index0, rx_fib_index0;
1058       ip_protocol_t proto0;
1059       ip4_header_t *ip0;
1060       udp_header_t *udp0;
1061       icmp46_header_t *icmp0;
1062       snat_session_t *s0 = 0;
1063       clib_bihash_kv_16_8_t kv0, value0;
1064       lb_nat_type_t lb_nat0;
1065       twice_nat_type_t twice_nat0;
1066       u8 identity_nat0;
1067       ip4_address_t sm_addr;
1068       u16 sm_port;
1069       u32 sm_fib_index;
1070       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
1071
1072       b0 = *b;
1073       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1074
1075       vnet_buffer (b0)->snat.flags = 0;
1076       ip0 = vlib_buffer_get_current (b0);
1077
1078       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1079       rx_fib_index0 =
1080         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1081
1082       if (PREDICT_FALSE (ip0->ttl == 1))
1083         {
1084           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1085           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1086                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1087                                        0);
1088           next[0] = NAT_NEXT_ICMP_ERROR;
1089           goto trace0;
1090         }
1091
1092       udp0 = ip4_next_header (ip0);
1093       icmp0 = (icmp46_header_t *) udp0;
1094       proto0 = ip0->protocol;
1095
1096       if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto0)))
1097         {
1098           s0 = nat44_ed_out2in_slowpath_unknown_proto (
1099             sm, b0, ip0, rx_fib_index0, thread_index, now, vm, node);
1100           if (!sm->forwarding_enabled)
1101             {
1102               if (!s0)
1103                 next[0] = NAT_NEXT_DROP;
1104             }
1105           if (NAT_NEXT_DROP != next[0] && s0 &&
1106               NAT_ED_TRNSL_ERR_SUCCESS !=
1107                 (translation_error = nat_6t_flow_buf_translate_o2i (
1108                    vm, sm, b0, ip0, &s0->o2i, proto0,
1109                    0 /* is_output_feature */)))
1110             {
1111               next[0] = NAT_NEXT_DROP;
1112               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1113               goto trace0;
1114             }
1115
1116           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.other,
1117                                          thread_index, sw_if_index0, 1);
1118           goto trace0;
1119         }
1120
1121       if (PREDICT_FALSE (proto0 == IP_PROTOCOL_ICMP))
1122         {
1123           next[0] = icmp_out2in_ed_slow_path
1124             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1125              next[0], now, thread_index, &s0);
1126
1127           if (NAT_NEXT_DROP != next[0] && s0 &&
1128               NAT_ED_TRNSL_ERR_SUCCESS !=
1129                 (translation_error = nat_6t_flow_buf_translate_o2i (
1130                    vm, sm, b0, ip0, &s0->o2i, proto0,
1131                    0 /* is_output_feature */)))
1132             {
1133               next[0] = NAT_NEXT_DROP;
1134               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1135               goto trace0;
1136             }
1137
1138           if (NAT_NEXT_DROP != next[0])
1139             {
1140               vlib_increment_simple_counter (
1141                 &sm->counters.slowpath.out2in.icmp, thread_index, sw_if_index0,
1142                 1);
1143             }
1144           goto trace0;
1145         }
1146
1147       init_ed_k (
1148         &kv0, ip0->src_address.as_u32, vnet_buffer (b0)->ip.reass.l4_src_port,
1149         ip0->dst_address.as_u32, vnet_buffer (b0)->ip.reass.l4_dst_port,
1150         rx_fib_index0, ip0->protocol);
1151
1152       s0 = NULL;
1153       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
1154         {
1155           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1156           s0 =
1157             pool_elt_at_index (tsm->sessions,
1158                                ed_value_get_session_index (&value0));
1159         }
1160
1161       if (!s0)
1162         {
1163           /* Try to match static mapping by external address and port,
1164              destination address and port in packet */
1165
1166           if (snat_static_mapping_match (
1167                 vm, ip0->dst_address, vnet_buffer (b0)->ip.reass.l4_dst_port,
1168                 rx_fib_index0, proto0, &sm_addr, &sm_port, &sm_fib_index, 1, 0,
1169                 &twice_nat0, &lb_nat0, &ip0->src_address, &identity_nat0, &m))
1170             {
1171               /*
1172                * Send DHCP packets to the ipv4 stack, or we won't
1173                * be able to use dhcp client on the outside interface
1174                */
1175               if (PREDICT_FALSE (
1176                     proto0 == IP_PROTOCOL_UDP &&
1177                     (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1178                      clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
1179                 {
1180                   goto trace0;
1181                 }
1182
1183               if (!sm->forwarding_enabled)
1184                 {
1185                   b0->error =
1186                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1187                   next[0] = NAT_NEXT_DROP;
1188                 }
1189               else
1190                 {
1191                   if (next_src_nat (
1192                         sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1193                         vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0))
1194                     {
1195                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1196                     }
1197                   else
1198                     {
1199                       create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
1200                                              thread_index);
1201                     }
1202                 }
1203               goto trace0;
1204             }
1205
1206           if (PREDICT_FALSE (identity_nat0))
1207             goto trace0;
1208
1209           if ((proto0 == IP_PROTOCOL_TCP) &&
1210               !tcp_flags_is_init (
1211                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
1212             {
1213               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1214               next[0] = NAT_NEXT_DROP;
1215               goto trace0;
1216             }
1217
1218           /* Create session initiated by host from external network */
1219           s0 = create_session_for_static_mapping_ed (
1220             sm, b0, sm_addr, sm_port, sm_fib_index, ip0->dst_address,
1221             vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0, proto0,
1222             node, thread_index, twice_nat0, lb_nat0, now, m);
1223           if (!s0)
1224             {
1225               next[0] = NAT_NEXT_DROP;
1226               goto trace0;
1227             }
1228         }
1229
1230       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1231           (translation_error = nat_6t_flow_buf_translate_o2i (
1232              vm, sm, b0, ip0, &s0->o2i, proto0, 0 /* is_output_feature */)))
1233         {
1234           next[0] = NAT_NEXT_DROP;
1235           goto trace0;
1236         }
1237
1238       if (PREDICT_TRUE (proto0 == IP_PROTOCOL_TCP))
1239         {
1240           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.tcp,
1241                                          thread_index, sw_if_index0, 1);
1242           nat44_set_tcp_session_state_o2i (sm, now, s0,
1243                                            vnet_buffer (b0)->ip.
1244                                            reass.icmp_type_or_tcp_flags,
1245                                            thread_index);
1246         }
1247       else
1248         {
1249           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.udp,
1250                                          thread_index, sw_if_index0, 1);
1251         }
1252
1253       /* Accounting */
1254       nat44_session_update_counters (s0, now,
1255                                      vlib_buffer_length_in_chain (vm, b0),
1256                                      thread_index);
1257       /* Per-user LRU list maintenance */
1258       nat44_session_update_lru (sm, s0, thread_index);
1259
1260     trace0:
1261       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1262                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1263         {
1264           nat44_ed_out2in_trace_t *t =
1265             vlib_add_trace (vm, node, b0, sizeof (*t));
1266           t->sw_if_index = sw_if_index0;
1267           t->next_index = next[0];
1268           t->is_slow_path = 1;
1269           t->translation_error = translation_error;
1270           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1271
1272           if (s0)
1273             {
1274               t->session_index = s0 - tsm->sessions;
1275               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1276               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1277               t->tcp_state = s0->tcp_state;
1278             }
1279           else
1280             {
1281               t->session_index = ~0;
1282             }
1283         }
1284
1285       if (next[0] == NAT_NEXT_DROP)
1286         {
1287           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.drops,
1288                                          thread_index, sw_if_index0, 1);
1289         }
1290
1291       n_left_from--;
1292       next++;
1293       b++;
1294     }
1295
1296   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1297                                frame->n_vectors);
1298
1299   return frame->n_vectors;
1300 }
1301
1302 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1303                                      vlib_node_runtime_t * node,
1304                                      vlib_frame_t * frame)
1305 {
1306   if (snat_main.num_workers > 1)
1307     {
1308       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1309     }
1310   else
1311     {
1312       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1313     }
1314 }
1315
1316 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1317   .name = "nat44-ed-out2in",
1318   .vector_size = sizeof (u32),
1319   .sibling_of = "nat-default",
1320   .format_trace = format_nat44_ed_out2in_trace,
1321   .type = VLIB_NODE_TYPE_INTERNAL,
1322   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1323   .error_strings = nat_out2in_ed_error_strings,
1324   .runtime_data_bytes = sizeof (snat_runtime_t),
1325 };
1326
1327 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1328                                               vlib_node_runtime_t * node,
1329                                               vlib_frame_t * frame)
1330 {
1331   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1332 }
1333
1334 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1335   .name = "nat44-ed-out2in-slowpath",
1336   .vector_size = sizeof (u32),
1337   .sibling_of = "nat-default",
1338   .format_trace = format_nat44_ed_out2in_trace,
1339   .type = VLIB_NODE_TYPE_INTERNAL,
1340   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1341   .error_strings = nat_out2in_ed_error_strings,
1342   .runtime_data_bytes = sizeof (snat_runtime_t),
1343 };
1344
1345 static u8 *
1346 format_nat_pre_trace (u8 * s, va_list * args)
1347 {
1348   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1349   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1350   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1351   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1352                  t->arc_next_index);
1353 }
1354
1355 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1356                                     vlib_node_runtime_t * node,
1357                                     vlib_frame_t * frame)
1358 {
1359   return nat_pre_node_fn_inline (vm, node, frame,
1360                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1361 }
1362
1363 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1364   .name = "nat-pre-out2in",
1365   .vector_size = sizeof (u32),
1366   .sibling_of = "nat-default",
1367   .format_trace = format_nat_pre_trace,
1368   .type = VLIB_NODE_TYPE_INTERNAL,
1369   .n_errors = 0,
1370  };
1371
1372 /*
1373  * fd.io coding-style-patch-verification: ON
1374  *
1375  * Local Variables:
1376  * eval: (c-set-style "gnu")
1377  * End:
1378  */