src/plugins/nat/nat44-ed/nat44_ed_out2in.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vppinfra/error.h>
  27
  28 #include <nat/lib/ipfix_logging.h>
  29
  30 #include <nat/nat44-ed/nat44_ed.h>
  31 #include <nat/nat44-ed/nat44_ed_inlines.h>
  32
  33 static char *nat_out2in_ed_error_strings[] = {
  34 #define _(sym,string) string,
  35   foreach_nat_out2in_ed_error
  36 #undef _
  37 };
  38
  39 typedef enum
  40 {
  41   NAT_ED_SP_REASON_NO_REASON,
  42   NAT_ED_SP_REASON_LOOKUP_FAILED,
  43   NAT_ED_SP_REASON_VRF_EXPIRED,
  44   NAT_ED_SP_SESS_EXPIRED,
  45 } nat_slow_path_reason_e;
  46
  47 typedef struct
  48 {
  49   u32 sw_if_index;
  50   u32 next_index;
  51   u32 session_index;
  52   nat_translation_error_e translation_error;
  53   nat_6t_flow_t i2of;
  54   nat_6t_flow_t o2if;
  55   clib_bihash_kv_16_8_t search_key;
  56   u8 is_slow_path;
  57   u8 translation_via_i2of;
  58   u8 lookup_skipped;
  59   u8 tcp_state;
  60   nat_slow_path_reason_e slow_path_reason;
  61 } nat44_ed_out2in_trace_t;
  62
  63 static u8 *
  64 format_slow_path_reason (u8 *s, va_list *args)
  65 {
  66   nat_slow_path_reason_e reason = va_arg (*args, nat_slow_path_reason_e);
  67   switch (reason)
  68     {
  69     case NAT_ED_SP_REASON_NO_REASON:
  70       return format (s, "no reason for slow path");
  71     case NAT_ED_SP_REASON_LOOKUP_FAILED:
  72       return format (s, "slow path because lookup failed");
  73     case NAT_ED_SP_REASON_VRF_EXPIRED:
  74       return format (s, "slow path because vrf expired");
  75     case NAT_ED_SP_SESS_EXPIRED:
  76       return format (s, "slow path because session expired");
  77     }
  78   return format (s, "invalid reason value");
  79 }
  80
  81 static u8 *
  82 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
  83 {
  84   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  85   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  86   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
  87   char *tag;
  88
  89   tag =
  90     t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
  91     "NAT44_OUT2IN_ED_FAST_PATH";
  92
  93   s = format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
  94               t->next_index);
  95   if (~0 != t->session_index)
  96     {
  97       s = format (s, ", session %d, translation result '%U' via %s",
  98                   t->session_index, format_nat_ed_translation_error,
  99                   t->translation_error,
 100                   t->translation_via_i2of ? "i2of" : "o2if");
 101       s = format (s, "\n  i2of %U", format_nat_6t_flow, &t->i2of);
 102       s = format (s, "\n  o2if %U", format_nat_6t_flow, &t->o2if);
 103     }
 104   if (!t->is_slow_path)
 105     {
 106       if (t->lookup_skipped)
 107         {
 108           s = format (s, "\n  lookup skipped - cached session index used");
 109         }
 110       else
 111         {
 112           s = format (s, "\n  search key %U", format_ed_session_kvp,
 113                       &t->search_key);
 114         }
 115       s = format (s, "\n  %U", format_slow_path_reason, t->slow_path_reason);
 116     }
 117   if (IP_PROTOCOL_TCP == t->i2of.match.proto)
 118     {
 119       s = format (s, "\n  TCP state: %U", format_nat44_ed_tcp_state,
 120                   t->tcp_state);
 121     }
 122
 123   return s;
 124 }
 125
 126 static int
 127 next_src_nat (snat_main_t *sm, ip4_header_t *ip, u16 src_port, u16 dst_port,
 128               u32 rx_fib_index)
 129 {
 130   clib_bihash_kv_16_8_t kv, value;
 131
 132   init_ed_k (&kv, ip->src_address.as_u32, src_port, ip->dst_address.as_u32,
 133              dst_port, rx_fib_index, ip->protocol);
 134   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 135     return 1;
 136
 137   return 0;
 138 }
 139
 140 static void create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b,
 141                                    snat_session_t *s, ip4_header_t *ip,
 142                                    u32 rx_fib_index, u32 thread_index);
 143
 144 static snat_session_t *create_session_for_static_mapping_ed (
 145   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 146   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 147   ip_protocol_t proto, vlib_node_runtime_t *node, u32 thread_index,
 148   twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 149   snat_static_mapping_t *mapping);
 150
 151 static inline u32
 152 icmp_out2in_ed_slow_path (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
 153                           icmp46_header_t *icmp, u32 sw_if_index,
 154                           u32 rx_fib_index, vlib_node_runtime_t *node,
 155                           u32 next, f64 now, u32 thread_index,
 156                           snat_session_t **s_p)
 157 {
 158   vlib_main_t *vm = vlib_get_main ();
 159
 160   ip_csum_t sum;
 161   u16 checksum;
 162
 163   snat_session_t *s = 0;
 164   u8 is_addr_only, identity_nat;
 165   ip4_address_t sm_addr;
 166   u16 sm_port;
 167   u32 sm_fib_index;
 168   snat_static_mapping_t *m;
 169   u8 lookup_protocol;
 170   ip4_address_t lookup_saddr, lookup_daddr;
 171   u16 lookup_sport, lookup_dport;
 172
 173   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 174   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 175
 176   if (nat_get_icmp_session_lookup_values (b, ip, &lookup_saddr, &lookup_sport,
 177                                           &lookup_daddr, &lookup_dport,
 178                                           &lookup_protocol))
 179     {
 180       b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
 181       next = NAT_NEXT_DROP;
 182       goto out;
 183     }
 184
 185   if (snat_static_mapping_match (vm, ip->dst_address, lookup_sport,
 186                                  rx_fib_index, ip->protocol, &sm_addr,
 187                                  &sm_port, &sm_fib_index, 1, &is_addr_only, 0,
 188                                  0, 0, &identity_nat, &m))
 189     {
 190       // static mapping not matched
 191       if (!sm->forwarding_enabled)
 192         {
 193           /* Don't NAT packet aimed at the intfc address */
 194           if (!is_interface_addr (sm, node, sw_if_index,
 195                                   ip->dst_address.as_u32))
 196             {
 197               b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 198               next = NAT_NEXT_DROP;
 199             }
 200         }
 201       else
 202         {
 203           if (next_src_nat (sm, ip, lookup_sport, lookup_dport, rx_fib_index))
 204             {
 205               next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
 206             }
 207           else
 208             {
 209               create_bypass_for_fwd (sm, b, s, ip, rx_fib_index, thread_index);
 210             }
 211         }
 212       goto out;
 213     }
 214
 215   if (PREDICT_FALSE (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 216                        ICMP4_echo_reply &&
 217                      (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 218                         ICMP4_echo_request ||
 219                       !is_addr_only)))
 220     {
 221       b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 222       next = NAT_NEXT_DROP;
 223       goto out;
 224     }
 225
 226   if (PREDICT_FALSE (identity_nat))
 227     {
 228       goto out;
 229     }
 230
 231   /* Create session initiated by host from external network */
 232   s = create_session_for_static_mapping_ed (
 233     sm, b, sm_addr, sm_port, sm_fib_index, ip->dst_address, lookup_sport,
 234     rx_fib_index, lookup_protocol, node, thread_index, 0, 0,
 235     vlib_time_now (vm), m);
 236   if (!s)
 237     next = NAT_NEXT_DROP;
 238
 239   if (PREDICT_TRUE (!ip4_is_fragment (ip)))
 240     {
 241       sum = ip_incremental_checksum_buffer (
 242         vm, b, (u8 *) icmp - (u8 *) vlib_buffer_get_current (b),
 243         ntohs (ip->length) - ip4_header_bytes (ip), 0);
 244       checksum = ~ip_csum_fold (sum);
 245       if (checksum != 0 && checksum != 0xffff)
 246         {
 247           next = NAT_NEXT_DROP;
 248           goto out;
 249         }
 250     }
 251
 252   if (PREDICT_TRUE (next != NAT_NEXT_DROP && s))
 253     {
 254       /* Accounting */
 255       nat44_session_update_counters (
 256         s, now, vlib_buffer_length_in_chain (vm, b), thread_index);
 257       /* Per-user LRU list maintenance */
 258       nat44_session_update_lru (sm, s, thread_index);
 259     }
 260 out:
 261   if (NAT_NEXT_DROP == next && s)
 262     {
 263       nat_ed_session_delete (sm, s, thread_index, 1);
 264       s = 0;
 265     }
 266   *s_p = s;
 267   return next;
 268 }
 269
 270 static_always_inline int
 271 nat44_ed_alloc_i2o_port (snat_main_t *sm, snat_address_t *a, snat_session_t *s,
 272                          ip4_address_t i2o_addr, u16 i2o_port,
 273                          u32 i2o_fib_index, ip_protocol_t proto,
 274                          u32 thread_index, u32 snat_thread_index,
 275                          ip4_address_t *outside_addr, u16 *outside_port)
 276 {
 277   u32 portnum;
 278
 279   for (int i = 0; i < ED_PORT_ALLOC_ATTEMPTS; ++i)
 280     {
 281       portnum = (sm->port_per_thread * snat_thread_index) +
 282                 snat_random_port (0, sm->port_per_thread - 1) +
 283                 ED_USER_PORT_OFFSET;
 284       portnum = clib_host_to_net_u16 (portnum);
 285       nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port, a->addr,
 286                             portnum, i2o_fib_index, proto);
 287       if (!nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s,
 288                                              1 /* is_add */))
 289         {
 290           *outside_addr = a->addr;
 291           *outside_port = portnum;
 292           return 0;
 293         }
 294     }
 295
 296   /* Totally out of translations to use... */
 297   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 298   return 1;
 299 }
 300
 301 static_always_inline int
 302 nat44_ed_alloc_i2o_addr_and_port (snat_main_t *sm, snat_address_t *addresses,
 303                                   snat_session_t *s, ip4_address_t i2o_addr,
 304                                   u16 i2o_port, u32 i2o_fib_index,
 305                                   ip_protocol_t proto, u32 thread_index,
 306                                   u32 snat_thread_index,
 307                                   ip4_address_t *outside_addr,
 308                                   u16 *outside_port)
 309 {
 310   snat_address_t *a, *ga = 0;
 311   int i;
 312
 313   if (vec_len (addresses) > 0)
 314     {
 315       int s_addr_offset = i2o_addr.as_u32 % vec_len (addresses);
 316
 317       for (i = s_addr_offset; i < vec_len (addresses); ++i)
 318         {
 319           a = addresses + i;
 320           if (a->fib_index == i2o_fib_index)
 321             {
 322               return nat44_ed_alloc_i2o_port (
 323                 sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 324                 thread_index, snat_thread_index, outside_addr, outside_port);
 325             }
 326           else if (a->fib_index == ~0)
 327             {
 328               ga = a;
 329             }
 330         }
 331
 332       for (i = 0; i < s_addr_offset; ++i)
 333         {
 334           a = addresses + i;
 335           if (a->fib_index == i2o_fib_index)
 336             {
 337               return nat44_ed_alloc_i2o_port (
 338                 sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 339                 thread_index, snat_thread_index, outside_addr, outside_port);
 340             }
 341           else if (a->fib_index == ~0)
 342             {
 343               ga = a;
 344             }
 345         }
 346
 347       if (ga)
 348         {
 349           return nat44_ed_alloc_i2o_port (
 350             sm, a, s, i2o_addr, i2o_port, i2o_fib_index, proto, thread_index,
 351             snat_thread_index, outside_addr, outside_port);
 352         }
 353     }
 354
 355   /* Totally out of translations to use... */
 356   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
 357   return 1;
 358 }
 359
 360 static snat_session_t *
 361 create_session_for_static_mapping_ed (
 362   snat_main_t *sm, vlib_buffer_t *b, ip4_address_t i2o_addr, u16 i2o_port,
 363   u32 i2o_fib_index, ip4_address_t o2i_addr, u16 o2i_port, u32 o2i_fib_index,
 364   ip_protocol_t proto, vlib_node_runtime_t *node, u32 thread_index,
 365   twice_nat_type_t twice_nat, lb_nat_type_t lb_nat, f64 now,
 366   snat_static_mapping_t *mapping)
 367 {
 368   snat_session_t *s;
 369   ip4_header_t *ip;
 370   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 371
 372   if (PREDICT_FALSE (
 373         nat44_ed_maximum_sessions_exceeded (sm, o2i_fib_index, thread_index)))
 374     {
 375       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 376       nat_elog_notice (sm, "maximum sessions exceeded");
 377       return 0;
 378     }
 379
 380   s = nat_ed_session_alloc (sm, thread_index, now, proto);
 381   if (!s)
 382     {
 383       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 384       nat_elog_warn (sm, "create NAT session failed");
 385       return 0;
 386     }
 387
 388   ip = vlib_buffer_get_current (b);
 389
 390   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 391   s->ext_host_port =
 392     proto == IP_PROTOCOL_ICMP ? 0 : vnet_buffer (b)->ip.reass.l4_src_port;
 393   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 394   if (lb_nat)
 395     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 396   if (lb_nat == AFFINITY_LB_NAT)
 397     s->flags |= SNAT_SESSION_FLAG_AFFINITY;
 398   s->out2in.addr = o2i_addr;
 399   s->out2in.port = o2i_port;
 400   s->out2in.fib_index = o2i_fib_index;
 401   s->in2out.addr = i2o_addr;
 402   s->in2out.port = i2o_port;
 403   s->in2out.fib_index = i2o_fib_index;
 404   s->proto = proto;
 405
 406   if (IP_PROTOCOL_ICMP == proto)
 407     {
 408       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr, o2i_port,
 409                             o2i_addr, o2i_port, o2i_fib_index, ip->protocol);
 410       nat_6t_flow_icmp_id_rewrite_set (&s->o2i, i2o_port);
 411     }
 412   else
 413     {
 414       nat_6t_o2i_flow_init (sm, thread_index, s, s->ext_host_addr,
 415                             s->ext_host_port, o2i_addr, o2i_port,
 416                             o2i_fib_index, ip->protocol);
 417       nat_6t_flow_dport_rewrite_set (&s->o2i, i2o_port);
 418     }
 419   nat_6t_flow_daddr_rewrite_set (&s->o2i, i2o_addr.as_u32);
 420   nat_6t_flow_txfib_rewrite_set (&s->o2i, i2o_fib_index);
 421
 422   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 423     {
 424       b->error = node->errors[NAT_OUT2IN_ED_ERROR_HASH_ADD_FAILED];
 425       nat_ed_session_delete (sm, s, thread_index, 1);
 426       nat_elog_warn (sm, "out2in flow hash add failed");
 427       return 0;
 428     }
 429
 430   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
 431                                  ip->src_address.as_u32 == i2o_addr.as_u32))
 432     {
 433       int rc = 0;
 434       snat_address_t *filter = 0;
 435
 436       // if exact address is specified use this address
 437       if (is_sm_exact_address (mapping->flags))
 438         {
 439           snat_address_t *ap;
 440           vec_foreach (ap, sm->twice_nat_addresses)
 441           {
 442             if (mapping->pool_addr.as_u32 == ap->addr.as_u32)
 443               {
 444                 filter = ap;
 445                 break;
 446               }
 447           }
 448         }
 449
 450       if (filter)
 451         {
 452           rc = nat44_ed_alloc_i2o_port (
 453             sm, filter, s, i2o_addr, i2o_port, i2o_fib_index, proto,
 454             thread_index, tsm->snat_thread_index, &s->ext_host_nat_addr,
 455             &s->ext_host_nat_port);
 456           s->flags |= SNAT_SESSION_FLAG_EXACT_ADDRESS;
 457         }
 458       else
 459         {
 460           rc = nat44_ed_alloc_i2o_addr_and_port (
 461             sm, sm->twice_nat_addresses, s, i2o_addr, i2o_port, i2o_fib_index,
 462             proto, thread_index, tsm->snat_thread_index, &s->ext_host_nat_addr,
 463             &s->ext_host_nat_port);
 464         }
 465
 466       if (rc)
 467         {
 468           b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
 469           nat_ed_session_delete (sm, s, thread_index, 1);
 470           return 0;
 471         }
 472
 473       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 474
 475       nat_6t_flow_saddr_rewrite_set (&s->o2i, s->ext_host_nat_addr.as_u32);
 476       if (IP_PROTOCOL_ICMP == proto)
 477         {
 478           nat_6t_flow_icmp_id_rewrite_set (&s->o2i, s->ext_host_nat_port);
 479         }
 480       else
 481         {
 482           nat_6t_flow_sport_rewrite_set (&s->o2i, s->ext_host_nat_port);
 483         }
 484
 485       nat_6t_l3_l4_csum_calc (&s->o2i);
 486
 487       nat_6t_flow_daddr_rewrite_set (&s->i2o, s->ext_host_addr.as_u32);
 488       if (IP_PROTOCOL_ICMP == proto)
 489         {
 490           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, s->ext_host_port);
 491         }
 492       else
 493         {
 494           nat_6t_flow_dport_rewrite_set (&s->i2o, s->ext_host_port);
 495         }
 496
 497       nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 498       if (IP_PROTOCOL_ICMP == proto)
 499         {
 500           nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 501         }
 502       else
 503         {
 504           nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 505         }
 506       nat_6t_l3_l4_csum_calc (&s->i2o);
 507     }
 508   else
 509     {
 510       if (IP_PROTOCOL_ICMP == proto)
 511         {
 512           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 513                                 s->ext_host_addr, i2o_port, i2o_fib_index,
 514                                 ip->protocol);
 515         }
 516       else
 517         {
 518           nat_6t_i2o_flow_init (sm, thread_index, s, i2o_addr, i2o_port,
 519                                 s->ext_host_addr, s->ext_host_port,
 520                                 i2o_fib_index, ip->protocol);
 521         }
 522
 523   nat_6t_flow_saddr_rewrite_set (&s->i2o, o2i_addr.as_u32);
 524   if (IP_PROTOCOL_ICMP == proto)
 525     {
 526       nat_6t_flow_icmp_id_rewrite_set (&s->i2o, o2i_port);
 527     }
 528   else
 529     {
 530       nat_6t_flow_sport_rewrite_set (&s->i2o, o2i_port);
 531     }
 532
 533   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 534     {
 535       nat_elog_notice (sm, "in2out flow hash add failed");
 536       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 537         {
 538           nat_elog_warn (sm, "out2in flow hash del failed");
 539         }
 540       nat_ed_session_delete (sm, s, thread_index, 1);
 541       return 0;
 542     }
 543     }
 544   nat_ipfix_logging_nat44_ses_create (
 545     thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 546     s->in2out.port, s->out2in.port, s->in2out.fib_index);
 547
 548   nat_syslog_nat44_sadd (0, s->in2out.fib_index, &s->in2out.addr,
 549                          s->in2out.port, &s->ext_host_nat_addr,
 550                          s->ext_host_nat_port, &s->out2in.addr, s->out2in.port,
 551                          &s->ext_host_addr, s->ext_host_port, s->proto,
 552                          nat44_ed_is_twice_nat_session (s));
 553
 554   per_vrf_sessions_register_session (s, thread_index);
 555
 556   return s;
 557 }
 558
 559 static void
 560 create_bypass_for_fwd (snat_main_t *sm, vlib_buffer_t *b, snat_session_t *s,
 561                        ip4_header_t *ip, u32 rx_fib_index, u32 thread_index)
 562 {
 563   clib_bihash_kv_16_8_t kv, value;
 564   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 565   vlib_main_t *vm = vlib_get_main ();
 566   f64 now = vlib_time_now (vm);
 567   u16 lookup_sport, lookup_dport;
 568   u8 lookup_protocol;
 569   ip4_address_t lookup_saddr, lookup_daddr;
 570
 571   if (ip->protocol == IP_PROTOCOL_ICMP)
 572     {
 573       if (nat_get_icmp_session_lookup_values (b, ip, &lookup_daddr,
 574                                               &lookup_sport, &lookup_saddr,
 575                                               &lookup_dport, &lookup_protocol))
 576         return;
 577     }
 578   else
 579     {
 580       if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 581         {
 582           lookup_sport = vnet_buffer (b)->ip.reass.l4_dst_port;
 583           lookup_dport = vnet_buffer (b)->ip.reass.l4_src_port;
 584         }
 585       else
 586         {
 587           lookup_sport = 0;
 588           lookup_dport = 0;
 589         }
 590       lookup_saddr.as_u32 = ip->dst_address.as_u32;
 591       lookup_daddr.as_u32 = ip->src_address.as_u32;
 592       lookup_protocol = ip->protocol;
 593     }
 594
 595   init_ed_k (&kv, lookup_saddr.as_u32, lookup_sport, lookup_daddr.as_u32,
 596              lookup_dport, rx_fib_index, lookup_protocol);
 597
 598   if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
 599     {
 600       ASSERT (thread_index == ed_value_get_thread_index (&value));
 601       s =
 602         pool_elt_at_index (tsm->sessions,
 603                            ed_value_get_session_index (&value));
 604     }
 605   else if (ip->protocol == IP_PROTOCOL_ICMP &&
 606            icmp_type_is_error_message
 607            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 608     {
 609       return;
 610     }
 611   else
 612     {
 613       if (PREDICT_FALSE
 614           (nat44_ed_maximum_sessions_exceeded
 615            (sm, rx_fib_index, thread_index)))
 616         return;
 617
 618       s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 619       if (!s)
 620         {
 621           nat_elog_warn (sm, "create NAT session failed");
 622           return;
 623         }
 624
 625       s->ext_host_addr = ip->src_address;
 626       s->ext_host_port = lookup_dport;
 627       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
 628       s->out2in.addr = ip->dst_address;
 629       s->out2in.port = lookup_sport;
 630       s->proto = ip->protocol;
 631       s->out2in.fib_index = rx_fib_index;
 632       s->in2out.addr = s->out2in.addr;
 633       s->in2out.port = s->out2in.port;
 634       s->in2out.fib_index = s->out2in.fib_index;
 635
 636       nat_6t_i2o_flow_init (sm, thread_index, s, ip->dst_address, lookup_sport,
 637                             ip->src_address, lookup_dport, rx_fib_index,
 638                             ip->protocol);
 639       nat_6t_flow_txfib_rewrite_set (&s->i2o, rx_fib_index);
 640       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 641         {
 642           nat_elog_notice (sm, "in2out flow add failed");
 643           nat_ed_session_delete (sm, s, thread_index, 1);
 644           return;
 645         }
 646
 647       per_vrf_sessions_register_session (s, thread_index);
 648     }
 649
 650   if (ip->protocol == IP_PROTOCOL_TCP)
 651     {
 652       nat44_set_tcp_session_state_o2i (
 653         sm, now, s, vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags,
 654         thread_index);
 655     }
 656
 657   /* Accounting */
 658   nat44_session_update_counters (s, now, 0, thread_index);
 659   /* Per-user LRU list maintenance */
 660   nat44_session_update_lru (sm, s, thread_index);
 661 }
 662
 663 static snat_session_t *
 664 nat44_ed_out2in_slowpath_unknown_proto (snat_main_t *sm, vlib_buffer_t *b,
 665                                         ip4_header_t *ip, u32 rx_fib_index,
 666                                         u32 thread_index, f64 now,
 667                                         vlib_main_t *vm,
 668                                         vlib_node_runtime_t *node)
 669 {
 670   snat_static_mapping_t *m;
 671   snat_session_t *s;
 672
 673   if (PREDICT_FALSE (
 674         nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
 675     {
 676       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 677       nat_elog_notice (sm, "maximum sessions exceeded");
 678       return 0;
 679     }
 680
 681   m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, ip->protocol);
 682   if (!m)
 683     {
 684       b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 685       return 0;
 686     }
 687
 688   /* Create a new session */
 689   s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
 690   if (!s)
 691     {
 692       b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 693       nat_elog_warn (sm, "create NAT session failed");
 694       return 0;
 695     }
 696
 697   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 698   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 699   s->out2in.addr.as_u32 = ip->dst_address.as_u32;
 700   s->out2in.fib_index = rx_fib_index;
 701   s->in2out.addr.as_u32 = m->local_addr.as_u32;
 702   s->in2out.fib_index = m->fib_index;
 703   s->in2out.port = s->out2in.port = ip->protocol;
 704
 705   nat_6t_o2i_flow_init (sm, thread_index, s, ip->dst_address, 0,
 706                         ip->src_address, 0, m->fib_index, ip->protocol);
 707   nat_6t_flow_saddr_rewrite_set (&s->i2o, ip->dst_address.as_u32);
 708   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 1))
 709     {
 710       nat_elog_notice (sm, "in2out key add failed");
 711       nat_ed_session_delete (sm, s, thread_index, 1);
 712       return NULL;
 713     }
 714
 715   nat_6t_o2i_flow_init (sm, thread_index, s, ip->src_address, 0,
 716                         ip->dst_address, 0, rx_fib_index, ip->protocol);
 717   nat_6t_flow_daddr_rewrite_set (&s->o2i, m->local_addr.as_u32);
 718   nat_6t_flow_txfib_rewrite_set (&s->o2i, m->fib_index);
 719   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 1))
 720     {
 721       nat_elog_notice (sm, "out2in flow hash add failed");
 722       nat_ed_session_delete (sm, s, thread_index, 1);
 723       return NULL;
 724     }
 725
 726   per_vrf_sessions_register_session (s, thread_index);
 727
 728   /* Accounting */
 729   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 730                                  thread_index);
 731   /* Per-user LRU list maintenance */
 732   nat44_session_update_lru (sm, s, thread_index);
 733
 734   return s;
 735 }
 736
 737 static inline uword
 738 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 739                                           vlib_node_runtime_t * node,
 740                                           vlib_frame_t * frame,
 741                                           int is_multi_worker)
 742 {
 743   u32 n_left_from, *from;
 744   snat_main_t *sm = &snat_main;
 745   f64 now = vlib_time_now (vm);
 746   u32 thread_index = vm->thread_index;
 747   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 748
 749   from = vlib_frame_vector_args (frame);
 750   n_left_from = frame->n_vectors;
 751
 752   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 753   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 754   vlib_get_buffers (vm, from, b, n_left_from);
 755
 756   while (n_left_from > 0)
 757     {
 758       vlib_buffer_t *b0;
 759       u32 sw_if_index0, rx_fib_index0;
 760       ip_protocol_t proto0;
 761       ip4_header_t *ip0;
 762       snat_session_t *s0 = 0;
 763       clib_bihash_kv_16_8_t kv0 = {}, value0;
 764       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
 765       nat_slow_path_reason_e slow_path_reason = NAT_ED_SP_REASON_NO_REASON;
 766       nat_6t_flow_t *f = 0;
 767       nat_6t_t lookup;
 768       int lookup_skipped = 0;
 769
 770       b0 = *b;
 771       b++;
 772
 773       /* Prefetch next iteration. */
 774       if (PREDICT_TRUE (n_left_from >= 2))
 775         {
 776           vlib_buffer_t *p2;
 777
 778           p2 = *b;
 779
 780           vlib_prefetch_buffer_header (p2, LOAD);
 781
 782           clib_prefetch_load (p2->data);
 783         }
 784
 785       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 786
 787       lookup.sport = vnet_buffer (b0)->ip.reass.l4_src_port;
 788       lookup.dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
 789
 790       vnet_buffer (b0)->snat.flags = 0;
 791       ip0 = vlib_buffer_get_current (b0);
 792
 793       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 794       rx_fib_index0 =
 795         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 796
 797       lookup.fib_index = rx_fib_index0;
 798
 799       if (PREDICT_FALSE (ip0->ttl == 1))
 800         {
 801           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 802           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 803                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 804                                        0);
 805           next[0] = NAT_NEXT_ICMP_ERROR;
 806           goto trace0;
 807         }
 808
 809       proto0 = ip0->protocol;
 810
 811       if (PREDICT_FALSE (proto0 == IP_PROTOCOL_ICMP))
 812         {
 813           if (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 814                 ICMP4_echo_request &&
 815               vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 816                 ICMP4_echo_reply &&
 817               !icmp_type_is_error_message (
 818                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 819             {
 820               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
 821               next[0] = NAT_NEXT_DROP;
 822               goto trace0;
 823             }
 824           int err = nat_get_icmp_session_lookup_values (
 825             b0, ip0, &lookup.saddr, &lookup.sport, &lookup.daddr,
 826             &lookup.dport, &lookup.proto);
 827           if (err != 0)
 828             {
 829               b0->error = node->errors[err];
 830               next[0] = NAT_NEXT_DROP;
 831               goto trace0;
 832             }
 833         }
 834       else
 835         {
 836           lookup.saddr.as_u32 = ip0->src_address.as_u32;
 837           lookup.daddr.as_u32 = ip0->dst_address.as_u32;
 838           lookup.proto = ip0->protocol;
 839         }
 840
 841       /* there might be a stashed index in vnet_buffer2 from handoff or
 842        * classify node, see if it can be used */
 843       if (is_multi_worker &&
 844           !pool_is_free_index (tsm->sessions,
 845                                vnet_buffer2 (b0)->nat.cached_session_index))
 846         {
 847           s0 = pool_elt_at_index (tsm->sessions,
 848                                   vnet_buffer2 (b0)->nat.cached_session_index);
 849           if (PREDICT_TRUE (nat_6t_t_eq (&s0->o2i.match, &lookup)) ||
 850               (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 851                nat_6t_t_eq (&s0->i2o.match, &lookup)))
 852             {
 853               /* yes, this is the droid we're looking for */
 854               lookup_skipped = 1;
 855               goto skip_lookup;
 856             }
 857           s0 = NULL;
 858         }
 859
 860       init_ed_k (&kv0, lookup.saddr.as_u32, lookup.sport, lookup.daddr.as_u32,
 861                  lookup.dport, lookup.fib_index, lookup.proto);
 862
 863       // lookup flow
 864       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
 865         {
 866           // flow does not exist go slow path
 867           slow_path_reason = NAT_ED_SP_REASON_LOOKUP_FAILED;
 868           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 869           goto trace0;
 870         }
 871       ASSERT (thread_index == ed_value_get_thread_index (&value0));
 872       s0 =
 873         pool_elt_at_index (tsm->sessions,
 874                            ed_value_get_session_index (&value0));
 875     skip_lookup:
 876
 877       ASSERT (thread_index == s0->thread_index);
 878
 879       if (PREDICT_FALSE (per_vrf_sessions_is_expired (s0, thread_index)))
 880         {
 881           // session is closed, go slow path
 882           nat44_ed_free_session_data (sm, s0, thread_index, 0);
 883           nat_ed_session_delete (sm, s0, thread_index, 1);
 884           slow_path_reason = NAT_ED_SP_REASON_VRF_EXPIRED;
 885           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 886           goto trace0;
 887         }
 888
 889       // drop if session expired
 890       u64 sess_timeout_time;
 891       sess_timeout_time =
 892         s0->last_heard + (f64) nat44_session_get_timeout (sm, s0);
 893       if (now >= sess_timeout_time)
 894         {
 895           // session is closed, go slow path
 896           nat44_ed_free_session_data (sm, s0, thread_index, 0);
 897           nat_ed_session_delete (sm, s0, thread_index, 1);
 898           slow_path_reason = NAT_ED_SP_SESS_EXPIRED;
 899           next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 900           goto trace0;
 901         }
 902
 903       if (nat_6t_t_eq (&s0->o2i.match, &lookup))
 904         {
 905           f = &s0->o2i;
 906         }
 907       else if (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
 908                nat_6t_t_eq (&s0->i2o.match, &lookup))
 909         {
 910           f = &s0->i2o;
 911         }
 912       else
 913         {
 914           /*
 915            * Send DHCP packets to the ipv4 stack, or we won't
 916            * be able to use dhcp client on the outside interface
 917            */
 918           if (PREDICT_FALSE (
 919                 proto0 == IP_PROTOCOL_UDP &&
 920                 (vnet_buffer (b0)->ip.reass.l4_dst_port ==
 921                  clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
 922             {
 923               goto trace0;
 924             }
 925
 926           if (!sm->forwarding_enabled)
 927             {
 928               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
 929               next[0] = NAT_NEXT_DROP;
 930               goto trace0;
 931             }
 932           else
 933             {
 934               if (nat_6t_t_eq (&s0->i2o.match, &lookup))
 935                 {
 936                   f = &s0->i2o;
 937                 }
 938               else
 939                 {
 940                   // FIXME TODO bypass ???
 941                   //  create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
 942                   //                       thread_index);
 943                   translation_error = NAT_ED_TRNSL_ERR_FLOW_MISMATCH;
 944                   nat44_ed_free_session_data (sm, s0, thread_index, 0);
 945                   nat_ed_session_delete (sm, s0, thread_index, 1);
 946                   next[0] = NAT_NEXT_DROP;
 947                   b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
 948                   goto trace0;
 949                 }
 950             }
 951         }
 952
 953       if (NAT_ED_TRNSL_ERR_SUCCESS !=
 954           (translation_error = nat_6t_flow_buf_translate_o2i (
 955              vm, sm, b0, ip0, f, proto0, 0 /* is_output_feature */)))
 956         {
 957           next[0] = NAT_NEXT_DROP;
 958           b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
 959           goto trace0;
 960         }
 961
 962       switch (proto0)
 963         {
 964         case IP_PROTOCOL_TCP:
 965           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.tcp,
 966                                          thread_index, sw_if_index0, 1);
 967           nat44_set_tcp_session_state_o2i (sm, now, s0,
 968                                            vnet_buffer (b0)->ip.
 969                                            reass.icmp_type_or_tcp_flags,
 970                                            thread_index);
 971           break;
 972         case IP_PROTOCOL_UDP:
 973           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.udp,
 974                                          thread_index, sw_if_index0, 1);
 975           break;
 976         case IP_PROTOCOL_ICMP:
 977           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.icmp,
 978                                          thread_index, sw_if_index0, 1);
 979           break;
 980         default:
 981           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.other,
 982                                          thread_index, sw_if_index0, 1);
 983           break;
 984         }
 985
 986       /* Accounting */
 987       nat44_session_update_counters (s0, now,
 988                                      vlib_buffer_length_in_chain (vm, b0),
 989                                      thread_index);
 990       /* Per-user LRU list maintenance */
 991       nat44_session_update_lru (sm, s0, thread_index);
 992
 993     trace0:
 994       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 995                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 996         {
 997           nat44_ed_out2in_trace_t *t =
 998             vlib_add_trace (vm, node, b0, sizeof (*t));
 999           t->sw_if_index = sw_if_index0;
1000           t->next_index = next[0];
1001           t->is_slow_path = 0;
1002           t->translation_error = translation_error;
1003           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1004           t->lookup_skipped = lookup_skipped;
1005           t->slow_path_reason = slow_path_reason;
1006
1007           if (s0)
1008             {
1009               t->session_index = s0 - tsm->sessions;
1010               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1011               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1012               t->translation_via_i2of = (&s0->i2o == f);
1013               t->tcp_state = s0->tcp_state;
1014             }
1015           else
1016             {
1017               t->session_index = ~0;
1018             }
1019         }
1020
1021       if (next[0] == NAT_NEXT_DROP)
1022         {
1023           vlib_increment_simple_counter (&sm->counters.fastpath.out2in.drops,
1024                                          thread_index, sw_if_index0, 1);
1025         }
1026
1027       n_left_from--;
1028       next++;
1029     }
1030
1031   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1032                                frame->n_vectors);
1033   return frame->n_vectors;
1034 }
1035
1036 static inline uword
1037 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
1038                                           vlib_node_runtime_t * node,
1039                                           vlib_frame_t * frame)
1040 {
1041   u32 n_left_from, *from;
1042   snat_main_t *sm = &snat_main;
1043   f64 now = vlib_time_now (vm);
1044   u32 thread_index = vm->thread_index;
1045   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1046   snat_static_mapping_t *m;
1047
1048   from = vlib_frame_vector_args (frame);
1049   n_left_from = frame->n_vectors;
1050
1051   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1052   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1053   vlib_get_buffers (vm, from, b, n_left_from);
1054
1055   while (n_left_from > 0)
1056     {
1057       vlib_buffer_t *b0;
1058       u32 sw_if_index0, rx_fib_index0;
1059       ip_protocol_t proto0;
1060       ip4_header_t *ip0;
1061       udp_header_t *udp0;
1062       icmp46_header_t *icmp0;
1063       snat_session_t *s0 = 0;
1064       clib_bihash_kv_16_8_t kv0 = {}, value0;
1065       lb_nat_type_t lb_nat0;
1066       twice_nat_type_t twice_nat0;
1067       u8 identity_nat0;
1068       ip4_address_t sm_addr;
1069       u16 sm_port;
1070       u32 sm_fib_index;
1071       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
1072
1073       b0 = *b;
1074       next[0] = vnet_buffer2 (b0)->nat.arc_next;
1075
1076       vnet_buffer (b0)->snat.flags = 0;
1077       ip0 = vlib_buffer_get_current (b0);
1078
1079       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1080       rx_fib_index0 =
1081         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
1082
1083       if (PREDICT_FALSE (ip0->ttl == 1))
1084         {
1085           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1086           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1087                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1088                                        0);
1089           next[0] = NAT_NEXT_ICMP_ERROR;
1090           goto trace0;
1091         }
1092
1093       udp0 = ip4_next_header (ip0);
1094       icmp0 = (icmp46_header_t *) udp0;
1095       proto0 = ip0->protocol;
1096
1097       if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto0)))
1098         {
1099           s0 = nat44_ed_out2in_slowpath_unknown_proto (
1100             sm, b0, ip0, rx_fib_index0, thread_index, now, vm, node);
1101           if (!sm->forwarding_enabled)
1102             {
1103               if (!s0)
1104                 next[0] = NAT_NEXT_DROP;
1105             }
1106           if (NAT_NEXT_DROP != next[0] && s0 &&
1107               NAT_ED_TRNSL_ERR_SUCCESS !=
1108                 (translation_error = nat_6t_flow_buf_translate_o2i (
1109                    vm, sm, b0, ip0, &s0->o2i, proto0,
1110                    0 /* is_output_feature */)))
1111             {
1112               next[0] = NAT_NEXT_DROP;
1113               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1114               goto trace0;
1115             }
1116
1117           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.other,
1118                                          thread_index, sw_if_index0, 1);
1119           goto trace0;
1120         }
1121
1122       if (PREDICT_FALSE (proto0 == IP_PROTOCOL_ICMP))
1123         {
1124           next[0] = icmp_out2in_ed_slow_path
1125             (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1126              next[0], now, thread_index, &s0);
1127
1128           if (NAT_NEXT_DROP != next[0] && s0 &&
1129               NAT_ED_TRNSL_ERR_SUCCESS !=
1130                 (translation_error = nat_6t_flow_buf_translate_o2i (
1131                    vm, sm, b0, ip0, &s0->o2i, proto0,
1132                    0 /* is_output_feature */)))
1133             {
1134               next[0] = NAT_NEXT_DROP;
1135               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TRNSL_FAILED];
1136               goto trace0;
1137             }
1138
1139           if (NAT_NEXT_DROP != next[0])
1140             {
1141               vlib_increment_simple_counter (
1142                 &sm->counters.slowpath.out2in.icmp, thread_index, sw_if_index0,
1143                 1);
1144             }
1145           goto trace0;
1146         }
1147
1148       init_ed_k (
1149         &kv0, ip0->src_address.as_u32, vnet_buffer (b0)->ip.reass.l4_src_port,
1150         ip0->dst_address.as_u32, vnet_buffer (b0)->ip.reass.l4_dst_port,
1151         rx_fib_index0, ip0->protocol);
1152
1153       s0 = NULL;
1154       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
1155         {
1156           ASSERT (thread_index == ed_value_get_thread_index (&value0));
1157           s0 =
1158             pool_elt_at_index (tsm->sessions,
1159                                ed_value_get_session_index (&value0));
1160         }
1161
1162       if (!s0)
1163         {
1164           /* Try to match static mapping by external address and port,
1165              destination address and port in packet */
1166
1167           if (snat_static_mapping_match (
1168                 vm, ip0->dst_address, vnet_buffer (b0)->ip.reass.l4_dst_port,
1169                 rx_fib_index0, proto0, &sm_addr, &sm_port, &sm_fib_index, 1, 0,
1170                 &twice_nat0, &lb_nat0, &ip0->src_address, &identity_nat0, &m))
1171             {
1172               /*
1173                * Send DHCP packets to the ipv4 stack, or we won't
1174                * be able to use dhcp client on the outside interface
1175                */
1176               if (PREDICT_FALSE (
1177                     proto0 == IP_PROTOCOL_UDP &&
1178                     (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1179                      clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_client))))
1180                 {
1181                   goto trace0;
1182                 }
1183
1184               if (!sm->forwarding_enabled)
1185                 {
1186                   b0->error =
1187                     node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1188                   next[0] = NAT_NEXT_DROP;
1189                 }
1190               else
1191                 {
1192                   if (next_src_nat (
1193                         sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1194                         vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0))
1195                     {
1196                       next[0] = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1197                     }
1198                   else
1199                     {
1200                       create_bypass_for_fwd (sm, b0, s0, ip0, rx_fib_index0,
1201                                              thread_index);
1202                     }
1203                 }
1204               goto trace0;
1205             }
1206
1207           if (PREDICT_FALSE (identity_nat0))
1208             goto trace0;
1209
1210           if ((proto0 == IP_PROTOCOL_TCP) &&
1211               !tcp_flags_is_init (
1212                 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
1213             {
1214               b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1215               next[0] = NAT_NEXT_DROP;
1216               goto trace0;
1217             }
1218
1219           /* Create session initiated by host from external network */
1220           s0 = create_session_for_static_mapping_ed (
1221             sm, b0, sm_addr, sm_port, sm_fib_index, ip0->dst_address,
1222             vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0, proto0,
1223             node, thread_index, twice_nat0, lb_nat0, now, m);
1224           if (!s0)
1225             {
1226               next[0] = NAT_NEXT_DROP;
1227               goto trace0;
1228             }
1229         }
1230
1231       if (NAT_ED_TRNSL_ERR_SUCCESS !=
1232           (translation_error = nat_6t_flow_buf_translate_o2i (
1233              vm, sm, b0, ip0, &s0->o2i, proto0, 0 /* is_output_feature */)))
1234         {
1235           next[0] = NAT_NEXT_DROP;
1236           goto trace0;
1237         }
1238
1239       if (PREDICT_TRUE (proto0 == IP_PROTOCOL_TCP))
1240         {
1241           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.tcp,
1242                                          thread_index, sw_if_index0, 1);
1243           nat44_set_tcp_session_state_o2i (sm, now, s0,
1244                                            vnet_buffer (b0)->ip.
1245                                            reass.icmp_type_or_tcp_flags,
1246                                            thread_index);
1247         }
1248       else
1249         {
1250           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.udp,
1251                                          thread_index, sw_if_index0, 1);
1252         }
1253
1254       /* Accounting */
1255       nat44_session_update_counters (s0, now,
1256                                      vlib_buffer_length_in_chain (vm, b0),
1257                                      thread_index);
1258       /* Per-user LRU list maintenance */
1259       nat44_session_update_lru (sm, s0, thread_index);
1260
1261     trace0:
1262       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1263                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1264         {
1265           nat44_ed_out2in_trace_t *t =
1266             vlib_add_trace (vm, node, b0, sizeof (*t));
1267           t->sw_if_index = sw_if_index0;
1268           t->next_index = next[0];
1269           t->is_slow_path = 1;
1270           t->translation_error = translation_error;
1271           clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
1272
1273           if (s0)
1274             {
1275               t->session_index = s0 - tsm->sessions;
1276               clib_memcpy (&t->i2of, &s0->i2o, sizeof (t->i2of));
1277               clib_memcpy (&t->o2if, &s0->o2i, sizeof (t->o2if));
1278               t->tcp_state = s0->tcp_state;
1279             }
1280           else
1281             {
1282               t->session_index = ~0;
1283             }
1284         }
1285
1286       if (next[0] == NAT_NEXT_DROP)
1287         {
1288           vlib_increment_simple_counter (&sm->counters.slowpath.out2in.drops,
1289                                          thread_index, sw_if_index0, 1);
1290         }
1291
1292       n_left_from--;
1293       next++;
1294       b++;
1295     }
1296
1297   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1298                                frame->n_vectors);
1299
1300   return frame->n_vectors;
1301 }
1302
1303 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1304                                      vlib_node_runtime_t * node,
1305                                      vlib_frame_t * frame)
1306 {
1307   if (snat_main.num_workers > 1)
1308     {
1309       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1310     }
1311   else
1312     {
1313       return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1314     }
1315 }
1316
1317 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1318   .name = "nat44-ed-out2in",
1319   .vector_size = sizeof (u32),
1320   .sibling_of = "nat-default",
1321   .format_trace = format_nat44_ed_out2in_trace,
1322   .type = VLIB_NODE_TYPE_INTERNAL,
1323   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1324   .error_strings = nat_out2in_ed_error_strings,
1325   .runtime_data_bytes = sizeof (snat_runtime_t),
1326 };
1327
1328 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1329                                               vlib_node_runtime_t * node,
1330                                               vlib_frame_t * frame)
1331 {
1332   return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1333 }
1334
1335 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1336   .name = "nat44-ed-out2in-slowpath",
1337   .vector_size = sizeof (u32),
1338   .sibling_of = "nat-default",
1339   .format_trace = format_nat44_ed_out2in_trace,
1340   .type = VLIB_NODE_TYPE_INTERNAL,
1341   .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1342   .error_strings = nat_out2in_ed_error_strings,
1343   .runtime_data_bytes = sizeof (snat_runtime_t),
1344 };
1345
1346 static u8 *
1347 format_nat_pre_trace (u8 * s, va_list * args)
1348 {
1349   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1350   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1351   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1352   return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1353                  t->arc_next_index);
1354 }
1355
1356 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1357                                     vlib_node_runtime_t * node,
1358                                     vlib_frame_t * frame)
1359 {
1360   return nat_pre_node_fn_inline (vm, node, frame,
1361                                  NAT_NEXT_OUT2IN_ED_FAST_PATH);
1362 }
1363
1364 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1365   .name = "nat-pre-out2in",
1366   .vector_size = sizeof (u32),
1367   .sibling_of = "nat-default",
1368   .format_trace = format_nat_pre_trace,
1369   .type = VLIB_NODE_TYPE_INTERNAL,
1370   .n_errors = 0,
1371  };
1372
1373 /*
1374  * fd.io coding-style-patch-verification: ON
1375  *
1376  * Local Variables:
1377  * eval: (c-set-style "gnu")
1378  * End:
1379  */