src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 494                                       u32 thread_index)
 495 {
 496   nat_ed_ses_key_t key;
 497   clib_bihash_kv_16_8_t kv, value;
 498   udp_header_t *udp;
 499   snat_session_t *s = 0;
 500   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 501
 502   if (!sm->forwarding_enabled)
 503     return 0;
 504
 505   if (ip->protocol == IP_PROTOCOL_ICMP)
 506     {
 507       if (icmp_get_ed_key (ip, &key))
 508         return 0;
 509     }
 510   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 511     {
 512       udp = ip4_next_header(ip);
 513       key.l_addr = ip->src_address;
 514       key.r_addr = ip->dst_address;
 515       key.proto = ip->protocol;
 516       key.r_port = udp->dst_port;
 517       key.l_port = udp->src_port;
 518     }
 519   else
 520     {
 521       key.l_addr = ip->src_address;
 522       key.r_addr = ip->dst_address;
 523       key.proto = ip->protocol;
 524       key.l_port = key.r_port = 0;
 525     }
 526   key.fib_index = 0;
 527   kv.key[0] = key.as_u64[0];
 528   kv.key[1] = key.as_u64[1];
 529
 530   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 531     {
 532       s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value.value);
 533       if (is_fwd_bypass_session (s))
 534         {
 535           if (ip->protocol == IP_PROTOCOL_TCP)
 536             {
 537               tcp_header_t *tcp = ip4_next_header(ip);
 538               nat44_set_tcp_session_state (sm, s, tcp, thread_index);
 539             }
 540           /* Per-user LRU list maintenance */
 541           clib_dlist_remove (tsm->list_pool, s->per_user_index);
 542           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 543                               s->per_user_index);
 544           return 1;
 545         }
 546       else
 547         return 0;
 548     }
 549
 550   return 0;
 551 }
 552
 553 /**
 554  * Get address and port values to be used for ICMP packet translation
 555  * and create session if needed
 556  *
 557  * @param[in,out] sm             NAT main
 558  * @param[in,out] node           NAT node runtime
 559  * @param[in] thread_index       thread index
 560  * @param[in,out] b0             buffer containing packet to be translated
 561  * @param[out] p_proto           protocol used for matching
 562  * @param[out] p_value           address and port after NAT translation
 563  * @param[out] p_dont_translate  if packet should not be translated
 564  * @param d                      optional parameter
 565  * @param e                      optional parameter
 566  */
 567 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 568                            u32 thread_index, vlib_buffer_t *b0,
 569                            ip4_header_t *ip0, u8 *p_proto,
 570                            snat_session_key_t *p_value,
 571                            u8 *p_dont_translate, void *d, void *e)
 572 {
 573   icmp46_header_t *icmp0;
 574   u32 sw_if_index0;
 575   u32 rx_fib_index0;
 576   snat_session_key_t key0;
 577   snat_session_t *s0 = 0;
 578   u8 dont_translate = 0;
 579   clib_bihash_kv_8_8_t kv0, value0;
 580   u32 next0 = ~0;
 581   int err;
 582
 583   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 584   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 585   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 586
 587   err = icmp_get_key (ip0, &key0);
 588   if (err != -1)
 589     {
 590       b0->error = node->errors[err];
 591       next0 = SNAT_IN2OUT_NEXT_DROP;
 592       goto out;
 593     }
 594   key0.fib_index = rx_fib_index0;
 595
 596   kv0.key = key0.as_u64;
 597
 598   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 599                               &value0))
 600     {
 601       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 602         {
 603           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 604               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 605             {
 606               dont_translate = 1;
 607               goto out;
 608             }
 609         }
 610       else
 611         {
 612           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 613               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 614             {
 615               dont_translate = 1;
 616               goto out;
 617             }
 618         }
 619
 620       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 621         {
 622           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 623           next0 = SNAT_IN2OUT_NEXT_DROP;
 624           goto out;
 625         }
 626
 627       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 628                          &s0, node, next0, thread_index);
 629
 630       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 631         goto out;
 632     }
 633   else
 634     {
 635       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 636                         icmp0->type != ICMP4_echo_reply &&
 637                         !icmp_is_error_message (icmp0)))
 638         {
 639           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 640           next0 = SNAT_IN2OUT_NEXT_DROP;
 641           goto out;
 642         }
 643
 644       if (PREDICT_FALSE (value0.value == ~0ULL))
 645         {
 646           nat_ed_ses_key_t key;
 647           clib_bihash_kv_16_8_t s_kv, s_value;
 648
 649           key.as_u64[0] = 0;
 650           key.as_u64[1] = 0;
 651           if (icmp_get_ed_key (ip0, &key))
 652             {
 653               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 654               next0 = SNAT_IN2OUT_NEXT_DROP;
 655               goto out;
 656             }
 657           key.fib_index = rx_fib_index0;
 658           s_kv.key[0] = key.as_u64[0];
 659           s_kv.key[1] = key.as_u64[1];
 660           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 661             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 662                                     s_value.value);
 663           else
 664            {
 665               next0 = SNAT_IN2OUT_NEXT_DROP;
 666               goto out;
 667            }
 668         }
 669       else
 670         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 671                                 value0.value);
 672     }
 673
 674 out:
 675   *p_proto = key0.protocol;
 676   if (s0)
 677     *p_value = s0->out2in;
 678   *p_dont_translate = dont_translate;
 679   if (d)
 680     *(snat_session_t**)d = s0;
 681   return next0;
 682 }
 683
 684 /**
 685  * Get address and port values to be used for ICMP packet translation
 686  *
 687  * @param[in] sm                 NAT main
 688  * @param[in,out] node           NAT node runtime
 689  * @param[in] thread_index       thread index
 690  * @param[in,out] b0             buffer containing packet to be translated
 691  * @param[out] p_proto           protocol used for matching
 692  * @param[out] p_value           address and port after NAT translation
 693  * @param[out] p_dont_translate  if packet should not be translated
 694  * @param d                      optional parameter
 695  * @param e                      optional parameter
 696  */
 697 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 698                            u32 thread_index, vlib_buffer_t *b0,
 699                            ip4_header_t *ip0, u8 *p_proto,
 700                            snat_session_key_t *p_value,
 701                            u8 *p_dont_translate, void *d, void *e)
 702 {
 703   icmp46_header_t *icmp0;
 704   u32 sw_if_index0;
 705   u32 rx_fib_index0;
 706   snat_session_key_t key0;
 707   snat_session_key_t sm0;
 708   u8 dont_translate = 0;
 709   u8 is_addr_only;
 710   u32 next0 = ~0;
 711   int err;
 712
 713   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 714   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 715   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 716
 717   err = icmp_get_key (ip0, &key0);
 718   if (err != -1)
 719     {
 720       b0->error = node->errors[err];
 721       next0 = SNAT_IN2OUT_NEXT_DROP;
 722       goto out2;
 723     }
 724   key0.fib_index = rx_fib_index0;
 725
 726   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 727     {
 728       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 729           IP_PROTOCOL_ICMP, rx_fib_index0)))
 730         {
 731           dont_translate = 1;
 732           goto out;
 733         }
 734
 735       if (icmp_is_error_message (icmp0))
 736         {
 737           next0 = SNAT_IN2OUT_NEXT_DROP;
 738           goto out;
 739         }
 740
 741       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 742       next0 = SNAT_IN2OUT_NEXT_DROP;
 743       goto out;
 744     }
 745
 746   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 747                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 748                     !icmp_is_error_message (icmp0)))
 749     {
 750       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 751       next0 = SNAT_IN2OUT_NEXT_DROP;
 752       goto out;
 753     }
 754
 755 out:
 756   *p_value = sm0;
 757 out2:
 758   *p_proto = key0.protocol;
 759   *p_dont_translate = dont_translate;
 760   return next0;
 761 }
 762
 763 static inline u32 icmp_in2out (snat_main_t *sm,
 764                                vlib_buffer_t * b0,
 765                                ip4_header_t * ip0,
 766                                icmp46_header_t * icmp0,
 767                                u32 sw_if_index0,
 768                                u32 rx_fib_index0,
 769                                vlib_node_runtime_t * node,
 770                                u32 next0,
 771                                u32 thread_index,
 772                                void *d,
 773                                void *e)
 774 {
 775   snat_session_key_t sm0;
 776   u8 protocol;
 777   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 778   ip4_header_t *inner_ip0;
 779   void *l4_header = 0;
 780   icmp46_header_t *inner_icmp0;
 781   u8 dont_translate;
 782   u32 new_addr0, old_addr0;
 783   u16 old_id0, new_id0;
 784   ip_csum_t sum0;
 785   u16 checksum0;
 786   u32 next0_tmp;
 787
 788   echo0 = (icmp_echo_header_t *)(icmp0+1);
 789
 790   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 791                                        &protocol, &sm0, &dont_translate, d, e);
 792   if (next0_tmp != ~0)
 793     next0 = next0_tmp;
 794   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 795     goto out;
 796
 797   sum0 = ip_incremental_checksum (0, icmp0,
 798                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 799   checksum0 = ~ip_csum_fold (sum0);
 800   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 801     {
 802       next0 = SNAT_IN2OUT_NEXT_DROP;
 803       goto out;
 804     }
 805
 806   old_addr0 = ip0->src_address.as_u32;
 807   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 808   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 809     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 810
 811   sum0 = ip0->checksum;
 812   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 813                          src_address /* changed member */);
 814   ip0->checksum = ip_csum_fold (sum0);
 815
 816   if (icmp0->checksum == 0)
 817     icmp0->checksum = 0xffff;
 818
 819   if (!icmp_is_error_message (icmp0))
 820     {
 821       new_id0 = sm0.port;
 822       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 823         {
 824           old_id0 = echo0->identifier;
 825           new_id0 = sm0.port;
 826           echo0->identifier = new_id0;
 827
 828           sum0 = icmp0->checksum;
 829           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 830                                  identifier);
 831           icmp0->checksum = ip_csum_fold (sum0);
 832         }
 833     }
 834   else
 835     {
 836       inner_ip0 = (ip4_header_t *)(echo0+1);
 837       l4_header = ip4_next_header (inner_ip0);
 838
 839       if (!ip4_header_checksum_is_valid (inner_ip0))
 840         {
 841           next0 = SNAT_IN2OUT_NEXT_DROP;
 842           goto out;
 843         }
 844
 845       old_addr0 = inner_ip0->dst_address.as_u32;
 846       inner_ip0->dst_address = sm0.addr;
 847       new_addr0 = inner_ip0->dst_address.as_u32;
 848
 849       sum0 = icmp0->checksum;
 850       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 851                              dst_address /* changed member */);
 852       icmp0->checksum = ip_csum_fold (sum0);
 853
 854       switch (protocol)
 855         {
 856           case SNAT_PROTOCOL_ICMP:
 857             inner_icmp0 = (icmp46_header_t*)l4_header;
 858             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 859
 860             old_id0 = inner_echo0->identifier;
 861             new_id0 = sm0.port;
 862             inner_echo0->identifier = new_id0;
 863
 864             sum0 = icmp0->checksum;
 865             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 866                                    identifier);
 867             icmp0->checksum = ip_csum_fold (sum0);
 868             break;
 869           case SNAT_PROTOCOL_UDP:
 870           case SNAT_PROTOCOL_TCP:
 871             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 872             new_id0 = sm0.port;
 873             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 874
 875             sum0 = icmp0->checksum;
 876             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 877                                    dst_port);
 878             icmp0->checksum = ip_csum_fold (sum0);
 879             break;
 880           default:
 881             ASSERT(0);
 882         }
 883     }
 884
 885 out:
 886   return next0;
 887 }
 888
 889 /**
 890  * @brief Hairpinning
 891  *
 892  * Hairpinning allows two endpoints on the internal side of the NAT to
 893  * communicate even if they only use each other's external IP addresses
 894  * and ports.
 895  *
 896  * @param sm     NAT main.
 897  * @param b0     Vlib buffer.
 898  * @param ip0    IP header.
 899  * @param udp0   UDP header.
 900  * @param tcp0   TCP header.
 901  * @param proto0 NAT protocol.
 902  */
 903 static inline int
 904 snat_hairpinning (snat_main_t *sm,
 905                   vlib_buffer_t * b0,
 906                   ip4_header_t * ip0,
 907                   udp_header_t * udp0,
 908                   tcp_header_t * tcp0,
 909                   u32 proto0)
 910 {
 911   snat_session_key_t key0, sm0;
 912   snat_session_t * s0;
 913   clib_bihash_kv_8_8_t kv0, value0;
 914   ip_csum_t sum0;
 915   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 916   u16 new_dst_port0, old_dst_port0;
 917
 918   key0.addr = ip0->dst_address;
 919   key0.port = udp0->dst_port;
 920   key0.protocol = proto0;
 921   key0.fib_index = sm->outside_fib_index;
 922   kv0.key = key0.as_u64;
 923
 924   /* Check if destination is static mappings */
 925   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 926     {
 927       new_dst_addr0 = sm0.addr.as_u32;
 928       new_dst_port0 = sm0.port;
 929       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 930     }
 931   /* or active session */
 932   else
 933     {
 934       if (sm->num_workers > 1)
 935         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 936       else
 937         ti = sm->num_workers;
 938
 939       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 940         {
 941           si = value0.value;
 942
 943           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 944           new_dst_addr0 = s0->in2out.addr.as_u32;
 945           new_dst_port0 = s0->in2out.port;
 946           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 947         }
 948     }
 949
 950   /* Destination is behind the same NAT, use internal address and port */
 951   if (new_dst_addr0)
 952     {
 953       old_dst_addr0 = ip0->dst_address.as_u32;
 954       ip0->dst_address.as_u32 = new_dst_addr0;
 955       sum0 = ip0->checksum;
 956       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 957                              ip4_header_t, dst_address);
 958       ip0->checksum = ip_csum_fold (sum0);
 959
 960       old_dst_port0 = tcp0->dst;
 961       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 962         {
 963           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 964             {
 965               tcp0->dst = new_dst_port0;
 966               sum0 = tcp0->checksum;
 967               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 968                                      ip4_header_t, dst_address);
 969               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 970                                      ip4_header_t /* cheat */, length);
 971               tcp0->checksum = ip_csum_fold(sum0);
 972             }
 973           else
 974             {
 975               udp0->dst_port = new_dst_port0;
 976               udp0->checksum = 0;
 977             }
 978         }
 979       else
 980         {
 981           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 982             {
 983               sum0 = tcp0->checksum;
 984               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 985                                      ip4_header_t, dst_address);
 986               tcp0->checksum = ip_csum_fold(sum0);
 987             }
 988         }
 989       return 1;
 990     }
 991   return 0;
 992 }
 993
 994 static inline void
 995 snat_icmp_hairpinning (snat_main_t *sm,
 996                        vlib_buffer_t * b0,
 997                        ip4_header_t * ip0,
 998                        icmp46_header_t * icmp0)
 999 {
1000   snat_session_key_t key0, sm0;
1001   clib_bihash_kv_8_8_t kv0, value0;
1002   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
1003   ip_csum_t sum0;
1004   snat_session_t *s0;
1005
1006   if (!icmp_is_error_message (icmp0))
1007     {
1008       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1009       u16 icmp_id0 = echo0->identifier;
1010       key0.addr = ip0->dst_address;
1011       key0.port = icmp_id0;
1012       key0.protocol = SNAT_PROTOCOL_ICMP;
1013       key0.fib_index = sm->outside_fib_index;
1014       kv0.key = key0.as_u64;
1015
1016       if (sm->num_workers > 1)
1017         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1018       else
1019         ti = sm->num_workers;
1020
1021       /* Check if destination is in active sessions */
1022       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1023                                   &value0))
1024         {
1025           /* or static mappings */
1026           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1027             {
1028               new_dst_addr0 = sm0.addr.as_u32;
1029               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1030             }
1031         }
1032       else
1033         {
1034           si = value0.value;
1035
1036           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1037           new_dst_addr0 = s0->in2out.addr.as_u32;
1038           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1039           echo0->identifier = s0->in2out.port;
1040           sum0 = icmp0->checksum;
1041           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1042                                  icmp_echo_header_t, identifier);
1043           icmp0->checksum = ip_csum_fold (sum0);
1044         }
1045
1046       /* Destination is behind the same NAT, use internal address and port */
1047       if (new_dst_addr0)
1048         {
1049           old_dst_addr0 = ip0->dst_address.as_u32;
1050           ip0->dst_address.as_u32 = new_dst_addr0;
1051           sum0 = ip0->checksum;
1052           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1053                                  ip4_header_t, dst_address);
1054           ip0->checksum = ip_csum_fold (sum0);
1055         }
1056     }
1057
1058 }
1059
1060 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1061                                          vlib_buffer_t * b0,
1062                                          ip4_header_t * ip0,
1063                                          icmp46_header_t * icmp0,
1064                                          u32 sw_if_index0,
1065                                          u32 rx_fib_index0,
1066                                          vlib_node_runtime_t * node,
1067                                          u32 next0,
1068                                          f64 now,
1069                                          u32 thread_index,
1070                                          snat_session_t ** p_s0)
1071 {
1072   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1073                       next0, thread_index, p_s0, 0);
1074   snat_session_t * s0 = *p_s0;
1075   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1076     {
1077       /* Hairpinning */
1078       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1079         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1080       /* Accounting */
1081       s0->last_heard = now;
1082       s0->total_pkts++;
1083       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1084       /* Per-user LRU list maintenance */
1085       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1086                          s0->per_user_index);
1087       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1088                           s0->per_user_list_head_index,
1089                           s0->per_user_index);
1090     }
1091   return next0;
1092 }
1093 static inline void
1094 snat_hairpinning_unknown_proto (snat_main_t *sm,
1095                                 vlib_buffer_t * b,
1096                                 ip4_header_t * ip)
1097 {
1098   u32 old_addr, new_addr = 0, ti = 0;
1099   clib_bihash_kv_8_8_t kv, value;
1100   clib_bihash_kv_16_8_t s_kv, s_value;
1101   nat_ed_ses_key_t key;
1102   snat_session_key_t m_key;
1103   snat_static_mapping_t *m;
1104   ip_csum_t sum;
1105   snat_session_t *s;
1106
1107   old_addr = ip->dst_address.as_u32;
1108   key.l_addr.as_u32 = ip->dst_address.as_u32;
1109   key.r_addr.as_u32 = ip->src_address.as_u32;
1110   key.fib_index = sm->outside_fib_index;
1111   key.proto = ip->protocol;
1112   key.r_port = 0;
1113   key.l_port = 0;
1114   s_kv.key[0] = key.as_u64[0];
1115   s_kv.key[1] = key.as_u64[1];
1116   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1117     {
1118       m_key.addr = ip->dst_address;
1119       m_key.fib_index = sm->outside_fib_index;
1120       m_key.port = 0;
1121       m_key.protocol = 0;
1122       kv.key = m_key.as_u64;
1123       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1124         return;
1125
1126       m = pool_elt_at_index (sm->static_mappings, value.value);
1127       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1128         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1129       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1130     }
1131   else
1132     {
1133       if (sm->num_workers > 1)
1134         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1135       else
1136         ti = sm->num_workers;
1137
1138       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1139       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1140         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1141       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1142     }
1143   sum = ip->checksum;
1144   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1145   ip->checksum = ip_csum_fold (sum);
1146 }
1147
1148 static snat_session_t *
1149 snat_in2out_unknown_proto (snat_main_t *sm,
1150                            vlib_buffer_t * b,
1151                            ip4_header_t * ip,
1152                            u32 rx_fib_index,
1153                            u32 thread_index,
1154                            f64 now,
1155                            vlib_main_t * vm,
1156                            vlib_node_runtime_t * node)
1157 {
1158   clib_bihash_kv_8_8_t kv, value;
1159   clib_bihash_kv_16_8_t s_kv, s_value;
1160   snat_static_mapping_t *m;
1161   snat_session_key_t m_key;
1162   u32 old_addr, new_addr = 0;
1163   ip_csum_t sum;
1164   snat_user_t *u;
1165   dlist_elt_t *head, *elt;
1166   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1167   u32 elt_index, head_index, ses_index;
1168   snat_session_t * s;
1169   nat_ed_ses_key_t key;
1170   u32 address_index = ~0;
1171   int i;
1172   u8 is_sm = 0;
1173
1174   old_addr = ip->src_address.as_u32;
1175
1176   key.l_addr = ip->src_address;
1177   key.r_addr = ip->dst_address;
1178   key.fib_index = rx_fib_index;
1179   key.proto = ip->protocol;
1180   key.l_port = 0;
1181   key.r_port = 0;
1182   s_kv.key[0] = key.as_u64[0];
1183   s_kv.key[1] = key.as_u64[1];
1184
1185   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1186     {
1187       s = pool_elt_at_index (tsm->sessions, s_value.value);
1188       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1189     }
1190   else
1191     {
1192       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1193         {
1194           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1195           nat_ipfix_logging_max_sessions(sm->max_translations);
1196           return 0;
1197         }
1198
1199       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1200                                   thread_index);
1201       if (!u)
1202         {
1203           clib_warning ("create NAT user failed");
1204           return 0;
1205         }
1206
1207       m_key.addr = ip->src_address;
1208       m_key.port = 0;
1209       m_key.protocol = 0;
1210       m_key.fib_index = rx_fib_index;
1211       kv.key = m_key.as_u64;
1212
1213       /* Try to find static mapping first */
1214       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1215         {
1216           m = pool_elt_at_index (sm->static_mappings, value.value);
1217           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1218           is_sm = 1;
1219           goto create_ses;
1220         }
1221       /* Fallback to 3-tuple key */
1222       else
1223         {
1224           /* Choose same out address as for TCP/UDP session to same destination */
1225           head_index = u->sessions_per_user_list_head_index;
1226           head = pool_elt_at_index (tsm->list_pool, head_index);
1227           elt_index = head->next;
1228           if (PREDICT_FALSE (elt_index == ~0))
1229             ses_index = ~0;
1230           else
1231             {
1232               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1233               ses_index = elt->value;
1234             }
1235
1236           while (ses_index != ~0)
1237             {
1238               s =  pool_elt_at_index (tsm->sessions, ses_index);
1239               elt_index = elt->next;
1240               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1241               ses_index = elt->value;
1242
1243               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1244                 {
1245                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1246                   address_index = s->outside_address_index;
1247
1248                   key.fib_index = sm->outside_fib_index;
1249                   key.l_addr.as_u32 = new_addr;
1250                   s_kv.key[0] = key.as_u64[0];
1251                   s_kv.key[1] = key.as_u64[1];
1252                   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1253                     break;
1254
1255                   goto create_ses;
1256                 }
1257             }
1258           key.fib_index = sm->outside_fib_index;
1259           for (i = 0; i < vec_len (sm->addresses); i++)
1260             {
1261               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1262               s_kv.key[0] = key.as_u64[0];
1263               s_kv.key[1] = key.as_u64[1];
1264               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1265                 {
1266                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1267                   address_index = i;
1268                   goto create_ses;
1269                 }
1270             }
1271           return 0;
1272         }
1273
1274 create_ses:
1275       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1276       if (!s)
1277         {
1278           clib_warning ("create NAT session failed");
1279           return 0;
1280         }
1281
1282       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1283       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1284       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1285       s->outside_address_index = address_index;
1286       s->out2in.addr.as_u32 = new_addr;
1287       s->out2in.fib_index = sm->outside_fib_index;
1288       s->in2out.addr.as_u32 = old_addr;
1289       s->in2out.fib_index = rx_fib_index;
1290       s->in2out.port = s->out2in.port = ip->protocol;
1291       if (is_sm)
1292         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1293       user_session_increment (sm, u, is_sm);
1294
1295       /* Add to lookup tables */
1296       key.l_addr.as_u32 = old_addr;
1297       key.r_addr = ip->dst_address;
1298       key.proto = ip->protocol;
1299       key.fib_index = rx_fib_index;
1300       s_kv.key[0] = key.as_u64[0];
1301       s_kv.key[1] = key.as_u64[1];
1302       s_kv.value = s - tsm->sessions;
1303       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1304         clib_warning ("in2out key add failed");
1305
1306       key.l_addr.as_u32 = new_addr;
1307       key.fib_index = sm->outside_fib_index;
1308       s_kv.key[0] = key.as_u64[0];
1309       s_kv.key[1] = key.as_u64[1];
1310       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1311         clib_warning ("out2in key add failed");
1312   }
1313
1314   /* Update IP checksum */
1315   sum = ip->checksum;
1316   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1317   ip->checksum = ip_csum_fold (sum);
1318
1319   /* Accounting */
1320   s->last_heard = now;
1321   s->total_pkts++;
1322   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1323   /* Per-user LRU list maintenance */
1324   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1325   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1326                       s->per_user_index);
1327
1328   /* Hairpinning */
1329   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1330     snat_hairpinning_unknown_proto(sm, b, ip);
1331
1332   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1333     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1334
1335   return s;
1336 }
1337
1338 static snat_session_t *
1339 snat_in2out_lb (snat_main_t *sm,
1340                 vlib_buffer_t * b,
1341                 ip4_header_t * ip,
1342                 u32 rx_fib_index,
1343                 u32 thread_index,
1344                 f64 now,
1345                 vlib_main_t * vm,
1346                 vlib_node_runtime_t * node)
1347 {
1348   nat_ed_ses_key_t key;
1349   clib_bihash_kv_16_8_t s_kv, s_value;
1350   udp_header_t *udp = ip4_next_header (ip);
1351   tcp_header_t *tcp = (tcp_header_t *) udp;
1352   snat_session_t *s = 0;
1353   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1354   u32 old_addr, new_addr;
1355   u16 new_port, old_port;
1356   ip_csum_t sum;
1357   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1358   snat_session_key_t e_key, l_key;
1359   snat_user_t *u;
1360   u8 lb;
1361
1362   old_addr = ip->src_address.as_u32;
1363
1364   key.l_addr = ip->src_address;
1365   key.r_addr = ip->dst_address;
1366   key.fib_index = rx_fib_index;
1367   key.proto = ip->protocol;
1368   key.r_port = udp->dst_port;
1369   key.l_port = udp->src_port;
1370   s_kv.key[0] = key.as_u64[0];
1371   s_kv.key[1] = key.as_u64[1];
1372
1373   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1374     {
1375       s = pool_elt_at_index (tsm->sessions, s_value.value);
1376       if (is_fwd_bypass_session (s))
1377         {
1378           if (ip->protocol == IP_PROTOCOL_TCP)
1379             nat44_set_tcp_session_state (sm, s, tcp, thread_index);
1380           /* Per-user LRU list maintenance */
1381           clib_dlist_remove (tsm->list_pool, s->per_user_index);
1382           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1383                               s->per_user_index);
1384           return 0;
1385         }
1386     }
1387   else
1388     {
1389       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1390         {
1391           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1392           nat_ipfix_logging_max_sessions(sm->max_translations);
1393           return 0;
1394         }
1395
1396       l_key.addr = ip->src_address;
1397       l_key.port = udp->src_port;
1398       l_key.protocol = proto;
1399       l_key.fib_index = rx_fib_index;
1400       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1401         return 0;
1402
1403       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1404                                   thread_index);
1405       if (!u)
1406         {
1407           clib_warning ("create NAT user failed");
1408           return 0;
1409         }
1410
1411       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1412       if (!s)
1413         {
1414           clib_warning ("create NAT session failed");
1415           return 0;
1416         }
1417
1418       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1419       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1420       if (lb)
1421         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1422       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1423       s->outside_address_index = ~0;
1424       s->in2out = l_key;
1425       s->out2in = e_key;
1426       s->out2in.protocol = l_key.protocol;
1427       user_session_increment (sm, u, 1 /* static */);
1428
1429       /* Add to lookup tables */
1430       s_kv.value = s - tsm->sessions;
1431       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1432         clib_warning ("in2out-ed key add failed");
1433
1434       key.l_addr = e_key.addr;
1435       key.fib_index = e_key.fib_index;
1436       key.l_port = e_key.port;
1437       s_kv.key[0] = key.as_u64[0];
1438       s_kv.key[1] = key.as_u64[1];
1439       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1440         clib_warning ("out2in-ed key add failed");
1441     }
1442
1443   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1444
1445   /* Update IP checksum */
1446   sum = ip->checksum;
1447   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1448   if (is_twice_nat_session (s))
1449     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1450                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1451   ip->checksum = ip_csum_fold (sum);
1452
1453   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1454     {
1455       old_port = tcp->src_port;
1456       tcp->src_port = s->out2in.port;
1457       new_port = tcp->src_port;
1458
1459       sum = tcp->checksum;
1460       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1461       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1462       if (is_twice_nat_session (s))
1463         {
1464           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1465                                 s->ext_host_addr.as_u32, ip4_header_t,
1466                                 dst_address);
1467           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1468                                 ip4_header_t, length);
1469           tcp->dst_port = s->ext_host_port;
1470           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1471         }
1472       tcp->checksum = ip_csum_fold(sum);
1473       nat44_set_tcp_session_state (sm, s, tcp, thread_index);
1474     }
1475   else
1476     {
1477       udp->src_port = s->out2in.port;
1478       if (is_twice_nat_session (s))
1479         {
1480           udp->dst_port = s->ext_host_port;
1481           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1482         }
1483       udp->checksum = 0;
1484     }
1485
1486   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1487     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1488
1489   /* Accounting */
1490   s->last_heard = now;
1491   s->total_pkts++;
1492   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1493   /* Per-user LRU list maintenance */
1494   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1495   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1496                       s->per_user_index);
1497   return s;
1498 }
1499
1500 static inline uword
1501 snat_in2out_node_fn_inline (vlib_main_t * vm,
1502                             vlib_node_runtime_t * node,
1503                             vlib_frame_t * frame, int is_slow_path,
1504                             int is_output_feature)
1505 {
1506   u32 n_left_from, * from, * to_next;
1507   snat_in2out_next_t next_index;
1508   u32 pkts_processed = 0;
1509   snat_main_t * sm = &snat_main;
1510   f64 now = vlib_time_now (vm);
1511   u32 stats_node_index;
1512   u32 thread_index = vlib_get_thread_index ();
1513
1514   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1515     snat_in2out_node.index;
1516
1517   from = vlib_frame_vector_args (frame);
1518   n_left_from = frame->n_vectors;
1519   next_index = node->cached_next_index;
1520
1521   while (n_left_from > 0)
1522     {
1523       u32 n_left_to_next;
1524
1525       vlib_get_next_frame (vm, node, next_index,
1526                            to_next, n_left_to_next);
1527
1528       while (n_left_from >= 4 && n_left_to_next >= 2)
1529         {
1530           u32 bi0, bi1;
1531           vlib_buffer_t * b0, * b1;
1532           u32 next0, next1;
1533           u32 sw_if_index0, sw_if_index1;
1534           ip4_header_t * ip0, * ip1;
1535           ip_csum_t sum0, sum1;
1536           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1537           u16 old_port0, new_port0, old_port1, new_port1;
1538           udp_header_t * udp0, * udp1;
1539           tcp_header_t * tcp0, * tcp1;
1540           icmp46_header_t * icmp0, * icmp1;
1541           snat_session_key_t key0, key1;
1542           u32 rx_fib_index0, rx_fib_index1;
1543           u32 proto0, proto1;
1544           snat_session_t * s0 = 0, * s1 = 0;
1545           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1546           u32 iph_offset0 = 0, iph_offset1 = 0;
1547
1548           /* Prefetch next iteration. */
1549           {
1550             vlib_buffer_t * p2, * p3;
1551
1552             p2 = vlib_get_buffer (vm, from[2]);
1553             p3 = vlib_get_buffer (vm, from[3]);
1554
1555             vlib_prefetch_buffer_header (p2, LOAD);
1556             vlib_prefetch_buffer_header (p3, LOAD);
1557
1558             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1559             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1560           }
1561
1562           /* speculatively enqueue b0 and b1 to the current next frame */
1563           to_next[0] = bi0 = from[0];
1564           to_next[1] = bi1 = from[1];
1565           from += 2;
1566           to_next += 2;
1567           n_left_from -= 2;
1568           n_left_to_next -= 2;
1569
1570           b0 = vlib_get_buffer (vm, bi0);
1571           b1 = vlib_get_buffer (vm, bi1);
1572
1573           if (is_output_feature)
1574             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1575
1576           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1577                  iph_offset0);
1578
1579           udp0 = ip4_next_header (ip0);
1580           tcp0 = (tcp_header_t *) udp0;
1581           icmp0 = (icmp46_header_t *) udp0;
1582
1583           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1584           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1585                                    sw_if_index0);
1586
1587           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1588
1589           if (PREDICT_FALSE(ip0->ttl == 1))
1590             {
1591               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1592               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1593                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1594                                            0);
1595               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1596               goto trace00;
1597             }
1598
1599           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1600
1601           /* Next configured feature, probably ip4-lookup */
1602           if (is_slow_path)
1603             {
1604               if (PREDICT_FALSE (proto0 == ~0))
1605                 {
1606                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1607                                                   thread_index, now, vm, node);
1608                   if (!s0)
1609                     next0 = SNAT_IN2OUT_NEXT_DROP;
1610                   goto trace00;
1611                 }
1612
1613               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1614                 {
1615                   next0 = icmp_in2out_slow_path
1616                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1617                      node, next0, now, thread_index, &s0);
1618                   goto trace00;
1619                 }
1620             }
1621           else
1622             {
1623               if (is_output_feature)
1624                 {
1625                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
1626                     goto trace00;
1627                 }
1628
1629               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1630                 {
1631                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1632                   goto trace00;
1633                 }
1634
1635               if (ip4_is_fragment (ip0))
1636                 {
1637                   next0 = SNAT_IN2OUT_NEXT_REASS;
1638                   goto trace00;
1639                 }
1640             }
1641
1642           key0.addr = ip0->src_address;
1643           key0.port = udp0->src_port;
1644           key0.protocol = proto0;
1645           key0.fib_index = rx_fib_index0;
1646
1647           kv0.key = key0.as_u64;
1648
1649           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1650               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1651             {
1652               if (is_slow_path)
1653                 {
1654                   if (is_output_feature)
1655                     {
1656                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1657                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1658                         goto trace00;
1659                     }
1660                   else
1661                     {
1662                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1663                           ip0, proto0, rx_fib_index0, thread_index)))
1664                         goto trace00;
1665                     }
1666
1667                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1668                                      &s0, node, next0, thread_index);
1669                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1670                     goto trace00;
1671                 }
1672               else
1673                 {
1674                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1675                   goto trace00;
1676                 }
1677             }
1678           else
1679             {
1680               if (PREDICT_FALSE (value0.value == ~0ULL))
1681                 {
1682                   if (is_slow_path)
1683                     {
1684                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1685                                           thread_index, now, vm, node);
1686                       if (!s0 && !sm->forwarding_enabled)
1687                         next0 = SNAT_IN2OUT_NEXT_DROP;
1688                       goto trace00;
1689                     }
1690                   else
1691                     {
1692                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1693                       goto trace00;
1694                     }
1695                 }
1696               else
1697                 {
1698                   s0 = pool_elt_at_index (
1699                     sm->per_thread_data[thread_index].sessions,
1700                     value0.value);
1701                 }
1702             }
1703
1704           b0->flags |= VNET_BUFFER_F_IS_NATED;
1705
1706           old_addr0 = ip0->src_address.as_u32;
1707           ip0->src_address = s0->out2in.addr;
1708           new_addr0 = ip0->src_address.as_u32;
1709           if (!is_output_feature)
1710             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1711
1712           sum0 = ip0->checksum;
1713           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1714                                  ip4_header_t,
1715                                  src_address /* changed member */);
1716           ip0->checksum = ip_csum_fold (sum0);
1717
1718           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1719             {
1720               old_port0 = tcp0->src_port;
1721               tcp0->src_port = s0->out2in.port;
1722               new_port0 = tcp0->src_port;
1723
1724               sum0 = tcp0->checksum;
1725               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1726                                      ip4_header_t,
1727                                      dst_address /* changed member */);
1728               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1729                                      ip4_header_t /* cheat */,
1730                                      length /* changed member */);
1731               tcp0->checksum = ip_csum_fold(sum0);
1732               nat44_set_tcp_session_state (sm, s0, tcp0, thread_index);
1733             }
1734           else
1735             {
1736               old_port0 = udp0->src_port;
1737               udp0->src_port = s0->out2in.port;
1738               udp0->checksum = 0;
1739             }
1740
1741           /* Accounting */
1742           s0->last_heard = now;
1743           s0->total_pkts++;
1744           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1745           /* Per-user LRU list maintenance */
1746           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1747                              s0->per_user_index);
1748           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1749                               s0->per_user_list_head_index,
1750                               s0->per_user_index);
1751         trace00:
1752
1753           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1754                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1755             {
1756               snat_in2out_trace_t *t =
1757                  vlib_add_trace (vm, node, b0, sizeof (*t));
1758               t->is_slow_path = is_slow_path;
1759               t->sw_if_index = sw_if_index0;
1760               t->next_index = next0;
1761                   t->session_index = ~0;
1762               if (s0)
1763                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1764             }
1765
1766           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1767
1768           if (is_output_feature)
1769             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1770
1771           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1772                  iph_offset1);
1773
1774           udp1 = ip4_next_header (ip1);
1775           tcp1 = (tcp_header_t *) udp1;
1776           icmp1 = (icmp46_header_t *) udp1;
1777
1778           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1779           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1780                                    sw_if_index1);
1781
1782           if (PREDICT_FALSE(ip1->ttl == 1))
1783             {
1784               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1785               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1786                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1787                                            0);
1788               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1789               goto trace01;
1790             }
1791
1792           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1793
1794           /* Next configured feature, probably ip4-lookup */
1795           if (is_slow_path)
1796             {
1797               if (PREDICT_FALSE (proto1 == ~0))
1798                 {
1799                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1800                                                   thread_index, now, vm, node);
1801                   if (!s1)
1802                     next1 = SNAT_IN2OUT_NEXT_DROP;
1803                   goto trace01;
1804                 }
1805
1806               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1807                 {
1808                   next1 = icmp_in2out_slow_path
1809                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1810                      next1, now, thread_index, &s1);
1811                   goto trace01;
1812                 }
1813             }
1814           else
1815             {
1816               if (is_output_feature)
1817                 {
1818                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1, thread_index)))
1819                     goto trace01;
1820                 }
1821
1822               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1823                 {
1824                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1825                   goto trace01;
1826                 }
1827
1828               if (ip4_is_fragment (ip1))
1829                 {
1830                   next1 = SNAT_IN2OUT_NEXT_REASS;
1831                   goto trace01;
1832                 }
1833             }
1834
1835           key1.addr = ip1->src_address;
1836           key1.port = udp1->src_port;
1837           key1.protocol = proto1;
1838           key1.fib_index = rx_fib_index1;
1839
1840           kv1.key = key1.as_u64;
1841
1842             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1843                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1844             {
1845               if (is_slow_path)
1846                 {
1847                   if (is_output_feature)
1848                     {
1849                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1850                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1851                         goto trace01;
1852                     }
1853                   else
1854                     {
1855                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1856                           ip1, proto1, rx_fib_index1, thread_index)))
1857                         goto trace01;
1858                     }
1859
1860                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1861                                      &s1, node, next1, thread_index);
1862                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1863                     goto trace01;
1864                 }
1865               else
1866                 {
1867                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1868                   goto trace01;
1869                 }
1870             }
1871           else
1872             {
1873               if (PREDICT_FALSE (value1.value == ~0ULL))
1874                 {
1875                   if (is_slow_path)
1876                     {
1877                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1878                                           thread_index, now, vm, node);
1879                       if (!s1 && !sm->forwarding_enabled)
1880                         next1 = SNAT_IN2OUT_NEXT_DROP;
1881                       goto trace01;
1882                     }
1883                   else
1884                     {
1885                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1886                       goto trace01;
1887                     }
1888                 }
1889               else
1890                 {
1891                   s1 = pool_elt_at_index (
1892                     sm->per_thread_data[thread_index].sessions,
1893                     value1.value);
1894                 }
1895             }
1896
1897           b1->flags |= VNET_BUFFER_F_IS_NATED;
1898
1899           old_addr1 = ip1->src_address.as_u32;
1900           ip1->src_address = s1->out2in.addr;
1901           new_addr1 = ip1->src_address.as_u32;
1902           if (!is_output_feature)
1903             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1904
1905           sum1 = ip1->checksum;
1906           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1907                                  ip4_header_t,
1908                                  src_address /* changed member */);
1909           ip1->checksum = ip_csum_fold (sum1);
1910
1911           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1912             {
1913               old_port1 = tcp1->src_port;
1914               tcp1->src_port = s1->out2in.port;
1915               new_port1 = tcp1->src_port;
1916
1917               sum1 = tcp1->checksum;
1918               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1919                                      ip4_header_t,
1920                                      dst_address /* changed member */);
1921               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1922                                      ip4_header_t /* cheat */,
1923                                      length /* changed member */);
1924               tcp1->checksum = ip_csum_fold(sum1);
1925               nat44_set_tcp_session_state (sm, s1, tcp1, thread_index);
1926             }
1927           else
1928             {
1929               old_port1 = udp1->src_port;
1930               udp1->src_port = s1->out2in.port;
1931               udp1->checksum = 0;
1932             }
1933
1934           /* Accounting */
1935           s1->last_heard = now;
1936           s1->total_pkts++;
1937           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1938           /* Per-user LRU list maintenance */
1939           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1940                              s1->per_user_index);
1941           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1942                               s1->per_user_list_head_index,
1943                               s1->per_user_index);
1944         trace01:
1945
1946           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1947                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1948             {
1949               snat_in2out_trace_t *t =
1950                  vlib_add_trace (vm, node, b1, sizeof (*t));
1951               t->sw_if_index = sw_if_index1;
1952               t->next_index = next1;
1953               t->session_index = ~0;
1954               if (s1)
1955                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1956             }
1957
1958           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1959
1960           /* verify speculative enqueues, maybe switch current next frame */
1961           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1962                                            to_next, n_left_to_next,
1963                                            bi0, bi1, next0, next1);
1964         }
1965
1966       while (n_left_from > 0 && n_left_to_next > 0)
1967         {
1968           u32 bi0;
1969           vlib_buffer_t * b0;
1970           u32 next0;
1971           u32 sw_if_index0;
1972           ip4_header_t * ip0;
1973           ip_csum_t sum0;
1974           u32 new_addr0, old_addr0;
1975           u16 old_port0, new_port0;
1976           udp_header_t * udp0;
1977           tcp_header_t * tcp0;
1978           icmp46_header_t * icmp0;
1979           snat_session_key_t key0;
1980           u32 rx_fib_index0;
1981           u32 proto0;
1982           snat_session_t * s0 = 0;
1983           clib_bihash_kv_8_8_t kv0, value0;
1984           u32 iph_offset0 = 0;
1985
1986           /* speculatively enqueue b0 to the current next frame */
1987           bi0 = from[0];
1988           to_next[0] = bi0;
1989           from += 1;
1990           to_next += 1;
1991           n_left_from -= 1;
1992           n_left_to_next -= 1;
1993
1994           b0 = vlib_get_buffer (vm, bi0);
1995           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1996
1997           if (is_output_feature)
1998             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1999
2000           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
2001                  iph_offset0);
2002
2003           udp0 = ip4_next_header (ip0);
2004           tcp0 = (tcp_header_t *) udp0;
2005           icmp0 = (icmp46_header_t *) udp0;
2006
2007           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2008           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
2009                                    sw_if_index0);
2010
2011           if (PREDICT_FALSE(ip0->ttl == 1))
2012             {
2013               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2014               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2015                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2016                                            0);
2017               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2018               goto trace0;
2019             }
2020
2021           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2022
2023           /* Next configured feature, probably ip4-lookup */
2024           if (is_slow_path)
2025             {
2026               if (PREDICT_FALSE (proto0 == ~0))
2027                 {
2028                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2029                                                   thread_index, now, vm, node);
2030                   if (!s0)
2031                     next0 = SNAT_IN2OUT_NEXT_DROP;
2032                   goto trace0;
2033                 }
2034
2035               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2036                 {
2037                   next0 = icmp_in2out_slow_path
2038                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2039                      next0, now, thread_index, &s0);
2040                   goto trace0;
2041                 }
2042             }
2043           else
2044             {
2045                if (is_output_feature)
2046                 {
2047                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
2048                     goto trace0;
2049                 }
2050
2051               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2052                 {
2053                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2054                   goto trace0;
2055                 }
2056
2057               if (ip4_is_fragment (ip0))
2058                 {
2059                   next0 = SNAT_IN2OUT_NEXT_REASS;
2060                   goto trace0;
2061                 }
2062             }
2063
2064           key0.addr = ip0->src_address;
2065           key0.port = udp0->src_port;
2066           key0.protocol = proto0;
2067           key0.fib_index = rx_fib_index0;
2068
2069           kv0.key = key0.as_u64;
2070
2071           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2072                                       &kv0, &value0))
2073             {
2074               if (is_slow_path)
2075                 {
2076                   if (is_output_feature)
2077                     {
2078                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2079                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2080                         goto trace0;
2081                     }
2082                   else
2083                     {
2084                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2085                           ip0, proto0, rx_fib_index0, thread_index)))
2086                         goto trace0;
2087                     }
2088
2089                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2090                                      &s0, node, next0, thread_index);
2091
2092                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2093                     goto trace0;
2094                 }
2095               else
2096                 {
2097                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2098                   goto trace0;
2099                 }
2100             }
2101           else
2102             {
2103               if (PREDICT_FALSE (value0.value == ~0ULL))
2104                 {
2105                   if (is_slow_path)
2106                     {
2107                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2108                                           thread_index, now, vm, node);
2109                       if (!s0 && !sm->forwarding_enabled)
2110                         next0 = SNAT_IN2OUT_NEXT_DROP;
2111                       goto trace0;
2112                     }
2113                   else
2114                     {
2115                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2116                       goto trace0;
2117                     }
2118                 }
2119               else
2120                 {
2121                   s0 = pool_elt_at_index (
2122                     sm->per_thread_data[thread_index].sessions,
2123                     value0.value);
2124                 }
2125             }
2126
2127           b0->flags |= VNET_BUFFER_F_IS_NATED;
2128
2129           old_addr0 = ip0->src_address.as_u32;
2130           ip0->src_address = s0->out2in.addr;
2131           new_addr0 = ip0->src_address.as_u32;
2132           if (!is_output_feature)
2133             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2134
2135           sum0 = ip0->checksum;
2136           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2137                                  ip4_header_t,
2138                                  src_address /* changed member */);
2139           ip0->checksum = ip_csum_fold (sum0);
2140
2141           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2142             {
2143               old_port0 = tcp0->src_port;
2144               tcp0->src_port = s0->out2in.port;
2145               new_port0 = tcp0->src_port;
2146
2147               sum0 = tcp0->checksum;
2148               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2149                                      ip4_header_t,
2150                                      dst_address /* changed member */);
2151               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2152                                      ip4_header_t /* cheat */,
2153                                      length /* changed member */);
2154               tcp0->checksum = ip_csum_fold(sum0);
2155               nat44_set_tcp_session_state (sm, s0, tcp0, thread_index);
2156             }
2157           else
2158             {
2159               old_port0 = udp0->src_port;
2160               udp0->src_port = s0->out2in.port;
2161               udp0->checksum = 0;
2162             }
2163
2164           /* Accounting */
2165           s0->last_heard = now;
2166           s0->total_pkts++;
2167           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2168           /* Per-user LRU list maintenance */
2169           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2170                              s0->per_user_index);
2171           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2172                               s0->per_user_list_head_index,
2173                               s0->per_user_index);
2174
2175         trace0:
2176           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2177                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2178             {
2179               snat_in2out_trace_t *t =
2180                  vlib_add_trace (vm, node, b0, sizeof (*t));
2181               t->is_slow_path = is_slow_path;
2182               t->sw_if_index = sw_if_index0;
2183               t->next_index = next0;
2184                   t->session_index = ~0;
2185               if (s0)
2186                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2187             }
2188
2189           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2190
2191           /* verify speculative enqueue, maybe switch current next frame */
2192           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2193                                            to_next, n_left_to_next,
2194                                            bi0, next0);
2195         }
2196
2197       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2198     }
2199
2200   vlib_node_increment_counter (vm, stats_node_index,
2201                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2202                                pkts_processed);
2203   return frame->n_vectors;
2204 }
2205
2206 static uword
2207 snat_in2out_fast_path_fn (vlib_main_t * vm,
2208                           vlib_node_runtime_t * node,
2209                           vlib_frame_t * frame)
2210 {
2211   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2212 }
2213
2214 VLIB_REGISTER_NODE (snat_in2out_node) = {
2215   .function = snat_in2out_fast_path_fn,
2216   .name = "nat44-in2out",
2217   .vector_size = sizeof (u32),
2218   .format_trace = format_snat_in2out_trace,
2219   .type = VLIB_NODE_TYPE_INTERNAL,
2220
2221   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2222   .error_strings = snat_in2out_error_strings,
2223
2224   .runtime_data_bytes = sizeof (snat_runtime_t),
2225
2226   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2227
2228   /* edit / add dispositions here */
2229   .next_nodes = {
2230     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2231     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2232     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2233     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2234     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2235   },
2236 };
2237
2238 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2239
2240 static uword
2241 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2242                                  vlib_node_runtime_t * node,
2243                                  vlib_frame_t * frame)
2244 {
2245   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2246 }
2247
2248 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2249   .function = snat_in2out_output_fast_path_fn,
2250   .name = "nat44-in2out-output",
2251   .vector_size = sizeof (u32),
2252   .format_trace = format_snat_in2out_trace,
2253   .type = VLIB_NODE_TYPE_INTERNAL,
2254
2255   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2256   .error_strings = snat_in2out_error_strings,
2257
2258   .runtime_data_bytes = sizeof (snat_runtime_t),
2259
2260   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2261
2262   /* edit / add dispositions here */
2263   .next_nodes = {
2264     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2265     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2266     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2267     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2268     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2269   },
2270 };
2271
2272 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2273                               snat_in2out_output_fast_path_fn);
2274
2275 static uword
2276 snat_in2out_slow_path_fn (vlib_main_t * vm,
2277                           vlib_node_runtime_t * node,
2278                           vlib_frame_t * frame)
2279 {
2280   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2281 }
2282
2283 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2284   .function = snat_in2out_slow_path_fn,
2285   .name = "nat44-in2out-slowpath",
2286   .vector_size = sizeof (u32),
2287   .format_trace = format_snat_in2out_trace,
2288   .type = VLIB_NODE_TYPE_INTERNAL,
2289
2290   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2291   .error_strings = snat_in2out_error_strings,
2292
2293   .runtime_data_bytes = sizeof (snat_runtime_t),
2294
2295   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2296
2297   /* edit / add dispositions here */
2298   .next_nodes = {
2299     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2300     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2301     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2302     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2303     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2304   },
2305 };
2306
2307 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2308                               snat_in2out_slow_path_fn);
2309
2310 static uword
2311 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2312                                  vlib_node_runtime_t * node,
2313                                  vlib_frame_t * frame)
2314 {
2315   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2316 }
2317
2318 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2319   .function = snat_in2out_output_slow_path_fn,
2320   .name = "nat44-in2out-output-slowpath",
2321   .vector_size = sizeof (u32),
2322   .format_trace = format_snat_in2out_trace,
2323   .type = VLIB_NODE_TYPE_INTERNAL,
2324
2325   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2326   .error_strings = snat_in2out_error_strings,
2327
2328   .runtime_data_bytes = sizeof (snat_runtime_t),
2329
2330   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2331
2332   /* edit / add dispositions here */
2333   .next_nodes = {
2334     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2335     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2336     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2337     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2338     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2339   },
2340 };
2341
2342 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2343                               snat_in2out_output_slow_path_fn);
2344
2345 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2346
2347 static uword
2348 nat44_hairpinning_fn (vlib_main_t * vm,
2349                       vlib_node_runtime_t * node,
2350                       vlib_frame_t * frame)
2351 {
2352   u32 n_left_from, * from, * to_next;
2353   snat_in2out_next_t next_index;
2354   u32 pkts_processed = 0;
2355   snat_main_t * sm = &snat_main;
2356   vnet_feature_main_t *fm = &feature_main;
2357   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2358   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2359
2360   from = vlib_frame_vector_args (frame);
2361   n_left_from = frame->n_vectors;
2362   next_index = node->cached_next_index;
2363
2364   while (n_left_from > 0)
2365     {
2366       u32 n_left_to_next;
2367
2368       vlib_get_next_frame (vm, node, next_index,
2369                            to_next, n_left_to_next);
2370
2371       while (n_left_from > 0 && n_left_to_next > 0)
2372         {
2373           u32 bi0;
2374           vlib_buffer_t * b0;
2375           u32 next0;
2376           ip4_header_t * ip0;
2377           u32 proto0;
2378           udp_header_t * udp0;
2379           tcp_header_t * tcp0;
2380
2381           /* speculatively enqueue b0 to the current next frame */
2382           bi0 = from[0];
2383           to_next[0] = bi0;
2384           from += 1;
2385           to_next += 1;
2386           n_left_from -= 1;
2387           n_left_to_next -= 1;
2388
2389           b0 = vlib_get_buffer (vm, bi0);
2390           ip0 = vlib_buffer_get_current (b0);
2391           udp0 = ip4_next_header (ip0);
2392           tcp0 = (tcp_header_t *) udp0;
2393
2394           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2395
2396           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2397                                 &next0, 0);
2398
2399           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2400             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2401
2402           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2403
2404           /* verify speculative enqueue, maybe switch current next frame */
2405           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2406                                            to_next, n_left_to_next,
2407                                            bi0, next0);
2408          }
2409
2410       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2411     }
2412
2413   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2414                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2415                                pkts_processed);
2416   return frame->n_vectors;
2417 }
2418
2419 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2420   .function = nat44_hairpinning_fn,
2421   .name = "nat44-hairpinning",
2422   .vector_size = sizeof (u32),
2423   .type = VLIB_NODE_TYPE_INTERNAL,
2424   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2425   .error_strings = snat_in2out_error_strings,
2426   .n_next_nodes = 2,
2427   .next_nodes = {
2428     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2429     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2430   },
2431 };
2432
2433 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2434                               nat44_hairpinning_fn);
2435
2436 static inline void
2437 nat44_reass_hairpinning (snat_main_t *sm,
2438                          vlib_buffer_t * b0,
2439                          ip4_header_t * ip0,
2440                          u16 sport,
2441                          u16 dport,
2442                          u32 proto0)
2443 {
2444   snat_session_key_t key0, sm0;
2445   snat_session_t * s0;
2446   clib_bihash_kv_8_8_t kv0, value0;
2447   ip_csum_t sum0;
2448   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2449   u16 new_dst_port0, old_dst_port0;
2450   udp_header_t * udp0;
2451   tcp_header_t * tcp0;
2452
2453   key0.addr = ip0->dst_address;
2454   key0.port = dport;
2455   key0.protocol = proto0;
2456   key0.fib_index = sm->outside_fib_index;
2457   kv0.key = key0.as_u64;
2458
2459   udp0 = ip4_next_header (ip0);
2460
2461   /* Check if destination is static mappings */
2462   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2463     {
2464       new_dst_addr0 = sm0.addr.as_u32;
2465       new_dst_port0 = sm0.port;
2466       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2467     }
2468   /* or active sessions */
2469   else
2470     {
2471       if (sm->num_workers > 1)
2472         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2473       else
2474         ti = sm->num_workers;
2475
2476       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2477         {
2478           si = value0.value;
2479           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2480           new_dst_addr0 = s0->in2out.addr.as_u32;
2481           new_dst_port0 = s0->in2out.port;
2482           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2483         }
2484     }
2485
2486   /* Destination is behind the same NAT, use internal address and port */
2487   if (new_dst_addr0)
2488     {
2489       old_dst_addr0 = ip0->dst_address.as_u32;
2490       ip0->dst_address.as_u32 = new_dst_addr0;
2491       sum0 = ip0->checksum;
2492       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2493                              ip4_header_t, dst_address);
2494       ip0->checksum = ip_csum_fold (sum0);
2495
2496       old_dst_port0 = dport;
2497       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2498                        ip4_is_first_fragment (ip0)))
2499         {
2500           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2501             {
2502               tcp0 = ip4_next_header (ip0);
2503               tcp0->dst = new_dst_port0;
2504               sum0 = tcp0->checksum;
2505               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2506                                      ip4_header_t, dst_address);
2507               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2508                                      ip4_header_t /* cheat */, length);
2509               tcp0->checksum = ip_csum_fold(sum0);
2510             }
2511           else
2512             {
2513               udp0->dst_port = new_dst_port0;
2514               udp0->checksum = 0;
2515             }
2516         }
2517       else
2518         {
2519           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2520             {
2521               tcp0 = ip4_next_header (ip0);
2522               sum0 = tcp0->checksum;
2523               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2524                                      ip4_header_t, dst_address);
2525               tcp0->checksum = ip_csum_fold(sum0);
2526             }
2527         }
2528     }
2529 }
2530
2531 static uword
2532 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2533                             vlib_node_runtime_t * node,
2534                             vlib_frame_t * frame)
2535 {
2536   u32 n_left_from, *from, *to_next;
2537   snat_in2out_next_t next_index;
2538   u32 pkts_processed = 0;
2539   snat_main_t *sm = &snat_main;
2540   f64 now = vlib_time_now (vm);
2541   u32 thread_index = vlib_get_thread_index ();
2542   snat_main_per_thread_data_t *per_thread_data =
2543     &sm->per_thread_data[thread_index];
2544   u32 *fragments_to_drop = 0;
2545   u32 *fragments_to_loopback = 0;
2546
2547   from = vlib_frame_vector_args (frame);
2548   n_left_from = frame->n_vectors;
2549   next_index = node->cached_next_index;
2550
2551   while (n_left_from > 0)
2552     {
2553       u32 n_left_to_next;
2554
2555       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2556
2557       while (n_left_from > 0 && n_left_to_next > 0)
2558        {
2559           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2560           vlib_buffer_t *b0;
2561           u32 next0;
2562           u8 cached0 = 0;
2563           ip4_header_t *ip0;
2564           nat_reass_ip4_t *reass0;
2565           udp_header_t * udp0;
2566           tcp_header_t * tcp0;
2567           snat_session_key_t key0;
2568           clib_bihash_kv_8_8_t kv0, value0;
2569           snat_session_t * s0 = 0;
2570           u16 old_port0, new_port0;
2571           ip_csum_t sum0;
2572
2573           /* speculatively enqueue b0 to the current next frame */
2574           bi0 = from[0];
2575           to_next[0] = bi0;
2576           from += 1;
2577           to_next += 1;
2578           n_left_from -= 1;
2579           n_left_to_next -= 1;
2580
2581           b0 = vlib_get_buffer (vm, bi0);
2582           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2583
2584           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2585           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2586                                                                sw_if_index0);
2587
2588           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2589             {
2590               next0 = SNAT_IN2OUT_NEXT_DROP;
2591               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2592               goto trace0;
2593             }
2594
2595           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2596           udp0 = ip4_next_header (ip0);
2597           tcp0 = (tcp_header_t *) udp0;
2598           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2599
2600           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2601                                                  ip0->dst_address,
2602                                                  ip0->fragment_id,
2603                                                  ip0->protocol,
2604                                                  1,
2605                                                  &fragments_to_drop);
2606
2607           if (PREDICT_FALSE (!reass0))
2608             {
2609               next0 = SNAT_IN2OUT_NEXT_DROP;
2610               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2611               goto trace0;
2612             }
2613
2614           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2615             {
2616               key0.addr = ip0->src_address;
2617               key0.port = udp0->src_port;
2618               key0.protocol = proto0;
2619               key0.fib_index = rx_fib_index0;
2620               kv0.key = key0.as_u64;
2621
2622               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2623                 {
2624                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2625                       ip0, proto0, rx_fib_index0, thread_index)))
2626                     goto trace0;
2627
2628                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2629                                      &s0, node, next0, thread_index);
2630
2631                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2632                     goto trace0;
2633
2634                   reass0->sess_index = s0 - per_thread_data->sessions;
2635                 }
2636               else
2637                 {
2638                   s0 = pool_elt_at_index (per_thread_data->sessions,
2639                                           value0.value);
2640                   reass0->sess_index = value0.value;
2641                 }
2642               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2643             }
2644           else
2645             {
2646               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2647                 {
2648                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2649                     {
2650                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2651                       next0 = SNAT_IN2OUT_NEXT_DROP;
2652                       goto trace0;
2653                     }
2654                   cached0 = 1;
2655                   goto trace0;
2656                 }
2657               s0 = pool_elt_at_index (per_thread_data->sessions,
2658                                       reass0->sess_index);
2659             }
2660
2661           old_addr0 = ip0->src_address.as_u32;
2662           ip0->src_address = s0->out2in.addr;
2663           new_addr0 = ip0->src_address.as_u32;
2664           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2665
2666           sum0 = ip0->checksum;
2667           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2668                                  ip4_header_t,
2669                                  src_address /* changed member */);
2670           ip0->checksum = ip_csum_fold (sum0);
2671
2672           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2673             {
2674               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2675                 {
2676                   old_port0 = tcp0->src_port;
2677                   tcp0->src_port = s0->out2in.port;
2678                   new_port0 = tcp0->src_port;
2679
2680                   sum0 = tcp0->checksum;
2681                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2682                                          ip4_header_t,
2683                                          dst_address /* changed member */);
2684                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2685                                          ip4_header_t /* cheat */,
2686                                          length /* changed member */);
2687                   tcp0->checksum = ip_csum_fold(sum0);
2688                   nat44_set_tcp_session_state (sm, s0, tcp0, thread_index);
2689                 }
2690               else
2691                 {
2692                   old_port0 = udp0->src_port;
2693                   udp0->src_port = s0->out2in.port;
2694                   udp0->checksum = 0;
2695                 }
2696             }
2697
2698           /* Hairpinning */
2699           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2700                                    s0->ext_host_port, proto0);
2701
2702           /* Accounting */
2703           s0->last_heard = now;
2704           s0->total_pkts++;
2705           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2706           /* Per-user LRU list maintenance */
2707           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2708                              s0->per_user_index);
2709           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2710                               s0->per_user_list_head_index,
2711                               s0->per_user_index);
2712
2713         trace0:
2714           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2715                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2716             {
2717               nat44_in2out_reass_trace_t *t =
2718                  vlib_add_trace (vm, node, b0, sizeof (*t));
2719               t->cached = cached0;
2720               t->sw_if_index = sw_if_index0;
2721               t->next_index = next0;
2722             }
2723
2724           if (cached0)
2725             {
2726               n_left_to_next++;
2727               to_next--;
2728             }
2729           else
2730             {
2731               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2732
2733               /* verify speculative enqueue, maybe switch current next frame */
2734               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2735                                                to_next, n_left_to_next,
2736                                                bi0, next0);
2737             }
2738
2739           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2740             {
2741               from = vlib_frame_vector_args (frame);
2742               u32 len = vec_len (fragments_to_loopback);
2743               if (len <= VLIB_FRAME_SIZE)
2744                 {
2745                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2746                   n_left_from = len;
2747                   vec_reset_length (fragments_to_loopback);
2748                 }
2749               else
2750                 {
2751                   clib_memcpy (from,
2752                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2753                                sizeof (u32) * VLIB_FRAME_SIZE);
2754                   n_left_from = VLIB_FRAME_SIZE;
2755                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2756                 }
2757             }
2758        }
2759
2760       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2761     }
2762
2763   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2764                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2765                                pkts_processed);
2766
2767   nat_send_all_to_node (vm, fragments_to_drop, node,
2768                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2769                         SNAT_IN2OUT_NEXT_DROP);
2770
2771   vec_free (fragments_to_drop);
2772   vec_free (fragments_to_loopback);
2773   return frame->n_vectors;
2774 }
2775
2776 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2777   .function = nat44_in2out_reass_node_fn,
2778   .name = "nat44-in2out-reass",
2779   .vector_size = sizeof (u32),
2780   .format_trace = format_nat44_in2out_reass_trace,
2781   .type = VLIB_NODE_TYPE_INTERNAL,
2782
2783   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2784   .error_strings = snat_in2out_error_strings,
2785
2786   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2787   .next_nodes = {
2788     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2789     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2790     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2791     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2792     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2793   },
2794 };
2795
2796 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2797                               nat44_in2out_reass_node_fn);
2798
2799 /**************************/
2800 /*** deterministic mode ***/
2801 /**************************/
2802 static uword
2803 snat_det_in2out_node_fn (vlib_main_t * vm,
2804                          vlib_node_runtime_t * node,
2805                          vlib_frame_t * frame)
2806 {
2807   u32 n_left_from, * from, * to_next;
2808   snat_in2out_next_t next_index;
2809   u32 pkts_processed = 0;
2810   snat_main_t * sm = &snat_main;
2811   u32 now = (u32) vlib_time_now (vm);
2812   u32 thread_index = vlib_get_thread_index ();
2813
2814   from = vlib_frame_vector_args (frame);
2815   n_left_from = frame->n_vectors;
2816   next_index = node->cached_next_index;
2817
2818   while (n_left_from > 0)
2819     {
2820       u32 n_left_to_next;
2821
2822       vlib_get_next_frame (vm, node, next_index,
2823                            to_next, n_left_to_next);
2824
2825       while (n_left_from >= 4 && n_left_to_next >= 2)
2826         {
2827           u32 bi0, bi1;
2828           vlib_buffer_t * b0, * b1;
2829           u32 next0, next1;
2830           u32 sw_if_index0, sw_if_index1;
2831           ip4_header_t * ip0, * ip1;
2832           ip_csum_t sum0, sum1;
2833           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2834           u16 old_port0, new_port0, lo_port0, i0;
2835           u16 old_port1, new_port1, lo_port1, i1;
2836           udp_header_t * udp0, * udp1;
2837           tcp_header_t * tcp0, * tcp1;
2838           u32 proto0, proto1;
2839           snat_det_out_key_t key0, key1;
2840           snat_det_map_t * dm0, * dm1;
2841           snat_det_session_t * ses0 = 0, * ses1 = 0;
2842           u32 rx_fib_index0, rx_fib_index1;
2843           icmp46_header_t * icmp0, * icmp1;
2844
2845           /* Prefetch next iteration. */
2846           {
2847             vlib_buffer_t * p2, * p3;
2848
2849             p2 = vlib_get_buffer (vm, from[2]);
2850             p3 = vlib_get_buffer (vm, from[3]);
2851
2852             vlib_prefetch_buffer_header (p2, LOAD);
2853             vlib_prefetch_buffer_header (p3, LOAD);
2854
2855             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2856             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2857           }
2858
2859           /* speculatively enqueue b0 and b1 to the current next frame */
2860           to_next[0] = bi0 = from[0];
2861           to_next[1] = bi1 = from[1];
2862           from += 2;
2863           to_next += 2;
2864           n_left_from -= 2;
2865           n_left_to_next -= 2;
2866
2867           b0 = vlib_get_buffer (vm, bi0);
2868           b1 = vlib_get_buffer (vm, bi1);
2869
2870           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2871           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2872
2873           ip0 = vlib_buffer_get_current (b0);
2874           udp0 = ip4_next_header (ip0);
2875           tcp0 = (tcp_header_t *) udp0;
2876
2877           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2878
2879           if (PREDICT_FALSE(ip0->ttl == 1))
2880             {
2881               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2882               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2883                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2884                                            0);
2885               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2886               goto trace0;
2887             }
2888
2889           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2890
2891           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2892             {
2893               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2894               icmp0 = (icmp46_header_t *) udp0;
2895
2896               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2897                                   rx_fib_index0, node, next0, thread_index,
2898                                   &ses0, &dm0);
2899               goto trace0;
2900             }
2901
2902           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2903           if (PREDICT_FALSE(!dm0))
2904             {
2905               clib_warning("no match for internal host %U",
2906                            format_ip4_address, &ip0->src_address);
2907               next0 = SNAT_IN2OUT_NEXT_DROP;
2908               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2909               goto trace0;
2910             }
2911
2912           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2913
2914           key0.ext_host_addr = ip0->dst_address;
2915           key0.ext_host_port = tcp0->dst;
2916
2917           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2918           if (PREDICT_FALSE(!ses0))
2919             {
2920               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2921                 {
2922                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2923                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2924
2925                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2926                     continue;
2927
2928                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2929                   break;
2930                 }
2931               if (PREDICT_FALSE(!ses0))
2932                 {
2933                   /* too many sessions for user, send ICMP error packet */
2934
2935                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2936                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2937                                                ICMP4_destination_unreachable_destination_unreachable_host,
2938                                                0);
2939                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2940                   goto trace0;
2941                 }
2942             }
2943
2944           new_port0 = ses0->out.out_port;
2945
2946           old_addr0.as_u32 = ip0->src_address.as_u32;
2947           ip0->src_address.as_u32 = new_addr0.as_u32;
2948           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2949
2950           sum0 = ip0->checksum;
2951           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2952                                  ip4_header_t,
2953                                  src_address /* changed member */);
2954           ip0->checksum = ip_csum_fold (sum0);
2955
2956           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2957             {
2958               if (tcp0->flags & TCP_FLAG_SYN)
2959                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2960               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2961                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2962               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2963                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2964               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2965                 snat_det_ses_close(dm0, ses0);
2966               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2967                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2968               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2969                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2970
2971               old_port0 = tcp0->src;
2972               tcp0->src = new_port0;
2973
2974               sum0 = tcp0->checksum;
2975               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2976                                      ip4_header_t,
2977                                      dst_address /* changed member */);
2978               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2979                                      ip4_header_t /* cheat */,
2980                                      length /* changed member */);
2981               tcp0->checksum = ip_csum_fold(sum0);
2982             }
2983           else
2984             {
2985               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2986               old_port0 = udp0->src_port;
2987               udp0->src_port = new_port0;
2988               udp0->checksum = 0;
2989             }
2990
2991           switch(ses0->state)
2992             {
2993             case SNAT_SESSION_UDP_ACTIVE:
2994                 ses0->expire = now + sm->udp_timeout;
2995                 break;
2996             case SNAT_SESSION_TCP_SYN_SENT:
2997             case SNAT_SESSION_TCP_FIN_WAIT:
2998             case SNAT_SESSION_TCP_CLOSE_WAIT:
2999             case SNAT_SESSION_TCP_LAST_ACK:
3000                 ses0->expire = now + sm->tcp_transitory_timeout;
3001                 break;
3002             case SNAT_SESSION_TCP_ESTABLISHED:
3003                 ses0->expire = now + sm->tcp_established_timeout;
3004                 break;
3005             }
3006
3007         trace0:
3008           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3009                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3010             {
3011               snat_in2out_trace_t *t =
3012                  vlib_add_trace (vm, node, b0, sizeof (*t));
3013               t->is_slow_path = 0;
3014               t->sw_if_index = sw_if_index0;
3015               t->next_index = next0;
3016               t->session_index = ~0;
3017               if (ses0)
3018                 t->session_index = ses0 - dm0->sessions;
3019             }
3020
3021           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3022
3023           ip1 = vlib_buffer_get_current (b1);
3024           udp1 = ip4_next_header (ip1);
3025           tcp1 = (tcp_header_t *) udp1;
3026
3027           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3028
3029           if (PREDICT_FALSE(ip1->ttl == 1))
3030             {
3031               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3032               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3033                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3034                                            0);
3035               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3036               goto trace1;
3037             }
3038
3039           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3040
3041           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3042             {
3043               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3044               icmp1 = (icmp46_header_t *) udp1;
3045
3046               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3047                                   rx_fib_index1, node, next1, thread_index,
3048                                   &ses1, &dm1);
3049               goto trace1;
3050             }
3051
3052           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3053           if (PREDICT_FALSE(!dm1))
3054             {
3055               clib_warning("no match for internal host %U",
3056                            format_ip4_address, &ip0->src_address);
3057               next1 = SNAT_IN2OUT_NEXT_DROP;
3058               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3059               goto trace1;
3060             }
3061
3062           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3063
3064           key1.ext_host_addr = ip1->dst_address;
3065           key1.ext_host_port = tcp1->dst;
3066
3067           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3068           if (PREDICT_FALSE(!ses1))
3069             {
3070               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3071                 {
3072                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3073                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3074
3075                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3076                     continue;
3077
3078                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3079                   break;
3080                 }
3081               if (PREDICT_FALSE(!ses1))
3082                 {
3083                   /* too many sessions for user, send ICMP error packet */
3084
3085                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3086                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3087                                                ICMP4_destination_unreachable_destination_unreachable_host,
3088                                                0);
3089                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3090                   goto trace1;
3091                 }
3092             }
3093
3094           new_port1 = ses1->out.out_port;
3095
3096           old_addr1.as_u32 = ip1->src_address.as_u32;
3097           ip1->src_address.as_u32 = new_addr1.as_u32;
3098           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3099
3100           sum1 = ip1->checksum;
3101           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3102                                  ip4_header_t,
3103                                  src_address /* changed member */);
3104           ip1->checksum = ip_csum_fold (sum1);
3105
3106           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3107             {
3108               if (tcp1->flags & TCP_FLAG_SYN)
3109                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3110               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3111                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3112               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3113                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3114               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3115                 snat_det_ses_close(dm1, ses1);
3116               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3117                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3118               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3119                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3120
3121               old_port1 = tcp1->src;
3122               tcp1->src = new_port1;
3123
3124               sum1 = tcp1->checksum;
3125               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3126                                      ip4_header_t,
3127                                      dst_address /* changed member */);
3128               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3129                                      ip4_header_t /* cheat */,
3130                                      length /* changed member */);
3131               tcp1->checksum = ip_csum_fold(sum1);
3132             }
3133           else
3134             {
3135               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3136               old_port1 = udp1->src_port;
3137               udp1->src_port = new_port1;
3138               udp1->checksum = 0;
3139             }
3140
3141           switch(ses1->state)
3142             {
3143             case SNAT_SESSION_UDP_ACTIVE:
3144                 ses1->expire = now + sm->udp_timeout;
3145                 break;
3146             case SNAT_SESSION_TCP_SYN_SENT:
3147             case SNAT_SESSION_TCP_FIN_WAIT:
3148             case SNAT_SESSION_TCP_CLOSE_WAIT:
3149             case SNAT_SESSION_TCP_LAST_ACK:
3150                 ses1->expire = now + sm->tcp_transitory_timeout;
3151                 break;
3152             case SNAT_SESSION_TCP_ESTABLISHED:
3153                 ses1->expire = now + sm->tcp_established_timeout;
3154                 break;
3155             }
3156
3157         trace1:
3158           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3159                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3160             {
3161               snat_in2out_trace_t *t =
3162                  vlib_add_trace (vm, node, b1, sizeof (*t));
3163               t->is_slow_path = 0;
3164               t->sw_if_index = sw_if_index1;
3165               t->next_index = next1;
3166               t->session_index = ~0;
3167               if (ses1)
3168                 t->session_index = ses1 - dm1->sessions;
3169             }
3170
3171           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3172
3173           /* verify speculative enqueues, maybe switch current next frame */
3174           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3175                                            to_next, n_left_to_next,
3176                                            bi0, bi1, next0, next1);
3177          }
3178
3179       while (n_left_from > 0 && n_left_to_next > 0)
3180         {
3181           u32 bi0;
3182           vlib_buffer_t * b0;
3183           u32 next0;
3184           u32 sw_if_index0;
3185           ip4_header_t * ip0;
3186           ip_csum_t sum0;
3187           ip4_address_t new_addr0, old_addr0;
3188           u16 old_port0, new_port0, lo_port0, i0;
3189           udp_header_t * udp0;
3190           tcp_header_t * tcp0;
3191           u32 proto0;
3192           snat_det_out_key_t key0;
3193           snat_det_map_t * dm0;
3194           snat_det_session_t * ses0 = 0;
3195           u32 rx_fib_index0;
3196           icmp46_header_t * icmp0;
3197
3198           /* speculatively enqueue b0 to the current next frame */
3199           bi0 = from[0];
3200           to_next[0] = bi0;
3201           from += 1;
3202           to_next += 1;
3203           n_left_from -= 1;
3204           n_left_to_next -= 1;
3205
3206           b0 = vlib_get_buffer (vm, bi0);
3207           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3208
3209           ip0 = vlib_buffer_get_current (b0);
3210           udp0 = ip4_next_header (ip0);
3211           tcp0 = (tcp_header_t *) udp0;
3212
3213           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3214
3215           if (PREDICT_FALSE(ip0->ttl == 1))
3216             {
3217               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3218               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3219                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3220                                            0);
3221               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3222               goto trace00;
3223             }
3224
3225           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3226
3227           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3228             {
3229               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3230               icmp0 = (icmp46_header_t *) udp0;
3231
3232               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3233                                   rx_fib_index0, node, next0, thread_index,
3234                                   &ses0, &dm0);
3235               goto trace00;
3236             }
3237
3238           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3239           if (PREDICT_FALSE(!dm0))
3240             {
3241               clib_warning("no match for internal host %U",
3242                            format_ip4_address, &ip0->src_address);
3243               next0 = SNAT_IN2OUT_NEXT_DROP;
3244               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3245               goto trace00;
3246             }
3247
3248           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3249
3250           key0.ext_host_addr = ip0->dst_address;
3251           key0.ext_host_port = tcp0->dst;
3252
3253           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3254           if (PREDICT_FALSE(!ses0))
3255             {
3256               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3257                 {
3258                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3259                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3260
3261                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3262                     continue;
3263
3264                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3265                   break;
3266                 }
3267               if (PREDICT_FALSE(!ses0))
3268                 {
3269                   /* too many sessions for user, send ICMP error packet */
3270
3271                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3272                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3273                                                ICMP4_destination_unreachable_destination_unreachable_host,
3274                                                0);
3275                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3276                   goto trace00;
3277                 }
3278             }
3279
3280           new_port0 = ses0->out.out_port;
3281
3282           old_addr0.as_u32 = ip0->src_address.as_u32;
3283           ip0->src_address.as_u32 = new_addr0.as_u32;
3284           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3285
3286           sum0 = ip0->checksum;
3287           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3288                                  ip4_header_t,
3289                                  src_address /* changed member */);
3290           ip0->checksum = ip_csum_fold (sum0);
3291
3292           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3293             {
3294               if (tcp0->flags & TCP_FLAG_SYN)
3295                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3296               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3297                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3298               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3299                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3300               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3301                 snat_det_ses_close(dm0, ses0);
3302               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3303                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3304               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3305                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3306
3307               old_port0 = tcp0->src;
3308               tcp0->src = new_port0;
3309
3310               sum0 = tcp0->checksum;
3311               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3312                                      ip4_header_t,
3313                                      dst_address /* changed member */);
3314               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3315                                      ip4_header_t /* cheat */,
3316                                      length /* changed member */);
3317               tcp0->checksum = ip_csum_fold(sum0);
3318             }
3319           else
3320             {
3321               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3322               old_port0 = udp0->src_port;
3323               udp0->src_port = new_port0;
3324               udp0->checksum = 0;
3325             }
3326
3327           switch(ses0->state)
3328             {
3329             case SNAT_SESSION_UDP_ACTIVE:
3330                 ses0->expire = now + sm->udp_timeout;
3331                 break;
3332             case SNAT_SESSION_TCP_SYN_SENT:
3333             case SNAT_SESSION_TCP_FIN_WAIT:
3334             case SNAT_SESSION_TCP_CLOSE_WAIT:
3335             case SNAT_SESSION_TCP_LAST_ACK:
3336                 ses0->expire = now + sm->tcp_transitory_timeout;
3337                 break;
3338             case SNAT_SESSION_TCP_ESTABLISHED:
3339                 ses0->expire = now + sm->tcp_established_timeout;
3340                 break;
3341             }
3342
3343         trace00:
3344           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3345                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3346             {
3347               snat_in2out_trace_t *t =
3348                  vlib_add_trace (vm, node, b0, sizeof (*t));
3349               t->is_slow_path = 0;
3350               t->sw_if_index = sw_if_index0;
3351               t->next_index = next0;
3352               t->session_index = ~0;
3353               if (ses0)
3354                 t->session_index = ses0 - dm0->sessions;
3355             }
3356
3357           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3358
3359           /* verify speculative enqueue, maybe switch current next frame */
3360           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3361                                            to_next, n_left_to_next,
3362                                            bi0, next0);
3363         }
3364
3365       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3366     }
3367
3368   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3369                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3370                                pkts_processed);
3371   return frame->n_vectors;
3372 }
3373
3374 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3375   .function = snat_det_in2out_node_fn,
3376   .name = "nat44-det-in2out",
3377   .vector_size = sizeof (u32),
3378   .format_trace = format_snat_in2out_trace,
3379   .type = VLIB_NODE_TYPE_INTERNAL,
3380
3381   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3382   .error_strings = snat_in2out_error_strings,
3383
3384   .runtime_data_bytes = sizeof (snat_runtime_t),
3385
3386   .n_next_nodes = 3,
3387
3388   /* edit / add dispositions here */
3389   .next_nodes = {
3390     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3391     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3392     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3393   },
3394 };
3395
3396 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3397
3398 /**
3399  * Get address and port values to be used for ICMP packet translation
3400  * and create session if needed
3401  *
3402  * @param[in,out] sm             NAT main
3403  * @param[in,out] node           NAT node runtime
3404  * @param[in] thread_index       thread index
3405  * @param[in,out] b0             buffer containing packet to be translated
3406  * @param[out] p_proto           protocol used for matching
3407  * @param[out] p_value           address and port after NAT translation
3408  * @param[out] p_dont_translate  if packet should not be translated
3409  * @param d                      optional parameter
3410  * @param e                      optional parameter
3411  */
3412 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3413                           u32 thread_index, vlib_buffer_t *b0,
3414                           ip4_header_t *ip0, u8 *p_proto,
3415                           snat_session_key_t *p_value,
3416                           u8 *p_dont_translate, void *d, void *e)
3417 {
3418   icmp46_header_t *icmp0;
3419   u32 sw_if_index0;
3420   u32 rx_fib_index0;
3421   u8 protocol;
3422   snat_det_out_key_t key0;
3423   u8 dont_translate = 0;
3424   u32 next0 = ~0;
3425   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3426   ip4_header_t *inner_ip0;
3427   void *l4_header = 0;
3428   icmp46_header_t *inner_icmp0;
3429   snat_det_map_t * dm0 = 0;
3430   ip4_address_t new_addr0;
3431   u16 lo_port0, i0;
3432   snat_det_session_t * ses0 = 0;
3433   ip4_address_t in_addr;
3434   u16 in_port;
3435
3436   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3437   echo0 = (icmp_echo_header_t *)(icmp0+1);
3438   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3439   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3440
3441   if (!icmp_is_error_message (icmp0))
3442     {
3443       protocol = SNAT_PROTOCOL_ICMP;
3444       in_addr = ip0->src_address;
3445       in_port = echo0->identifier;
3446     }
3447   else
3448     {
3449       inner_ip0 = (ip4_header_t *)(echo0+1);
3450       l4_header = ip4_next_header (inner_ip0);
3451       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3452       in_addr = inner_ip0->dst_address;
3453       switch (protocol)
3454         {
3455         case SNAT_PROTOCOL_ICMP:
3456           inner_icmp0 = (icmp46_header_t*)l4_header;
3457           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3458           in_port = inner_echo0->identifier;
3459           break;
3460         case SNAT_PROTOCOL_UDP:
3461         case SNAT_PROTOCOL_TCP:
3462           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3463           break;
3464         default:
3465           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3466           next0 = SNAT_IN2OUT_NEXT_DROP;
3467           goto out;
3468         }
3469     }
3470
3471   dm0 = snat_det_map_by_user(sm, &in_addr);
3472   if (PREDICT_FALSE(!dm0))
3473     {
3474       clib_warning("no match for internal host %U",
3475                    format_ip4_address, &in_addr);
3476       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3477           IP_PROTOCOL_ICMP, rx_fib_index0)))
3478         {
3479           dont_translate = 1;
3480           goto out;
3481         }
3482       next0 = SNAT_IN2OUT_NEXT_DROP;
3483       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3484       goto out;
3485     }
3486
3487   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3488
3489   key0.ext_host_addr = ip0->dst_address;
3490   key0.ext_host_port = 0;
3491
3492   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3493   if (PREDICT_FALSE(!ses0))
3494     {
3495       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3496           IP_PROTOCOL_ICMP, rx_fib_index0)))
3497         {
3498           dont_translate = 1;
3499           goto out;
3500         }
3501       if (icmp0->type != ICMP4_echo_request)
3502         {
3503           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3504           next0 = SNAT_IN2OUT_NEXT_DROP;
3505           goto out;
3506         }
3507       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3508         {
3509           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3510             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3511
3512           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3513             continue;
3514
3515           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3516           break;
3517         }
3518       if (PREDICT_FALSE(!ses0))
3519         {
3520           next0 = SNAT_IN2OUT_NEXT_DROP;
3521           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3522           goto out;
3523         }
3524     }
3525
3526   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3527                     !icmp_is_error_message (icmp0)))
3528     {
3529       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3530       next0 = SNAT_IN2OUT_NEXT_DROP;
3531       goto out;
3532     }
3533
3534   u32 now = (u32) vlib_time_now (sm->vlib_main);
3535
3536   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3537   ses0->expire = now + sm->icmp_timeout;
3538
3539 out:
3540   *p_proto = protocol;
3541   if (ses0)
3542     {
3543       p_value->addr = new_addr0;
3544       p_value->fib_index = sm->outside_fib_index;
3545       p_value->port = ses0->out.out_port;
3546     }
3547   *p_dont_translate = dont_translate;
3548   if (d)
3549     *(snat_det_session_t**)d = ses0;
3550   if (e)
3551     *(snat_det_map_t**)e = dm0;
3552   return next0;
3553 }
3554
3555 /**********************/
3556 /*** worker handoff ***/
3557 /**********************/
3558 static inline uword
3559 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3560                                       vlib_node_runtime_t * node,
3561                                       vlib_frame_t * frame,
3562                                       u8 is_output)
3563 {
3564   snat_main_t *sm = &snat_main;
3565   vlib_thread_main_t *tm = vlib_get_thread_main ();
3566   u32 n_left_from, *from, *to_next = 0;
3567   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3568   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3569     = 0;
3570   vlib_frame_queue_elt_t *hf = 0;
3571   vlib_frame_t *f = 0;
3572   int i;
3573   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3574   u32 next_worker_index = 0;
3575   u32 current_worker_index = ~0;
3576   u32 thread_index = vlib_get_thread_index ();
3577   u32 fq_index;
3578   u32 to_node_index;
3579
3580   ASSERT (vec_len (sm->workers));
3581
3582   if (is_output)
3583     {
3584       fq_index = sm->fq_in2out_output_index;
3585       to_node_index = sm->in2out_output_node_index;
3586     }
3587   else
3588     {
3589       fq_index = sm->fq_in2out_index;
3590       to_node_index = sm->in2out_node_index;
3591     }
3592
3593   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3594     {
3595       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3596
3597       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3598                                sm->first_worker_index + sm->num_workers - 1,
3599                                (vlib_frame_queue_t *) (~0));
3600     }
3601
3602   from = vlib_frame_vector_args (frame);
3603   n_left_from = frame->n_vectors;
3604
3605   while (n_left_from > 0)
3606     {
3607       u32 bi0;
3608       vlib_buffer_t *b0;
3609       u32 sw_if_index0;
3610       u32 rx_fib_index0;
3611       ip4_header_t * ip0;
3612       u8 do_handoff;
3613
3614       bi0 = from[0];
3615       from += 1;
3616       n_left_from -= 1;
3617
3618       b0 = vlib_get_buffer (vm, bi0);
3619
3620       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3621       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3622
3623       ip0 = vlib_buffer_get_current (b0);
3624
3625       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3626
3627       if (PREDICT_FALSE (next_worker_index != thread_index))
3628         {
3629           do_handoff = 1;
3630
3631           if (next_worker_index != current_worker_index)
3632             {
3633               if (hf)
3634                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3635
3636               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3637                                                       next_worker_index,
3638                                                       handoff_queue_elt_by_worker_index);
3639
3640               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3641               to_next_worker = &hf->buffer_index[hf->n_vectors];
3642               current_worker_index = next_worker_index;
3643             }
3644
3645           /* enqueue to correct worker thread */
3646           to_next_worker[0] = bi0;
3647           to_next_worker++;
3648           n_left_to_next_worker--;
3649
3650           if (n_left_to_next_worker == 0)
3651             {
3652               hf->n_vectors = VLIB_FRAME_SIZE;
3653               vlib_put_frame_queue_elt (hf);
3654               current_worker_index = ~0;
3655               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3656               hf = 0;
3657             }
3658         }
3659       else
3660         {
3661           do_handoff = 0;
3662           /* if this is 1st frame */
3663           if (!f)
3664             {
3665               f = vlib_get_frame_to_node (vm, to_node_index);
3666               to_next = vlib_frame_vector_args (f);
3667             }
3668
3669           to_next[0] = bi0;
3670           to_next += 1;
3671           f->n_vectors++;
3672         }
3673
3674       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3675                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3676         {
3677           snat_in2out_worker_handoff_trace_t *t =
3678             vlib_add_trace (vm, node, b0, sizeof (*t));
3679           t->next_worker_index = next_worker_index;
3680           t->do_handoff = do_handoff;
3681         }
3682     }
3683
3684   if (f)
3685     vlib_put_frame_to_node (vm, to_node_index, f);
3686
3687   if (hf)
3688     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3689
3690   /* Ship frames to the worker nodes */
3691   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3692     {
3693       if (handoff_queue_elt_by_worker_index[i])
3694         {
3695           hf = handoff_queue_elt_by_worker_index[i];
3696           /*
3697            * It works better to let the handoff node
3698            * rate-adapt, always ship the handoff queue element.
3699            */
3700           if (1 || hf->n_vectors == hf->last_n_vectors)
3701             {
3702               vlib_put_frame_queue_elt (hf);
3703               handoff_queue_elt_by_worker_index[i] = 0;
3704             }
3705           else
3706             hf->last_n_vectors = hf->n_vectors;
3707         }
3708       congested_handoff_queue_by_worker_index[i] =
3709         (vlib_frame_queue_t *) (~0);
3710     }
3711   hf = 0;
3712   current_worker_index = ~0;
3713   return frame->n_vectors;
3714 }
3715
3716 static uword
3717 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3718                                vlib_node_runtime_t * node,
3719                                vlib_frame_t * frame)
3720 {
3721   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3722 }
3723
3724 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3725   .function = snat_in2out_worker_handoff_fn,
3726   .name = "nat44-in2out-worker-handoff",
3727   .vector_size = sizeof (u32),
3728   .format_trace = format_snat_in2out_worker_handoff_trace,
3729   .type = VLIB_NODE_TYPE_INTERNAL,
3730
3731   .n_next_nodes = 1,
3732
3733   .next_nodes = {
3734     [0] = "error-drop",
3735   },
3736 };
3737
3738 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3739                               snat_in2out_worker_handoff_fn);
3740
3741 static uword
3742 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3743                                       vlib_node_runtime_t * node,
3744                                       vlib_frame_t * frame)
3745 {
3746   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3747 }
3748
3749 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3750   .function = snat_in2out_output_worker_handoff_fn,
3751   .name = "nat44-in2out-output-worker-handoff",
3752   .vector_size = sizeof (u32),
3753   .format_trace = format_snat_in2out_worker_handoff_trace,
3754   .type = VLIB_NODE_TYPE_INTERNAL,
3755
3756   .n_next_nodes = 1,
3757
3758   .next_nodes = {
3759     [0] = "error-drop",
3760   },
3761 };
3762
3763 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3764                               snat_in2out_output_worker_handoff_fn);
3765
3766 static_always_inline int
3767 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3768 {
3769   snat_address_t * ap;
3770   clib_bihash_kv_8_8_t kv, value;
3771   snat_session_key_t m_key;
3772
3773   vec_foreach (ap, sm->addresses)
3774     {
3775       if (ap->addr.as_u32 == dst_addr->as_u32)
3776         return 1;
3777     }
3778
3779   m_key.addr.as_u32 = dst_addr->as_u32;
3780   m_key.fib_index = sm->outside_fib_index;
3781   m_key.port = 0;
3782   m_key.protocol = 0;
3783   kv.key = m_key.as_u64;
3784   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3785     return 1;
3786
3787   return 0;
3788 }
3789
3790 static uword
3791 snat_hairpin_dst_fn (vlib_main_t * vm,
3792                      vlib_node_runtime_t * node,
3793                      vlib_frame_t * frame)
3794 {
3795   u32 n_left_from, * from, * to_next;
3796   snat_in2out_next_t next_index;
3797   u32 pkts_processed = 0;
3798   snat_main_t * sm = &snat_main;
3799
3800   from = vlib_frame_vector_args (frame);
3801   n_left_from = frame->n_vectors;
3802   next_index = node->cached_next_index;
3803
3804   while (n_left_from > 0)
3805     {
3806       u32 n_left_to_next;
3807
3808       vlib_get_next_frame (vm, node, next_index,
3809                            to_next, n_left_to_next);
3810
3811       while (n_left_from > 0 && n_left_to_next > 0)
3812         {
3813           u32 bi0;
3814           vlib_buffer_t * b0;
3815           u32 next0;
3816           ip4_header_t * ip0;
3817           u32 proto0;
3818
3819           /* speculatively enqueue b0 to the current next frame */
3820           bi0 = from[0];
3821           to_next[0] = bi0;
3822           from += 1;
3823           to_next += 1;
3824           n_left_from -= 1;
3825           n_left_to_next -= 1;
3826
3827           b0 = vlib_get_buffer (vm, bi0);
3828           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3829           ip0 = vlib_buffer_get_current (b0);
3830
3831           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3832
3833           vnet_buffer (b0)->snat.flags = 0;
3834           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3835             {
3836               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3837                 {
3838                   udp_header_t * udp0 = ip4_next_header (ip0);
3839                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3840
3841                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3842                 }
3843               else if (proto0 == SNAT_PROTOCOL_ICMP)
3844                 {
3845                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3846
3847                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3848                 }
3849               else
3850                 {
3851                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3852                 }
3853
3854               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3855             }
3856
3857           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3858
3859           /* verify speculative enqueue, maybe switch current next frame */
3860           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3861                                            to_next, n_left_to_next,
3862                                            bi0, next0);
3863          }
3864
3865       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3866     }
3867
3868   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3869                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3870                                pkts_processed);
3871   return frame->n_vectors;
3872 }
3873
3874 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3875   .function = snat_hairpin_dst_fn,
3876   .name = "nat44-hairpin-dst",
3877   .vector_size = sizeof (u32),
3878   .type = VLIB_NODE_TYPE_INTERNAL,
3879   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3880   .error_strings = snat_in2out_error_strings,
3881   .n_next_nodes = 2,
3882   .next_nodes = {
3883     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3884     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3885   },
3886 };
3887
3888 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3889                               snat_hairpin_dst_fn);
3890
3891 static uword
3892 snat_hairpin_src_fn (vlib_main_t * vm,
3893                      vlib_node_runtime_t * node,
3894                      vlib_frame_t * frame)
3895 {
3896   u32 n_left_from, * from, * to_next;
3897   snat_in2out_next_t next_index;
3898   u32 pkts_processed = 0;
3899   snat_main_t *sm = &snat_main;
3900
3901   from = vlib_frame_vector_args (frame);
3902   n_left_from = frame->n_vectors;
3903   next_index = node->cached_next_index;
3904
3905   while (n_left_from > 0)
3906     {
3907       u32 n_left_to_next;
3908
3909       vlib_get_next_frame (vm, node, next_index,
3910                            to_next, n_left_to_next);
3911
3912       while (n_left_from > 0 && n_left_to_next > 0)
3913         {
3914           u32 bi0;
3915           vlib_buffer_t * b0;
3916           u32 next0;
3917           snat_interface_t *i;
3918           u32 sw_if_index0;
3919
3920           /* speculatively enqueue b0 to the current next frame */
3921           bi0 = from[0];
3922           to_next[0] = bi0;
3923           from += 1;
3924           to_next += 1;
3925           n_left_from -= 1;
3926           n_left_to_next -= 1;
3927
3928           b0 = vlib_get_buffer (vm, bi0);
3929           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3930           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3931
3932           pool_foreach (i, sm->output_feature_interfaces,
3933           ({
3934             /* Only packets from NAT inside interface */
3935             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3936               {
3937                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3938                                     SNAT_FLAG_HAIRPINNING))
3939                   {
3940                     if (PREDICT_TRUE (sm->num_workers > 1))
3941                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3942                     else
3943                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3944                   }
3945                 break;
3946               }
3947           }));
3948
3949           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3950
3951           /* verify speculative enqueue, maybe switch current next frame */
3952           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3953                                            to_next, n_left_to_next,
3954                                            bi0, next0);
3955          }
3956
3957       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3958     }
3959
3960   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3961                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3962                                pkts_processed);
3963   return frame->n_vectors;
3964 }
3965
3966 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3967   .function = snat_hairpin_src_fn,
3968   .name = "nat44-hairpin-src",
3969   .vector_size = sizeof (u32),
3970   .type = VLIB_NODE_TYPE_INTERNAL,
3971   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3972   .error_strings = snat_in2out_error_strings,
3973   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3974   .next_nodes = {
3975      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3976      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3977      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3978      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3979   },
3980 };
3981
3982 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3983                               snat_hairpin_src_fn);
3984
3985 static uword
3986 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3987                                 vlib_node_runtime_t * node,
3988                                 vlib_frame_t * frame)
3989 {
3990   u32 n_left_from, * from, * to_next;
3991   snat_in2out_next_t next_index;
3992   u32 pkts_processed = 0;
3993   snat_main_t * sm = &snat_main;
3994   u32 stats_node_index;
3995
3996   stats_node_index = snat_in2out_fast_node.index;
3997
3998   from = vlib_frame_vector_args (frame);
3999   n_left_from = frame->n_vectors;
4000   next_index = node->cached_next_index;
4001
4002   while (n_left_from > 0)
4003     {
4004       u32 n_left_to_next;
4005
4006       vlib_get_next_frame (vm, node, next_index,
4007                            to_next, n_left_to_next);
4008
4009       while (n_left_from > 0 && n_left_to_next > 0)
4010         {
4011           u32 bi0;
4012           vlib_buffer_t * b0;
4013           u32 next0;
4014           u32 sw_if_index0;
4015           ip4_header_t * ip0;
4016           ip_csum_t sum0;
4017           u32 new_addr0, old_addr0;
4018           u16 old_port0, new_port0;
4019           udp_header_t * udp0;
4020           tcp_header_t * tcp0;
4021           icmp46_header_t * icmp0;
4022           snat_session_key_t key0, sm0;
4023           u32 proto0;
4024           u32 rx_fib_index0;
4025
4026           /* speculatively enqueue b0 to the current next frame */
4027           bi0 = from[0];
4028           to_next[0] = bi0;
4029           from += 1;
4030           to_next += 1;
4031           n_left_from -= 1;
4032           n_left_to_next -= 1;
4033
4034           b0 = vlib_get_buffer (vm, bi0);
4035           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4036
4037           ip0 = vlib_buffer_get_current (b0);
4038           udp0 = ip4_next_header (ip0);
4039           tcp0 = (tcp_header_t *) udp0;
4040           icmp0 = (icmp46_header_t *) udp0;
4041
4042           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4043           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4044
4045           if (PREDICT_FALSE(ip0->ttl == 1))
4046             {
4047               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4048               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4049                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4050                                            0);
4051               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4052               goto trace0;
4053             }
4054
4055           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4056
4057           if (PREDICT_FALSE (proto0 == ~0))
4058               goto trace0;
4059
4060           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4061             {
4062               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4063                                   rx_fib_index0, node, next0, ~0, 0, 0);
4064               goto trace0;
4065             }
4066
4067           key0.addr = ip0->src_address;
4068           key0.protocol = proto0;
4069           key0.port = udp0->src_port;
4070           key0.fib_index = rx_fib_index0;
4071
4072           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4073             {
4074               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4075               next0= SNAT_IN2OUT_NEXT_DROP;
4076               goto trace0;
4077             }
4078
4079           new_addr0 = sm0.addr.as_u32;
4080           new_port0 = sm0.port;
4081           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4082           old_addr0 = ip0->src_address.as_u32;
4083           ip0->src_address.as_u32 = new_addr0;
4084
4085           sum0 = ip0->checksum;
4086           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4087                                  ip4_header_t,
4088                                  src_address /* changed member */);
4089           ip0->checksum = ip_csum_fold (sum0);
4090
4091           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4092             {
4093               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4094                 {
4095                   old_port0 = tcp0->src_port;
4096                   tcp0->src_port = new_port0;
4097
4098                   sum0 = tcp0->checksum;
4099                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4100                                          ip4_header_t,
4101                                          dst_address /* changed member */);
4102                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4103                                          ip4_header_t /* cheat */,
4104                                          length /* changed member */);
4105                   tcp0->checksum = ip_csum_fold(sum0);
4106                 }
4107               else
4108                 {
4109                   old_port0 = udp0->src_port;
4110                   udp0->src_port = new_port0;
4111                   udp0->checksum = 0;
4112                 }
4113             }
4114           else
4115             {
4116               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4117                 {
4118                   sum0 = tcp0->checksum;
4119                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4120                                          ip4_header_t,
4121                                          dst_address /* changed member */);
4122                   tcp0->checksum = ip_csum_fold(sum0);
4123                 }
4124             }
4125
4126           /* Hairpinning */
4127           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4128
4129         trace0:
4130           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4131                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4132             {
4133               snat_in2out_trace_t *t =
4134                  vlib_add_trace (vm, node, b0, sizeof (*t));
4135               t->sw_if_index = sw_if_index0;
4136               t->next_index = next0;
4137             }
4138
4139           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4140
4141           /* verify speculative enqueue, maybe switch current next frame */
4142           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4143                                            to_next, n_left_to_next,
4144                                            bi0, next0);
4145         }
4146
4147       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4148     }
4149
4150   vlib_node_increment_counter (vm, stats_node_index,
4151                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4152                                pkts_processed);
4153   return frame->n_vectors;
4154 }
4155
4156
4157 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4158   .function = snat_in2out_fast_static_map_fn,
4159   .name = "nat44-in2out-fast",
4160   .vector_size = sizeof (u32),
4161   .format_trace = format_snat_in2out_fast_trace,
4162   .type = VLIB_NODE_TYPE_INTERNAL,
4163
4164   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4165   .error_strings = snat_in2out_error_strings,
4166
4167   .runtime_data_bytes = sizeof (snat_runtime_t),
4168
4169   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4170
4171   /* edit / add dispositions here */
4172   .next_nodes = {
4173     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4174     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4175     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4176     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4177     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4178   },
4179 };
4180
4181 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);