src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29 #include <nat/nat_inlines.h>
  30
  31 #include <vppinfra/hash.h>
  32 #include <vppinfra/error.h>
  33 #include <vppinfra/elog.h>
  34
  35 typedef struct {
  36   u32 sw_if_index;
  37   u32 next_index;
  38   u32 session_index;
  39 } snat_out2in_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_out2in_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  52
  53   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  54               t->sw_if_index, t->next_index, t->session_index);
  55   return s;
  56 }
  57
  58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  63
  64   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  65               t->sw_if_index, t->next_index);
  66   return s;
  67 }
  68
  69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  70 {
  71   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  72   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  73   snat_out2in_worker_handoff_trace_t * t =
  74     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  75   char * m;
  76
  77   m = t->do_handoff ? "next worker" : "same worker";
  78   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  79
  80   return s;
  81 }
  82
  83 typedef struct {
  84   u32 sw_if_index;
  85   u32 next_index;
  86   u8 cached;
  87 } nat44_out2in_reass_trace_t;
  88
  89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  90 {
  91   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  92   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  93   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  94
  95   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  96               t->sw_if_index, t->next_index,
  97               t->cached ? "cached" : "translated");
  98
  99   return s;
 100 }
 101
 102 vlib_node_registration_t snat_out2in_node;
 103 vlib_node_registration_t snat_out2in_fast_node;
 104 vlib_node_registration_t snat_out2in_worker_handoff_node;
 105 vlib_node_registration_t snat_det_out2in_node;
 106 vlib_node_registration_t nat44_out2in_reass_node;
 107 vlib_node_registration_t nat44_ed_out2in_node;
 108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
 109
 110 #define foreach_snat_out2in_error                       \
 111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 112 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 113 _(OUT_OF_PORTS, "Out of ports")                         \
 114 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 115 _(NO_TRANSLATION, "No translation")                     \
 116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 117 _(DROP_FRAGMENT, "Drop fragment")                       \
 118 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 120 _(FQ_CONGESTED, "Handoff frame queue congested")
 121
 122 typedef enum {
 123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 124   foreach_snat_out2in_error
 125 #undef _
 126   SNAT_OUT2IN_N_ERROR,
 127 } snat_out2in_error_t;
 128
 129 static char * snat_out2in_error_strings[] = {
 130 #define _(sym,string) string,
 131   foreach_snat_out2in_error
 132 #undef _
 133 };
 134
 135 typedef enum {
 136   SNAT_OUT2IN_NEXT_DROP,
 137   SNAT_OUT2IN_NEXT_LOOKUP,
 138   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 139   SNAT_OUT2IN_NEXT_REASS,
 140   SNAT_OUT2IN_N_NEXT,
 141 } snat_out2in_next_t;
 142
 143 /**
 144  * @brief Create session for static mapping.
 145  *
 146  * Create NAT session initiated by host from external network with static
 147  * mapping.
 148  *
 149  * @param sm     NAT main.
 150  * @param b0     Vlib buffer.
 151  * @param in2out In2out NAT44 session key.
 152  * @param out2in Out2in NAT44 session key.
 153  * @param node   Vlib node.
 154  *
 155  * @returns SNAT session if successfully created otherwise 0.
 156  */
 157 static inline snat_session_t *
 158 create_session_for_static_mapping (snat_main_t *sm,
 159                                    vlib_buffer_t *b0,
 160                                    snat_session_key_t in2out,
 161                                    snat_session_key_t out2in,
 162                                    vlib_node_runtime_t * node,
 163                                    u32 thread_index)
 164 {
 165   snat_user_t *u;
 166   snat_session_t *s;
 167   clib_bihash_kv_8_8_t kv0;
 168   ip4_header_t *ip0;
 169   udp_header_t *udp0;
 170
 171   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 172     {
 173       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 174       nat_log_notice ("maximum sessions exceeded");
 175       return 0;
 176     }
 177
 178   ip0 = vlib_buffer_get_current (b0);
 179   udp0 = ip4_next_header (ip0);
 180
 181   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 182   if (!u)
 183     {
 184       nat_log_warn ("create NAT user failed");
 185       return 0;
 186     }
 187
 188   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 189   if (!s)
 190     {
 191       nat44_delete_user_with_no_session (sm, u, thread_index);
 192       nat_log_warn ("create NAT session failed");
 193       return 0;
 194     }
 195
 196   s->outside_address_index = ~0;
 197   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 198   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 199   s->ext_host_port = udp0->src_port;
 200   user_session_increment (sm, u, 1 /* static */);
 201   s->in2out = in2out;
 202   s->out2in = out2in;
 203   s->in2out.protocol = out2in.protocol;
 204
 205   /* Add to translation hashes */
 206   kv0.key = s->in2out.as_u64;
 207   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 208   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 209                                1 /* is_add */))
 210       nat_log_notice ("in2out key add failed");
 211
 212   kv0.key = s->out2in.as_u64;
 213
 214   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 215                                1 /* is_add */))
 216       nat_log_notice ("out2in key add failed");
 217
 218   /* log NAT event */
 219   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 220                                       s->out2in.addr.as_u32,
 221                                       s->in2out.protocol,
 222                                       s->in2out.port,
 223                                       s->out2in.port,
 224                                       s->in2out.fib_index);
 225    return s;
 226 }
 227
 228 static_always_inline
 229 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 230                                  snat_session_key_t *p_key0)
 231 {
 232   icmp46_header_t *icmp0;
 233   snat_session_key_t key0;
 234   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 235   ip4_header_t *inner_ip0;
 236   void *l4_header = 0;
 237   icmp46_header_t *inner_icmp0;
 238
 239   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 240   echo0 = (icmp_echo_header_t *)(icmp0+1);
 241
 242   if (!icmp_is_error_message (icmp0))
 243     {
 244       key0.protocol = SNAT_PROTOCOL_ICMP;
 245       key0.addr = ip0->dst_address;
 246       key0.port = echo0->identifier;
 247     }
 248   else
 249     {
 250       inner_ip0 = (ip4_header_t *)(echo0+1);
 251       l4_header = ip4_next_header (inner_ip0);
 252       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 253       key0.addr = inner_ip0->src_address;
 254       switch (key0.protocol)
 255         {
 256         case SNAT_PROTOCOL_ICMP:
 257           inner_icmp0 = (icmp46_header_t*)l4_header;
 258           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 259           key0.port = inner_echo0->identifier;
 260           break;
 261         case SNAT_PROTOCOL_UDP:
 262         case SNAT_PROTOCOL_TCP:
 263           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 264           break;
 265         default:
 266           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 267         }
 268     }
 269   *p_key0 = key0;
 270   return -1; /* success */
 271 }
 272
 273 /**
 274  * Get address and port values to be used for ICMP packet translation
 275  * and create session if needed
 276  *
 277  * @param[in,out] sm             NAT main
 278  * @param[in,out] node           NAT node runtime
 279  * @param[in] thread_index       thread index
 280  * @param[in,out] b0             buffer containing packet to be translated
 281  * @param[out] p_proto           protocol used for matching
 282  * @param[out] p_value           address and port after NAT translation
 283  * @param[out] p_dont_translate  if packet should not be translated
 284  * @param d                      optional parameter
 285  * @param e                      optional parameter
 286  */
 287 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 288                            u32 thread_index, vlib_buffer_t *b0,
 289                            ip4_header_t *ip0, u8 *p_proto,
 290                            snat_session_key_t *p_value,
 291                            u8 *p_dont_translate, void *d, void *e)
 292 {
 293   icmp46_header_t *icmp0;
 294   u32 sw_if_index0;
 295   u32 rx_fib_index0;
 296   snat_session_key_t key0;
 297   snat_session_key_t sm0;
 298   snat_session_t *s0 = 0;
 299   u8 dont_translate = 0;
 300   clib_bihash_kv_8_8_t kv0, value0;
 301   u8 is_addr_only;
 302   u32 next0 = ~0;
 303   int err;
 304
 305   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 306   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 307   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 308
 309   key0.protocol = 0;
 310
 311   err = icmp_get_key (ip0, &key0);
 312   if (err != -1)
 313     {
 314       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 315       next0 = SNAT_OUT2IN_NEXT_DROP;
 316       goto out;
 317     }
 318   key0.fib_index = rx_fib_index0;
 319
 320   kv0.key = key0.as_u64;
 321
 322   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 323                               &value0))
 324     {
 325       /* Try to match static mapping by external address and port,
 326          destination address and port in packet */
 327       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 328         {
 329           if (!sm->forwarding_enabled)
 330             {
 331               /* Don't NAT packet aimed at the intfc address */
 332               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 333                                                   ip0->dst_address.as_u32)))
 334                 {
 335                   dont_translate = 1;
 336                   goto out;
 337                 }
 338               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 339               next0 = SNAT_OUT2IN_NEXT_DROP;
 340               goto out;
 341             }
 342           else
 343             {
 344               dont_translate = 1;
 345               goto out;
 346             }
 347         }
 348
 349       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 350                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 351         {
 352           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 353           next0 = SNAT_OUT2IN_NEXT_DROP;
 354           goto out;
 355         }
 356
 357       /* Create session initiated by host from external network */
 358       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 359                                              node, thread_index);
 360
 361       if (!s0)
 362         {
 363           next0 = SNAT_OUT2IN_NEXT_DROP;
 364           goto out;
 365         }
 366     }
 367   else
 368     {
 369       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 370                         icmp0->type != ICMP4_echo_request &&
 371                         !icmp_is_error_message (icmp0)))
 372         {
 373           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 374           next0 = SNAT_OUT2IN_NEXT_DROP;
 375           goto out;
 376         }
 377
 378       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 379                               value0.value);
 380     }
 381
 382 out:
 383   *p_proto = key0.protocol;
 384   if (s0)
 385     *p_value = s0->in2out;
 386   *p_dont_translate = dont_translate;
 387   if (d)
 388     *(snat_session_t**)d = s0;
 389   return next0;
 390 }
 391
 392 /**
 393  * Get address and port values to be used for ICMP packet translation
 394  *
 395  * @param[in] sm                 NAT main
 396  * @param[in,out] node           NAT node runtime
 397  * @param[in] thread_index       thread index
 398  * @param[in,out] b0             buffer containing packet to be translated
 399  * @param[out] p_proto           protocol used for matching
 400  * @param[out] p_value           address and port after NAT translation
 401  * @param[out] p_dont_translate  if packet should not be translated
 402  * @param d                      optional parameter
 403  * @param e                      optional parameter
 404  */
 405 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 406                            u32 thread_index, vlib_buffer_t *b0,
 407                            ip4_header_t *ip0, u8 *p_proto,
 408                            snat_session_key_t *p_value,
 409                            u8 *p_dont_translate, void *d, void *e)
 410 {
 411   icmp46_header_t *icmp0;
 412   u32 sw_if_index0;
 413   u32 rx_fib_index0;
 414   snat_session_key_t key0;
 415   snat_session_key_t sm0;
 416   u8 dont_translate = 0;
 417   u8 is_addr_only;
 418   u32 next0 = ~0;
 419   int err;
 420
 421   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 422   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 423   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 424
 425   err = icmp_get_key (ip0, &key0);
 426   if (err != -1)
 427     {
 428       b0->error = node->errors[err];
 429       next0 = SNAT_OUT2IN_NEXT_DROP;
 430       goto out2;
 431     }
 432   key0.fib_index = rx_fib_index0;
 433
 434   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 435     {
 436       /* Don't NAT packet aimed at the intfc address */
 437       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 438         {
 439           dont_translate = 1;
 440           goto out;
 441         }
 442       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 443       next0 = SNAT_OUT2IN_NEXT_DROP;
 444       goto out;
 445     }
 446
 447   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 448                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 449                     !icmp_is_error_message (icmp0)))
 450     {
 451       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 452       next0 = SNAT_OUT2IN_NEXT_DROP;
 453       goto out;
 454     }
 455
 456 out:
 457   *p_value = sm0;
 458 out2:
 459   *p_proto = key0.protocol;
 460   *p_dont_translate = dont_translate;
 461   return next0;
 462 }
 463
 464 static inline u32 icmp_out2in (snat_main_t *sm,
 465                                vlib_buffer_t * b0,
 466                                ip4_header_t * ip0,
 467                                icmp46_header_t * icmp0,
 468                                u32 sw_if_index0,
 469                                u32 rx_fib_index0,
 470                                vlib_node_runtime_t * node,
 471                                u32 next0,
 472                                u32 thread_index,
 473                                void *d,
 474                                void *e)
 475 {
 476   snat_session_key_t sm0;
 477   u8 protocol;
 478   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 479   ip4_header_t *inner_ip0 = 0;
 480   void *l4_header = 0;
 481   icmp46_header_t *inner_icmp0;
 482   u8 dont_translate;
 483   u32 new_addr0, old_addr0;
 484   u16 old_id0, new_id0;
 485   ip_csum_t sum0;
 486   u16 checksum0;
 487   u32 next0_tmp;
 488
 489   echo0 = (icmp_echo_header_t *)(icmp0+1);
 490
 491   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 492                                        &protocol, &sm0, &dont_translate, d, e);
 493   if (next0_tmp != ~0)
 494     next0 = next0_tmp;
 495   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 496     goto out;
 497
 498   sum0 = ip_incremental_checksum (0, icmp0,
 499                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 500   checksum0 = ~ip_csum_fold (sum0);
 501   if (checksum0 != 0 && checksum0 != 0xffff)
 502     {
 503       next0 = SNAT_OUT2IN_NEXT_DROP;
 504       goto out;
 505     }
 506
 507   old_addr0 = ip0->dst_address.as_u32;
 508   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 509   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 510
 511   sum0 = ip0->checksum;
 512   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 513                          dst_address /* changed member */);
 514   ip0->checksum = ip_csum_fold (sum0);
 515
 516   if (icmp0->checksum == 0)
 517     icmp0->checksum = 0xffff;
 518
 519   if (!icmp_is_error_message (icmp0))
 520     {
 521       new_id0 = sm0.port;
 522       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 523         {
 524           old_id0 = echo0->identifier;
 525           new_id0 = sm0.port;
 526           echo0->identifier = new_id0;
 527
 528           sum0 = icmp0->checksum;
 529           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 530                                  identifier /* changed member */);
 531           icmp0->checksum = ip_csum_fold (sum0);
 532         }
 533     }
 534   else
 535     {
 536       inner_ip0 = (ip4_header_t *)(echo0+1);
 537       l4_header = ip4_next_header (inner_ip0);
 538
 539       if (!ip4_header_checksum_is_valid (inner_ip0))
 540         {
 541           next0 = SNAT_OUT2IN_NEXT_DROP;
 542           goto out;
 543         }
 544
 545       old_addr0 = inner_ip0->src_address.as_u32;
 546       inner_ip0->src_address = sm0.addr;
 547       new_addr0 = inner_ip0->src_address.as_u32;
 548
 549       sum0 = icmp0->checksum;
 550       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 551                              src_address /* changed member */);
 552       icmp0->checksum = ip_csum_fold (sum0);
 553
 554       switch (protocol)
 555         {
 556         case SNAT_PROTOCOL_ICMP:
 557           inner_icmp0 = (icmp46_header_t*)l4_header;
 558           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 559
 560           old_id0 = inner_echo0->identifier;
 561           new_id0 = sm0.port;
 562           inner_echo0->identifier = new_id0;
 563
 564           sum0 = icmp0->checksum;
 565           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 566                                  identifier);
 567           icmp0->checksum = ip_csum_fold (sum0);
 568           break;
 569         case SNAT_PROTOCOL_UDP:
 570         case SNAT_PROTOCOL_TCP:
 571           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 572           new_id0 = sm0.port;
 573           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 574
 575           sum0 = icmp0->checksum;
 576           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 577                                  src_port);
 578           icmp0->checksum = ip_csum_fold (sum0);
 579           break;
 580         default:
 581           ASSERT(0);
 582         }
 583     }
 584
 585 out:
 586   return next0;
 587 }
 588
 589
 590 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 591                                          vlib_buffer_t * b0,
 592                                          ip4_header_t * ip0,
 593                                          icmp46_header_t * icmp0,
 594                                          u32 sw_if_index0,
 595                                          u32 rx_fib_index0,
 596                                          vlib_node_runtime_t * node,
 597                                          u32 next0, f64 now,
 598                                          u32 thread_index,
 599                                          snat_session_t ** p_s0)
 600 {
 601   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 602                       next0, thread_index, p_s0, 0);
 603   snat_session_t * s0 = *p_s0;
 604   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 605     {
 606       /* Accounting */
 607       nat44_session_update_counters (s0, now,
 608                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
 609       /* Per-user LRU list maintenance */
 610       nat44_session_update_lru (sm, s0, thread_index);
 611     }
 612   return next0;
 613 }
 614
 615 static int
 616 nat_out2in_sm_unknown_proto (snat_main_t *sm,
 617                              vlib_buffer_t * b,
 618                              ip4_header_t * ip,
 619                              u32 rx_fib_index)
 620 {
 621   clib_bihash_kv_8_8_t kv, value;
 622   snat_static_mapping_t *m;
 623   snat_session_key_t m_key;
 624   u32 old_addr, new_addr;
 625   ip_csum_t sum;
 626
 627   m_key.addr = ip->dst_address;
 628   m_key.port = 0;
 629   m_key.protocol = 0;
 630   m_key.fib_index = 0;
 631   kv.key = m_key.as_u64;
 632   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 633     return 1;
 634
 635   m = pool_elt_at_index (sm->static_mappings, value.value);
 636
 637   old_addr = ip->dst_address.as_u32;
 638   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 639   sum = ip->checksum;
 640   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 641   ip->checksum = ip_csum_fold (sum);
 642
 643   vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 644   return 0;
 645 }
 646
 647 static uword
 648 snat_out2in_node_fn (vlib_main_t * vm,
 649                   vlib_node_runtime_t * node,
 650                   vlib_frame_t * frame)
 651 {
 652   u32 n_left_from, * from, * to_next;
 653   snat_out2in_next_t next_index;
 654   u32 pkts_processed = 0;
 655   snat_main_t * sm = &snat_main;
 656   f64 now = vlib_time_now (vm);
 657   u32 thread_index = vm->thread_index;
 658
 659   from = vlib_frame_vector_args (frame);
 660   n_left_from = frame->n_vectors;
 661   next_index = node->cached_next_index;
 662
 663   while (n_left_from > 0)
 664     {
 665       u32 n_left_to_next;
 666
 667       vlib_get_next_frame (vm, node, next_index,
 668                            to_next, n_left_to_next);
 669
 670       while (n_left_from >= 4 && n_left_to_next >= 2)
 671         {
 672           u32 bi0, bi1;
 673           vlib_buffer_t * b0, * b1;
 674           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 675           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 676           u32 sw_if_index0, sw_if_index1;
 677           ip4_header_t * ip0, *ip1;
 678           ip_csum_t sum0, sum1;
 679           u32 new_addr0, old_addr0;
 680           u16 new_port0, old_port0;
 681           u32 new_addr1, old_addr1;
 682           u16 new_port1, old_port1;
 683           udp_header_t * udp0, * udp1;
 684           tcp_header_t * tcp0, * tcp1;
 685           icmp46_header_t * icmp0, * icmp1;
 686           snat_session_key_t key0, key1, sm0, sm1;
 687           u32 rx_fib_index0, rx_fib_index1;
 688           u32 proto0, proto1;
 689           snat_session_t * s0 = 0, * s1 = 0;
 690           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 691
 692           /* Prefetch next iteration. */
 693           {
 694             vlib_buffer_t * p2, * p3;
 695
 696             p2 = vlib_get_buffer (vm, from[2]);
 697             p3 = vlib_get_buffer (vm, from[3]);
 698
 699             vlib_prefetch_buffer_header (p2, LOAD);
 700             vlib_prefetch_buffer_header (p3, LOAD);
 701
 702             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 703             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 704           }
 705
 706           /* speculatively enqueue b0 and b1 to the current next frame */
 707           to_next[0] = bi0 = from[0];
 708           to_next[1] = bi1 = from[1];
 709           from += 2;
 710           to_next += 2;
 711           n_left_from -= 2;
 712           n_left_to_next -= 2;
 713
 714           b0 = vlib_get_buffer (vm, bi0);
 715           b1 = vlib_get_buffer (vm, bi1);
 716
 717           vnet_buffer (b0)->snat.flags = 0;
 718           vnet_buffer (b1)->snat.flags = 0;
 719
 720           ip0 = vlib_buffer_get_current (b0);
 721           udp0 = ip4_next_header (ip0);
 722           tcp0 = (tcp_header_t *) udp0;
 723           icmp0 = (icmp46_header_t *) udp0;
 724
 725           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 726           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 727                                    sw_if_index0);
 728
 729           if (PREDICT_FALSE(ip0->ttl == 1))
 730             {
 731               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 732               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 733                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 734                                            0);
 735               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 736               goto trace0;
 737             }
 738
 739           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 740
 741           if (PREDICT_FALSE (proto0 == ~0))
 742             {
 743               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
 744                 {
 745                   if (!sm->forwarding_enabled)
 746                     {
 747                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 748                       next0 = SNAT_OUT2IN_NEXT_DROP;
 749                     }
 750                 }
 751               goto trace0;
 752             }
 753
 754           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 755             {
 756               next0 = icmp_out2in_slow_path
 757                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 758                  next0, now, thread_index, &s0);
 759               goto trace0;
 760             }
 761
 762           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
 763             {
 764               next0 = SNAT_OUT2IN_NEXT_REASS;
 765               goto trace0;
 766             }
 767
 768           key0.addr = ip0->dst_address;
 769           key0.port = udp0->dst_port;
 770           key0.protocol = proto0;
 771           key0.fib_index = rx_fib_index0;
 772
 773           kv0.key = key0.as_u64;
 774
 775           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 776                                       &kv0, &value0))
 777             {
 778               /* Try to match static mapping by external address and port,
 779                  destination address and port in packet */
 780               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 781                 {
 782                   /*
 783                    * Send DHCP packets to the ipv4 stack, or we won't
 784                    * be able to use dhcp client on the outside interface
 785                    */
 786                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 787                       && (udp0->dst_port ==
 788                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 789                     {
 790                       vnet_feature_next (&next0, b0);
 791                       goto trace0;
 792                     }
 793
 794                   if (!sm->forwarding_enabled)
 795                     {
 796                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 797                       next0 = SNAT_OUT2IN_NEXT_DROP;
 798                     }
 799                   goto trace0;
 800                 }
 801
 802               /* Create session initiated by host from external network */
 803               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 804                                                      thread_index);
 805               if (!s0)
 806                 {
 807                   next0 = SNAT_OUT2IN_NEXT_DROP;
 808                   goto trace0;
 809                 }
 810             }
 811           else
 812             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 813                                     value0.value);
 814
 815           old_addr0 = ip0->dst_address.as_u32;
 816           ip0->dst_address = s0->in2out.addr;
 817           new_addr0 = ip0->dst_address.as_u32;
 818           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 819
 820           sum0 = ip0->checksum;
 821           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 822                                  ip4_header_t,
 823                                  dst_address /* changed member */);
 824           ip0->checksum = ip_csum_fold (sum0);
 825
 826           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 827             {
 828               old_port0 = tcp0->dst_port;
 829               tcp0->dst_port = s0->in2out.port;
 830               new_port0 = tcp0->dst_port;
 831
 832               sum0 = tcp0->checksum;
 833               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 834                                      ip4_header_t,
 835                                      dst_address /* changed member */);
 836
 837               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 838                                      ip4_header_t /* cheat */,
 839                                      length /* changed member */);
 840               tcp0->checksum = ip_csum_fold(sum0);
 841             }
 842           else
 843             {
 844               old_port0 = udp0->dst_port;
 845               udp0->dst_port = s0->in2out.port;
 846               udp0->checksum = 0;
 847             }
 848
 849           /* Accounting */
 850           nat44_session_update_counters (s0, now,
 851                                          vlib_buffer_length_in_chain (vm, b0));
 852           /* Per-user LRU list maintenance */
 853           nat44_session_update_lru (sm, s0, thread_index);
 854         trace0:
 855
 856           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 857                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 858             {
 859               snat_out2in_trace_t *t =
 860                  vlib_add_trace (vm, node, b0, sizeof (*t));
 861               t->sw_if_index = sw_if_index0;
 862               t->next_index = next0;
 863               t->session_index = ~0;
 864               if (s0)
 865                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 866             }
 867
 868           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 869
 870
 871           ip1 = vlib_buffer_get_current (b1);
 872           udp1 = ip4_next_header (ip1);
 873           tcp1 = (tcp_header_t *) udp1;
 874           icmp1 = (icmp46_header_t *) udp1;
 875
 876           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 877           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 878                                    sw_if_index1);
 879
 880           if (PREDICT_FALSE(ip1->ttl == 1))
 881             {
 882               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 883               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 884                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 885                                            0);
 886               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 887               goto trace1;
 888             }
 889
 890           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 891
 892           if (PREDICT_FALSE (proto1 == ~0))
 893             {
 894               if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
 895                 {
 896                   if (!sm->forwarding_enabled)
 897                     {
 898                       b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 899                       next1 = SNAT_OUT2IN_NEXT_DROP;
 900                     }
 901                 }
 902               goto trace1;
 903             }
 904
 905           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 906             {
 907               next1 = icmp_out2in_slow_path
 908                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 909                  next1, now, thread_index, &s1);
 910               goto trace1;
 911             }
 912
 913           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
 914             {
 915               next1 = SNAT_OUT2IN_NEXT_REASS;
 916               goto trace1;
 917             }
 918
 919           key1.addr = ip1->dst_address;
 920           key1.port = udp1->dst_port;
 921           key1.protocol = proto1;
 922           key1.fib_index = rx_fib_index1;
 923
 924           kv1.key = key1.as_u64;
 925
 926           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 927                                       &kv1, &value1))
 928             {
 929               /* Try to match static mapping by external address and port,
 930                  destination address and port in packet */
 931               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
 932                 {
 933                   /*
 934                    * Send DHCP packets to the ipv4 stack, or we won't
 935                    * be able to use dhcp client on the outside interface
 936                    */
 937                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
 938                       && (udp1->dst_port ==
 939                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 940                     {
 941                       vnet_feature_next (&next1, b1);
 942                       goto trace1;
 943                     }
 944
 945                   if (!sm->forwarding_enabled)
 946                     {
 947                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 948                       next1 = SNAT_OUT2IN_NEXT_DROP;
 949                     }
 950                   goto trace1;
 951                 }
 952
 953               /* Create session initiated by host from external network */
 954               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 955                                                      thread_index);
 956               if (!s1)
 957                 {
 958                   next1 = SNAT_OUT2IN_NEXT_DROP;
 959                   goto trace1;
 960                 }
 961             }
 962           else
 963             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 964                                     value1.value);
 965
 966           old_addr1 = ip1->dst_address.as_u32;
 967           ip1->dst_address = s1->in2out.addr;
 968           new_addr1 = ip1->dst_address.as_u32;
 969           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 970
 971           sum1 = ip1->checksum;
 972           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 973                                  ip4_header_t,
 974                                  dst_address /* changed member */);
 975           ip1->checksum = ip_csum_fold (sum1);
 976
 977           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 978             {
 979               old_port1 = tcp1->dst_port;
 980               tcp1->dst_port = s1->in2out.port;
 981               new_port1 = tcp1->dst_port;
 982
 983               sum1 = tcp1->checksum;
 984               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 985                                      ip4_header_t,
 986                                      dst_address /* changed member */);
 987
 988               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 989                                      ip4_header_t /* cheat */,
 990                                      length /* changed member */);
 991               tcp1->checksum = ip_csum_fold(sum1);
 992             }
 993           else
 994             {
 995               old_port1 = udp1->dst_port;
 996               udp1->dst_port = s1->in2out.port;
 997               udp1->checksum = 0;
 998             }
 999
1000           /* Accounting */
1001           nat44_session_update_counters (s1, now,
1002                                          vlib_buffer_length_in_chain (vm, b1));
1003           /* Per-user LRU list maintenance */
1004           nat44_session_update_lru (sm, s1, thread_index);
1005         trace1:
1006
1007           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1008                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1009             {
1010               snat_out2in_trace_t *t =
1011                  vlib_add_trace (vm, node, b1, sizeof (*t));
1012               t->sw_if_index = sw_if_index1;
1013               t->next_index = next1;
1014               t->session_index = ~0;
1015               if (s1)
1016                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1017             }
1018
1019           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1020
1021           /* verify speculative enqueues, maybe switch current next frame */
1022           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1023                                            to_next, n_left_to_next,
1024                                            bi0, bi1, next0, next1);
1025         }
1026
1027       while (n_left_from > 0 && n_left_to_next > 0)
1028         {
1029           u32 bi0;
1030           vlib_buffer_t * b0;
1031           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1032           u32 sw_if_index0;
1033           ip4_header_t * ip0;
1034           ip_csum_t sum0;
1035           u32 new_addr0, old_addr0;
1036           u16 new_port0, old_port0;
1037           udp_header_t * udp0;
1038           tcp_header_t * tcp0;
1039           icmp46_header_t * icmp0;
1040           snat_session_key_t key0, sm0;
1041           u32 rx_fib_index0;
1042           u32 proto0;
1043           snat_session_t * s0 = 0;
1044           clib_bihash_kv_8_8_t kv0, value0;
1045
1046           /* speculatively enqueue b0 to the current next frame */
1047           bi0 = from[0];
1048           to_next[0] = bi0;
1049           from += 1;
1050           to_next += 1;
1051           n_left_from -= 1;
1052           n_left_to_next -= 1;
1053
1054           b0 = vlib_get_buffer (vm, bi0);
1055
1056           vnet_buffer (b0)->snat.flags = 0;
1057
1058           ip0 = vlib_buffer_get_current (b0);
1059           udp0 = ip4_next_header (ip0);
1060           tcp0 = (tcp_header_t *) udp0;
1061           icmp0 = (icmp46_header_t *) udp0;
1062
1063           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1064           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1065                                    sw_if_index0);
1066
1067           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1068
1069           if (PREDICT_FALSE (proto0 == ~0))
1070             {
1071               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1072                 {
1073                   if (!sm->forwarding_enabled)
1074                     {
1075                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1076                       next0 = SNAT_OUT2IN_NEXT_DROP;
1077                     }
1078                 }
1079               goto trace00;
1080             }
1081
1082           if (PREDICT_FALSE(ip0->ttl == 1))
1083             {
1084               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1085               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1086                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1087                                            0);
1088               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1089               goto trace00;
1090             }
1091
1092           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1093             {
1094               next0 = icmp_out2in_slow_path
1095                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1096                  next0, now, thread_index, &s0);
1097               goto trace00;
1098             }
1099
1100           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1101             {
1102               next0 = SNAT_OUT2IN_NEXT_REASS;
1103               goto trace00;
1104             }
1105
1106           key0.addr = ip0->dst_address;
1107           key0.port = udp0->dst_port;
1108           key0.protocol = proto0;
1109           key0.fib_index = rx_fib_index0;
1110
1111           kv0.key = key0.as_u64;
1112
1113           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1114                                       &kv0, &value0))
1115             {
1116               /* Try to match static mapping by external address and port,
1117                  destination address and port in packet */
1118               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1119                 {
1120                   /*
1121                    * Send DHCP packets to the ipv4 stack, or we won't
1122                    * be able to use dhcp client on the outside interface
1123                    */
1124                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1125                       && (udp0->dst_port ==
1126                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1127                     {
1128                       vnet_feature_next (&next0, b0);
1129                       goto trace00;
1130                     }
1131
1132                   if (!sm->forwarding_enabled)
1133                     {
1134                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1135                       next0 = SNAT_OUT2IN_NEXT_DROP;
1136                     }
1137                   goto trace00;
1138                 }
1139
1140               /* Create session initiated by host from external network */
1141               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1142                                                      thread_index);
1143               if (!s0)
1144                 {
1145                   next0 = SNAT_OUT2IN_NEXT_DROP;
1146                   goto trace00;
1147                 }
1148             }
1149           else
1150             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1151                                     value0.value);
1152
1153           old_addr0 = ip0->dst_address.as_u32;
1154           ip0->dst_address = s0->in2out.addr;
1155           new_addr0 = ip0->dst_address.as_u32;
1156           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1157
1158           sum0 = ip0->checksum;
1159           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1160                                  ip4_header_t,
1161                                  dst_address /* changed member */);
1162           ip0->checksum = ip_csum_fold (sum0);
1163
1164           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1165             {
1166               old_port0 = tcp0->dst_port;
1167               tcp0->dst_port = s0->in2out.port;
1168               new_port0 = tcp0->dst_port;
1169
1170               sum0 = tcp0->checksum;
1171               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1172                                      ip4_header_t,
1173                                      dst_address /* changed member */);
1174
1175               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1176                                      ip4_header_t /* cheat */,
1177                                      length /* changed member */);
1178               tcp0->checksum = ip_csum_fold(sum0);
1179             }
1180           else
1181             {
1182               old_port0 = udp0->dst_port;
1183               udp0->dst_port = s0->in2out.port;
1184               udp0->checksum = 0;
1185             }
1186
1187           /* Accounting */
1188           nat44_session_update_counters (s0, now,
1189                                          vlib_buffer_length_in_chain (vm, b0));
1190           /* Per-user LRU list maintenance */
1191           nat44_session_update_lru (sm, s0, thread_index);
1192         trace00:
1193
1194           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1195                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1196             {
1197               snat_out2in_trace_t *t =
1198                  vlib_add_trace (vm, node, b0, sizeof (*t));
1199               t->sw_if_index = sw_if_index0;
1200               t->next_index = next0;
1201               t->session_index = ~0;
1202               if (s0)
1203                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1204             }
1205
1206           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1207
1208           /* verify speculative enqueue, maybe switch current next frame */
1209           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1210                                            to_next, n_left_to_next,
1211                                            bi0, next0);
1212         }
1213
1214       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1215     }
1216
1217   vlib_node_increment_counter (vm, snat_out2in_node.index,
1218                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1219                                pkts_processed);
1220   return frame->n_vectors;
1221 }
1222
1223 VLIB_REGISTER_NODE (snat_out2in_node) = {
1224   .function = snat_out2in_node_fn,
1225   .name = "nat44-out2in",
1226   .vector_size = sizeof (u32),
1227   .format_trace = format_snat_out2in_trace,
1228   .type = VLIB_NODE_TYPE_INTERNAL,
1229
1230   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1231   .error_strings = snat_out2in_error_strings,
1232
1233   .runtime_data_bytes = sizeof (snat_runtime_t),
1234
1235   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1236
1237   /* edit / add dispositions here */
1238   .next_nodes = {
1239     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1240     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1241     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1242     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1243   },
1244 };
1245 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1246
1247 static uword
1248 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1249                             vlib_node_runtime_t * node,
1250                             vlib_frame_t * frame)
1251 {
1252   u32 n_left_from, *from, *to_next;
1253   snat_out2in_next_t next_index;
1254   u32 pkts_processed = 0;
1255   snat_main_t *sm = &snat_main;
1256   f64 now = vlib_time_now (vm);
1257   u32 thread_index = vm->thread_index;
1258   snat_main_per_thread_data_t *per_thread_data =
1259     &sm->per_thread_data[thread_index];
1260   u32 *fragments_to_drop = 0;
1261   u32 *fragments_to_loopback = 0;
1262
1263   from = vlib_frame_vector_args (frame);
1264   n_left_from = frame->n_vectors;
1265   next_index = node->cached_next_index;
1266
1267   while (n_left_from > 0)
1268     {
1269       u32 n_left_to_next;
1270
1271       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1272
1273       while (n_left_from > 0 && n_left_to_next > 0)
1274        {
1275           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1276           vlib_buffer_t *b0;
1277           u32 next0;
1278           u8 cached0 = 0;
1279           ip4_header_t *ip0;
1280           nat_reass_ip4_t *reass0;
1281           udp_header_t * udp0;
1282           tcp_header_t * tcp0;
1283           snat_session_key_t key0, sm0;
1284           clib_bihash_kv_8_8_t kv0, value0;
1285           snat_session_t * s0 = 0;
1286           u16 old_port0, new_port0;
1287           ip_csum_t sum0;
1288
1289           /* speculatively enqueue b0 to the current next frame */
1290           bi0 = from[0];
1291           to_next[0] = bi0;
1292           from += 1;
1293           to_next += 1;
1294           n_left_from -= 1;
1295           n_left_to_next -= 1;
1296
1297           b0 = vlib_get_buffer (vm, bi0);
1298           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1299
1300           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1301           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1302                                                                sw_if_index0);
1303
1304           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1305             {
1306               next0 = SNAT_OUT2IN_NEXT_DROP;
1307               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1308               goto trace0;
1309             }
1310
1311           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1312           udp0 = ip4_next_header (ip0);
1313           tcp0 = (tcp_header_t *) udp0;
1314           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1315
1316           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1317                                                  ip0->dst_address,
1318                                                  ip0->fragment_id,
1319                                                  ip0->protocol,
1320                                                  1,
1321                                                  &fragments_to_drop);
1322
1323           if (PREDICT_FALSE (!reass0))
1324             {
1325               next0 = SNAT_OUT2IN_NEXT_DROP;
1326               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1327               nat_log_notice ("maximum reassemblies exceeded");
1328               goto trace0;
1329             }
1330
1331           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1332             {
1333               key0.addr = ip0->dst_address;
1334               key0.port = udp0->dst_port;
1335               key0.protocol = proto0;
1336               key0.fib_index = rx_fib_index0;
1337               kv0.key = key0.as_u64;
1338
1339               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1340                 {
1341                   /* Try to match static mapping by external address and port,
1342                      destination address and port in packet */
1343                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1344                     {
1345                       /*
1346                        * Send DHCP packets to the ipv4 stack, or we won't
1347                        * be able to use dhcp client on the outside interface
1348                        */
1349                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1350                           && (udp0->dst_port
1351                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1352                         {
1353                           vnet_feature_next (&next0, b0);
1354                           goto trace0;
1355                         }
1356
1357                       if (!sm->forwarding_enabled)
1358                         {
1359                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1360                           next0 = SNAT_OUT2IN_NEXT_DROP;
1361                         }
1362                       goto trace0;
1363                     }
1364
1365                   /* Create session initiated by host from external network */
1366                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1367                                                          thread_index);
1368                   if (!s0)
1369                     {
1370                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1371                       next0 = SNAT_OUT2IN_NEXT_DROP;
1372                       goto trace0;
1373                     }
1374                   reass0->sess_index = s0 - per_thread_data->sessions;
1375                   reass0->thread_index = thread_index;
1376                 }
1377               else
1378                 {
1379                   s0 = pool_elt_at_index (per_thread_data->sessions,
1380                                           value0.value);
1381                   reass0->sess_index = value0.value;
1382                 }
1383               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1384             }
1385           else
1386             {
1387               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1388                 {
1389                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1390                     {
1391                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1392                       nat_log_notice ("maximum fragments per reassembly exceeded");
1393                       next0 = SNAT_OUT2IN_NEXT_DROP;
1394                       goto trace0;
1395                     }
1396                   cached0 = 1;
1397                   goto trace0;
1398                 }
1399               s0 = pool_elt_at_index (per_thread_data->sessions,
1400                                       reass0->sess_index);
1401             }
1402
1403           old_addr0 = ip0->dst_address.as_u32;
1404           ip0->dst_address = s0->in2out.addr;
1405           new_addr0 = ip0->dst_address.as_u32;
1406           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1407
1408           sum0 = ip0->checksum;
1409           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1410                                  ip4_header_t,
1411                                  dst_address /* changed member */);
1412           ip0->checksum = ip_csum_fold (sum0);
1413
1414           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1415             {
1416               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1417                 {
1418                   old_port0 = tcp0->dst_port;
1419                   tcp0->dst_port = s0->in2out.port;
1420                   new_port0 = tcp0->dst_port;
1421
1422                   sum0 = tcp0->checksum;
1423                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1424                                          ip4_header_t,
1425                                          dst_address /* changed member */);
1426
1427                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1428                                          ip4_header_t /* cheat */,
1429                                          length /* changed member */);
1430                   tcp0->checksum = ip_csum_fold(sum0);
1431                 }
1432               else
1433                 {
1434                   old_port0 = udp0->dst_port;
1435                   udp0->dst_port = s0->in2out.port;
1436                   udp0->checksum = 0;
1437                 }
1438             }
1439
1440           /* Accounting */
1441           nat44_session_update_counters (s0, now,
1442                                          vlib_buffer_length_in_chain (vm, b0));
1443           /* Per-user LRU list maintenance */
1444           nat44_session_update_lru (sm, s0, thread_index);
1445
1446         trace0:
1447           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1448                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1449             {
1450               nat44_out2in_reass_trace_t *t =
1451                  vlib_add_trace (vm, node, b0, sizeof (*t));
1452               t->cached = cached0;
1453               t->sw_if_index = sw_if_index0;
1454               t->next_index = next0;
1455             }
1456
1457           if (cached0)
1458             {
1459               n_left_to_next++;
1460               to_next--;
1461             }
1462           else
1463             {
1464               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1465
1466               /* verify speculative enqueue, maybe switch current next frame */
1467               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1468                                                to_next, n_left_to_next,
1469                                                bi0, next0);
1470             }
1471
1472           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1473             {
1474               from = vlib_frame_vector_args (frame);
1475               u32 len = vec_len (fragments_to_loopback);
1476               if (len <= VLIB_FRAME_SIZE)
1477                 {
1478                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1479                   n_left_from = len;
1480                   vec_reset_length (fragments_to_loopback);
1481                 }
1482               else
1483                 {
1484                   clib_memcpy (from,
1485                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1486                                sizeof (u32) * VLIB_FRAME_SIZE);
1487                   n_left_from = VLIB_FRAME_SIZE;
1488                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1489                 }
1490             }
1491        }
1492
1493       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1494     }
1495
1496   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1497                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1498                                pkts_processed);
1499
1500   nat_send_all_to_node (vm, fragments_to_drop, node,
1501                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1502                         SNAT_OUT2IN_NEXT_DROP);
1503
1504   vec_free (fragments_to_drop);
1505   vec_free (fragments_to_loopback);
1506   return frame->n_vectors;
1507 }
1508
1509 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1510   .function = nat44_out2in_reass_node_fn,
1511   .name = "nat44-out2in-reass",
1512   .vector_size = sizeof (u32),
1513   .format_trace = format_nat44_out2in_reass_trace,
1514   .type = VLIB_NODE_TYPE_INTERNAL,
1515
1516   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1517   .error_strings = snat_out2in_error_strings,
1518
1519   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1520
1521   /* edit / add dispositions here */
1522   .next_nodes = {
1523     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1524     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1525     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1526     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1527   },
1528 };
1529 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1530                               nat44_out2in_reass_node_fn);
1531
1532 /*******************************/
1533 /*** endpoint-dependent mode ***/
1534 /*******************************/
1535 typedef enum {
1536   NAT44_ED_OUT2IN_NEXT_DROP,
1537   NAT44_ED_OUT2IN_NEXT_LOOKUP,
1538   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1539   NAT44_ED_OUT2IN_NEXT_IN2OUT,
1540   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1541   NAT44_ED_OUT2IN_N_NEXT,
1542 } nat44_ed_out2in_next_t;
1543
1544 typedef struct {
1545   u32 sw_if_index;
1546   u32 next_index;
1547   u32 session_index;
1548   u32 is_slow_path;
1549 } nat44_ed_out2in_trace_t;
1550
1551 static u8 *
1552 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1553 {
1554   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1555   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1556   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1557   char * tag;
1558
1559   tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1560
1561   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1562               t->sw_if_index, t->next_index, t->session_index);
1563
1564   return s;
1565 }
1566
1567 static snat_session_t *
1568 create_session_for_static_mapping_ed (snat_main_t * sm,
1569                                       vlib_buffer_t *b,
1570                                       snat_session_key_t l_key,
1571                                       snat_session_key_t e_key,
1572                                       vlib_node_runtime_t * node,
1573                                       u32 thread_index,
1574                                       twice_nat_type_t twice_nat,
1575                                       u8 is_lb)
1576 {
1577   snat_session_t *s;
1578   snat_user_t *u;
1579   ip4_header_t *ip;
1580   udp_header_t *udp;
1581   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1582   clib_bihash_kv_16_8_t kv;
1583   snat_session_key_t eh_key;
1584   u32 address_index;
1585
1586   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1587     {
1588       b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1589       nat_log_notice ("maximum sessions exceeded");
1590       return 0;
1591     }
1592
1593   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1594   if (!u)
1595     {
1596       nat_log_warn ("create NAT user failed");
1597       return 0;
1598     }
1599
1600   s = nat_session_alloc_or_recycle (sm, u, thread_index);
1601   if (!s)
1602     {
1603       nat44_delete_user_with_no_session (sm, u, thread_index);
1604       nat_log_warn ("create NAT session failed");
1605       return 0;
1606     }
1607
1608   ip = vlib_buffer_get_current (b);
1609   udp = ip4_next_header (ip);
1610
1611   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1612   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1613   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1614   if (is_lb)
1615     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1616   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1617   s->outside_address_index = ~0;
1618   s->out2in = e_key;
1619   s->in2out = l_key;
1620   s->in2out.protocol = s->out2in.protocol;
1621   user_session_increment (sm, u, 1);
1622
1623   /* Add to lookup tables */
1624   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1625               e_key.fib_index, e_key.port, s->ext_host_port);
1626   kv.value = s - tsm->sessions;
1627   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
1628     nat_log_notice ("out2in-ed key add failed");
1629
1630   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1631       ip->src_address.as_u32 == l_key.addr.as_u32))
1632     {
1633       eh_key.protocol = e_key.protocol;
1634       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1635                                                thread_index, &eh_key,
1636                                                &address_index,
1637                                                sm->port_per_thread,
1638                                                tsm->snat_thread_index))
1639         {
1640           b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1641           nat44_delete_session (sm, s, thread_index);
1642           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1643             nat_log_notice ("out2in-ed key del failed");
1644           return 0;
1645         }
1646       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1647       s->ext_host_nat_port = eh_key.port;
1648       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1649       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1650                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
1651     }
1652   else
1653     {
1654       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1655                   l_key.fib_index, l_key.port, s->ext_host_port);
1656     }
1657   kv.value = s - tsm->sessions;
1658   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1659     nat_log_notice ("in2out-ed key add failed");
1660
1661   return s;
1662 }
1663
1664 static_always_inline int
1665 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1666 {
1667   icmp46_header_t *icmp0;
1668   nat_ed_ses_key_t key0;
1669   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1670   ip4_header_t *inner_ip0;
1671   void *l4_header = 0;
1672   icmp46_header_t *inner_icmp0;
1673
1674   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1675   echo0 = (icmp_echo_header_t *)(icmp0+1);
1676
1677   if (!icmp_is_error_message (icmp0))
1678     {
1679       key0.proto = IP_PROTOCOL_ICMP;
1680       key0.l_addr = ip0->dst_address;
1681       key0.r_addr = ip0->src_address;
1682       key0.l_port = echo0->identifier;
1683       key0.r_port = 0;
1684     }
1685   else
1686     {
1687       inner_ip0 = (ip4_header_t *)(echo0+1);
1688       l4_header = ip4_next_header (inner_ip0);
1689       key0.proto = inner_ip0->protocol;
1690       key0.l_addr = inner_ip0->src_address;
1691       key0.r_addr = inner_ip0->dst_address;
1692       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1693         {
1694         case SNAT_PROTOCOL_ICMP:
1695           inner_icmp0 = (icmp46_header_t*)l4_header;
1696           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1697           key0.l_port = inner_echo0->identifier;
1698           key0.r_port = 0;
1699           break;
1700         case SNAT_PROTOCOL_UDP:
1701         case SNAT_PROTOCOL_TCP:
1702           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1703           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1704           break;
1705         default:
1706           return -1;
1707         }
1708     }
1709   *p_key0 = key0;
1710   return 0;
1711 }
1712
1713 static int
1714 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1715               u16 dst_port, u32 thread_index, u32 rx_fib_index)
1716 {
1717   clib_bihash_kv_16_8_t kv, value;
1718   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1719
1720   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1721               rx_fib_index, src_port, dst_port);
1722   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1723     return 1;
1724
1725   return 0;
1726 }
1727
1728 static void
1729 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1730                       u32 thread_index)
1731 {
1732   nat_ed_ses_key_t key;
1733   clib_bihash_kv_16_8_t kv, value;
1734   udp_header_t *udp;
1735   snat_user_t *u;
1736   snat_session_t *s = 0;
1737   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1738   f64 now = vlib_time_now (sm->vlib_main);
1739
1740   if (ip->protocol == IP_PROTOCOL_ICMP)
1741     {
1742       if (icmp_get_ed_key (ip, &key))
1743         return;
1744     }
1745   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1746     {
1747       udp = ip4_next_header(ip);
1748       key.r_addr = ip->src_address;
1749       key.l_addr = ip->dst_address;
1750       key.proto = ip->protocol;
1751       key.l_port = udp->dst_port;
1752       key.r_port = udp->src_port;
1753     }
1754   else
1755     {
1756       key.r_addr = ip->src_address;
1757       key.l_addr = ip->dst_address;
1758       key.proto = ip->protocol;
1759       key.l_port = key.r_port = 0;
1760     }
1761   key.fib_index = 0;
1762   kv.key[0] = key.as_u64[0];
1763   kv.key[1] = key.as_u64[1];
1764
1765   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1766     {
1767       s = pool_elt_at_index (tsm->sessions, value.value);
1768     }
1769   else
1770     {
1771       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1772         return;
1773
1774       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1775                                   thread_index);
1776       if (!u)
1777         {
1778           nat_log_warn ("create NAT user failed");
1779           return;
1780         }
1781
1782       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1783       if (!s)
1784         {
1785           nat44_delete_user_with_no_session (sm, u, thread_index);
1786           nat_log_warn ("create NAT session failed");
1787           return;
1788         }
1789
1790       s->ext_host_addr = key.r_addr;
1791       s->ext_host_port = key.r_port;
1792       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1793       s->outside_address_index = ~0;
1794       s->out2in.addr = key.l_addr;
1795       s->out2in.port = key.l_port;
1796       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1797       s->out2in.fib_index = 0;
1798       s->in2out = s->out2in;
1799       user_session_increment (sm, u, 0);
1800
1801       kv.value = s - tsm->sessions;
1802       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1803         nat_log_notice ("in2out_ed key add failed");
1804     }
1805
1806   if (ip->protocol == IP_PROTOCOL_TCP)
1807     {
1808       tcp_header_t *tcp = ip4_next_header(ip);
1809       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1810         return;
1811     }
1812
1813   /* Per-user LRU list maintenance */
1814   nat44_session_update_lru (sm, s, thread_index);
1815   /* Accounting */
1816   nat44_session_update_counters (s, now, 0);
1817 }
1818
1819 u32
1820 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1821                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1822                       u8 * p_proto, snat_session_key_t * p_value,
1823                       u8 * p_dont_translate, void * d, void * e)
1824 {
1825   u32 next = ~0, sw_if_index, rx_fib_index;
1826   icmp46_header_t *icmp;
1827   nat_ed_ses_key_t key;
1828   clib_bihash_kv_16_8_t kv, value;
1829   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1830   snat_session_t *s = 0;
1831   u8 dont_translate = 0, is_addr_only;
1832   snat_session_key_t e_key, l_key;
1833
1834   icmp = (icmp46_header_t *) ip4_next_header (ip);
1835   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1836   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1837
1838   if (icmp_get_ed_key (ip, &key))
1839     {
1840       b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1841       next = SNAT_OUT2IN_NEXT_DROP;
1842       goto out;
1843     }
1844   key.fib_index = rx_fib_index;
1845   kv.key[0] = key.as_u64[0];
1846   kv.key[1] = key.as_u64[1];
1847
1848   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
1849     {
1850       /* Try to match static mapping */
1851       e_key.addr = ip->dst_address;
1852       e_key.port = key.l_port;
1853       e_key.protocol = ip_proto_to_snat_proto (key.proto);
1854       e_key.fib_index = rx_fib_index;
1855       if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
1856         {
1857           if (!sm->forwarding_enabled)
1858             {
1859               /* Don't NAT packet aimed at the intfc address */
1860               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
1861                                                   ip->dst_address.as_u32)))
1862                 {
1863                   dont_translate = 1;
1864                   goto out;
1865                 }
1866               b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1867               next = NAT44_ED_OUT2IN_NEXT_DROP;
1868               goto out;
1869             }
1870           else
1871             {
1872               dont_translate = 1;
1873               if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port,
1874                                thread_index, rx_fib_index))
1875                 {
1876                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1877                   goto out;
1878                 }
1879               create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
1880               goto out;
1881             }
1882         }
1883
1884       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1885                         (icmp->type != ICMP4_echo_request || !is_addr_only)))
1886         {
1887           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1888           next = NAT44_ED_OUT2IN_NEXT_DROP;
1889           goto out;
1890         }
1891
1892       /* Create session initiated by host from external network */
1893       s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
1894                                                thread_index, 0, 0);
1895
1896       if (!s)
1897         {
1898           next = NAT44_ED_OUT2IN_NEXT_DROP;
1899           goto out;
1900         }
1901     }
1902   else
1903     {
1904       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1905                         icmp->type != ICMP4_echo_request &&
1906                         !icmp_is_error_message (icmp)))
1907         {
1908           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1909           next = SNAT_OUT2IN_NEXT_DROP;
1910           goto out;
1911         }
1912
1913       s = pool_elt_at_index (tsm->sessions, value.value);
1914     }
1915
1916   *p_proto = ip_proto_to_snat_proto (key.proto);
1917 out:
1918   if (s)
1919     *p_value = s->in2out;
1920   *p_dont_translate = dont_translate;
1921   if (d)
1922     *(snat_session_t**)d = s;
1923   return next;
1924 }
1925
1926 static snat_session_t *
1927 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
1928                                vlib_buffer_t * b,
1929                                ip4_header_t * ip,
1930                                u32 rx_fib_index,
1931                                u32 thread_index,
1932                                f64 now,
1933                                vlib_main_t * vm,
1934                                vlib_node_runtime_t * node)
1935 {
1936   clib_bihash_kv_8_8_t kv, value;
1937   clib_bihash_kv_16_8_t s_kv, s_value;
1938   snat_static_mapping_t *m;
1939   u32 old_addr, new_addr;
1940   ip_csum_t sum;
1941   snat_session_t * s;
1942   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1943   snat_user_t *u;
1944
1945   old_addr = ip->dst_address.as_u32;
1946
1947   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
1948               rx_fib_index, 0, 0);
1949
1950   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
1951     {
1952       s = pool_elt_at_index (tsm->sessions, s_value.value);
1953       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1954     }
1955   else
1956     {
1957       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1958         {
1959           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1960           nat_log_notice ("maximum sessions exceeded");
1961           return 0;
1962         }
1963
1964       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1965       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1966         {
1967           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1968           return 0;
1969         }
1970
1971       m = pool_elt_at_index (sm->static_mappings, value.value);
1972
1973       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1974
1975       u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
1976                                   thread_index);
1977       if (!u)
1978         {
1979           nat_log_warn ("create NAT user failed");
1980           return 0;
1981         }
1982
1983       /* Create a new session */
1984       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1985       if (!s)
1986         {
1987           nat44_delete_user_with_no_session (sm, u, thread_index);
1988           nat_log_warn ("create NAT session failed");
1989           return 0;
1990         }
1991
1992       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1993       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1994       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1995       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1996       s->outside_address_index = ~0;
1997       s->out2in.addr.as_u32 = old_addr;
1998       s->out2in.fib_index = rx_fib_index;
1999       s->in2out.addr.as_u32 = new_addr;
2000       s->in2out.fib_index = m->fib_index;
2001       s->in2out.port = s->out2in.port = ip->protocol;
2002       user_session_increment (sm, u, 1);
2003
2004       /* Add to lookup tables */
2005       s_kv.value = s - tsm->sessions;
2006       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2007         nat_log_notice ("out2in key add failed");
2008
2009       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2010                   m->fib_index, 0, 0);
2011       s_kv.value = s - tsm->sessions;
2012       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2013         nat_log_notice ("in2out key add failed");
2014    }
2015
2016   /* Update IP checksum */
2017   sum = ip->checksum;
2018   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2019   ip->checksum = ip_csum_fold (sum);
2020
2021   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2022
2023   /* Accounting */
2024   nat44_session_update_counters (s, now,
2025                                  vlib_buffer_length_in_chain (vm, b));
2026   /* Per-user LRU list maintenance */
2027   nat44_session_update_lru (sm, s, thread_index);
2028
2029   return s;
2030 }
2031
2032 static inline uword
2033 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2034                                 vlib_node_runtime_t * node,
2035                                 vlib_frame_t * frame, int is_slow_path)
2036 {
2037   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2038   nat44_ed_out2in_next_t next_index;
2039   snat_main_t *sm = &snat_main;
2040   f64 now = vlib_time_now (vm);
2041   u32 thread_index = vm->thread_index;
2042   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2043
2044   stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2045     nat44_ed_out2in_node.index;
2046
2047   from = vlib_frame_vector_args (frame);
2048   n_left_from = frame->n_vectors;
2049   next_index = node->cached_next_index;
2050
2051   while (n_left_from > 0)
2052     {
2053       u32 n_left_to_next;
2054
2055       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2056
2057       while (n_left_from >= 4 && n_left_to_next >= 2)
2058         {
2059           u32 bi0, bi1;
2060           vlib_buffer_t *b0, *b1;
2061           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2062           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2063           u16 old_port0, new_port0, old_port1, new_port1;
2064           ip4_header_t *ip0, *ip1;
2065           udp_header_t *udp0, *udp1;
2066           tcp_header_t *tcp0, *tcp1;
2067           icmp46_header_t *icmp0, *icmp1;
2068           snat_session_t *s0 = 0, *s1 = 0;
2069           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2070           ip_csum_t sum0, sum1;
2071           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2072           u8 is_lb0, is_lb1;
2073           twice_nat_type_t twice_nat0, twice_nat1;
2074
2075           /* Prefetch next iteration. */
2076           {
2077             vlib_buffer_t * p2, * p3;
2078
2079             p2 = vlib_get_buffer (vm, from[2]);
2080             p3 = vlib_get_buffer (vm, from[3]);
2081
2082             vlib_prefetch_buffer_header (p2, LOAD);
2083             vlib_prefetch_buffer_header (p3, LOAD);
2084
2085             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2086             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2087           }
2088
2089           /* speculatively enqueue b0 and b1 to the current next frame */
2090           to_next[0] = bi0 = from[0];
2091           to_next[1] = bi1 = from[1];
2092           from += 2;
2093           to_next += 2;
2094           n_left_from -= 2;
2095           n_left_to_next -= 2;
2096
2097           b0 = vlib_get_buffer (vm, bi0);
2098           b1 = vlib_get_buffer (vm, bi1);
2099
2100           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2101           vnet_buffer (b0)->snat.flags = 0;
2102           ip0 = vlib_buffer_get_current (b0);
2103
2104           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2105           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2106                                                                sw_if_index0);
2107
2108           if (PREDICT_FALSE(ip0->ttl == 1))
2109             {
2110               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2111               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2112                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2113                                            0);
2114               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2115               goto trace00;
2116             }
2117
2118           udp0 = ip4_next_header (ip0);
2119           tcp0 = (tcp_header_t *) udp0;
2120           icmp0 = (icmp46_header_t *) udp0;
2121           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2122
2123           if (is_slow_path)
2124             {
2125               if (PREDICT_FALSE (proto0 == ~0))
2126                 {
2127                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2128                                                      thread_index, now, vm, node);
2129                   if (!sm->forwarding_enabled)
2130                     {
2131                       if (!s0)
2132                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2133                       goto trace00;
2134                     }
2135                 }
2136
2137               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2138                 {
2139                   next0 = icmp_out2in_slow_path
2140                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2141                      next0, now, thread_index, &s0);
2142                   goto trace00;
2143                 }
2144             }
2145           else
2146             {
2147               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2148                 {
2149                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2150                   goto trace00;
2151                 }
2152
2153               if (ip4_is_fragment (ip0))
2154                 {
2155                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2156                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2157                   goto trace00;
2158                 }
2159             }
2160
2161           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2162                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2163
2164           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2165             {
2166               if (is_slow_path)
2167                 {
2168                   /* Try to match static mapping by external address and port,
2169                      destination address and port in packet */
2170                   e_key0.addr = ip0->dst_address;
2171                   e_key0.port = udp0->dst_port;
2172                   e_key0.protocol = proto0;
2173                   e_key0.fib_index = rx_fib_index0;
2174                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2175                       &twice_nat0, &is_lb0))
2176                     {
2177                       /*
2178                        * Send DHCP packets to the ipv4 stack, or we won't
2179                        * be able to use dhcp client on the outside interface
2180                        */
2181                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2182                           && (udp0->dst_port ==
2183                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2184                         {
2185                           vnet_feature_next (&next0, b0);
2186                           goto trace00;
2187                         }
2188
2189                       if (!sm->forwarding_enabled)
2190                         {
2191                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2192                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2193                         }
2194                       else
2195                         {
2196                           if (next_src_nat(sm, ip0, ip0->protocol,
2197                                            udp0->src_port, udp0->dst_port,
2198                                            thread_index, rx_fib_index0))
2199                             {
2200                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2201                               goto trace00;
2202                             }
2203                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2204                                                 thread_index);
2205                         }
2206                       goto trace00;
2207                     }
2208
2209                   /* Create session initiated by host from external network */
2210                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2211                                                             e_key0, node,
2212                                                             thread_index,
2213                                                             twice_nat0, is_lb0);
2214
2215                   if (!s0)
2216                     {
2217                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2218                       goto trace00;
2219                     }
2220                 }
2221               else
2222                 {
2223                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2224                   goto trace00;
2225                 }
2226             }
2227           else
2228             {
2229               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2230             }
2231
2232           old_addr0 = ip0->dst_address.as_u32;
2233           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2234           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2235
2236           sum0 = ip0->checksum;
2237           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2238                                  dst_address);
2239           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2240             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2241                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2242                                    src_address);
2243           ip0->checksum = ip_csum_fold (sum0);
2244
2245           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2246             {
2247               old_port0 = tcp0->dst_port;
2248               new_port0 = tcp0->dst_port = s0->in2out.port;
2249
2250               sum0 = tcp0->checksum;
2251               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2252                                      dst_address);
2253               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2254                                      length);
2255               if (is_twice_nat_session (s0))
2256                 {
2257                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2258                                          s0->ext_host_nat_addr.as_u32,
2259                                          ip4_header_t, dst_address);
2260                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2261                                          s0->ext_host_nat_port, ip4_header_t,
2262                                          length);
2263                   tcp0->src_port = s0->ext_host_nat_port;
2264                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2265                 }
2266               tcp0->checksum = ip_csum_fold(sum0);
2267               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2268                 goto trace00;
2269             }
2270           else
2271             {
2272               udp0->dst_port = s0->in2out.port;
2273               if (is_twice_nat_session (s0))
2274                 {
2275                   udp0->src_port = s0->ext_host_nat_port;
2276                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2277                 }
2278               udp0->checksum = 0;
2279             }
2280
2281           /* Accounting */
2282           nat44_session_update_counters (s0, now,
2283                                          vlib_buffer_length_in_chain (vm, b0));
2284           /* Per-user LRU list maintenance */
2285           nat44_session_update_lru (sm, s0, thread_index);
2286
2287         trace00:
2288           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2289                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2290             {
2291               nat44_ed_out2in_trace_t *t =
2292                 vlib_add_trace (vm, node, b0, sizeof (*t));
2293               t->is_slow_path = is_slow_path;
2294               t->sw_if_index = sw_if_index0;
2295               t->next_index = next0;
2296               t->session_index = ~0;
2297               if (s0)
2298                 t->session_index = s0 - tsm->sessions;
2299             }
2300
2301           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2302
2303           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2304           vnet_buffer (b1)->snat.flags = 0;
2305           ip1 = vlib_buffer_get_current (b1);
2306
2307           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2308           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2309                                                                sw_if_index1);
2310
2311           if (PREDICT_FALSE(ip1->ttl == 1))
2312             {
2313               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2314               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2315                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2316                                            0);
2317               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2318               goto trace01;
2319             }
2320
2321           udp1 = ip4_next_header (ip1);
2322           tcp1 = (tcp_header_t *) udp1;
2323           icmp1 = (icmp46_header_t *) udp1;
2324           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2325
2326           if (is_slow_path)
2327             {
2328               if (PREDICT_FALSE (proto1 == ~0))
2329                 {
2330                   s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2331                                                      thread_index, now, vm, node);
2332                   if (!sm->forwarding_enabled)
2333                     {
2334                       if (!s1)
2335                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2336                       goto trace01;
2337                     }
2338                 }
2339
2340               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2341                 {
2342                   next1 = icmp_out2in_slow_path
2343                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2344                      next1, now, thread_index, &s1);
2345                   goto trace01;
2346                 }
2347             }
2348           else
2349             {
2350               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2351                 {
2352                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2353                   goto trace01;
2354                 }
2355
2356               if (ip4_is_fragment (ip1))
2357                 {
2358                   b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2359                   next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2360                   goto trace01;
2361                 }
2362             }
2363
2364           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2365                       rx_fib_index1, udp1->dst_port, udp1->src_port);
2366
2367           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2368             {
2369               if (is_slow_path)
2370                 {
2371                   /* Try to match static mapping by external address and port,
2372                      destination address and port in packet */
2373                   e_key1.addr = ip1->dst_address;
2374                   e_key1.port = udp1->dst_port;
2375                   e_key1.protocol = proto1;
2376                   e_key1.fib_index = rx_fib_index1;
2377                   if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2378                       &twice_nat1, &is_lb1))
2379                     {
2380                       /*
2381                        * Send DHCP packets to the ipv4 stack, or we won't
2382                        * be able to use dhcp client on the outside interface
2383                        */
2384                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2385                           && (udp1->dst_port ==
2386                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2387                         {
2388                           vnet_feature_next (&next1, b1);
2389                           goto trace01;
2390                         }
2391
2392                       if (!sm->forwarding_enabled)
2393                         {
2394                           b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2395                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2396                         }
2397                       else
2398                         {
2399                           if (next_src_nat(sm, ip1, ip1->protocol,
2400                                            udp1->src_port, udp1->dst_port,
2401                                            thread_index, rx_fib_index1))
2402                             {
2403                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2404                               goto trace01;
2405                             }
2406                           create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2407                                                 thread_index);
2408                         }
2409                       goto trace01;
2410                     }
2411
2412                   /* Create session initiated by host from external network */
2413                   s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2414                                                             e_key1, node,
2415                                                             thread_index,
2416                                                             twice_nat1, is_lb1);
2417
2418                   if (!s1)
2419                     {
2420                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2421                       goto trace01;
2422                     }
2423                 }
2424               else
2425                 {
2426                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2427                   goto trace01;
2428                 }
2429             }
2430           else
2431             {
2432               s1 = pool_elt_at_index (tsm->sessions, value1.value);
2433             }
2434
2435           old_addr1 = ip1->dst_address.as_u32;
2436           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2437           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2438
2439           sum1 = ip1->checksum;
2440           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2441                                  dst_address);
2442           if (PREDICT_FALSE (is_twice_nat_session (s1)))
2443             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2444                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
2445                                    src_address);
2446           ip1->checksum = ip_csum_fold (sum1);
2447
2448           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2449             {
2450               old_port1 = tcp1->dst_port;
2451               new_port1 = tcp1->dst_port = s1->in2out.port;
2452
2453               sum1 = tcp1->checksum;
2454               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2455                                      dst_address);
2456               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2457                                      length);
2458               if (is_twice_nat_session (s1))
2459                 {
2460                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2461                                          s1->ext_host_nat_addr.as_u32,
2462                                          ip4_header_t, dst_address);
2463                   sum1 = ip_csum_update (sum1, tcp1->src_port,
2464                                          s1->ext_host_nat_port, ip4_header_t,
2465                                          length);
2466                   tcp1->src_port = s1->ext_host_nat_port;
2467                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2468                 }
2469               tcp1->checksum = ip_csum_fold(sum1);
2470               if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2471                 goto trace01;
2472             }
2473           else
2474             {
2475               udp1->dst_port = s1->in2out.port;
2476               if (is_twice_nat_session (s1))
2477                 {
2478                   udp1->src_port = s1->ext_host_nat_port;
2479                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2480                 }
2481               udp1->checksum = 0;
2482             }
2483
2484           /* Accounting */
2485           nat44_session_update_counters (s1, now,
2486                                          vlib_buffer_length_in_chain (vm, b1));
2487           /* Per-user LRU list maintenance */
2488           nat44_session_update_lru (sm, s1, thread_index);
2489
2490         trace01:
2491           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2492                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2493             {
2494               nat44_ed_out2in_trace_t *t =
2495                 vlib_add_trace (vm, node, b1, sizeof (*t));
2496               t->is_slow_path = is_slow_path;
2497               t->sw_if_index = sw_if_index1;
2498               t->next_index = next1;
2499               t->session_index = ~0;
2500               if (s1)
2501                 t->session_index = s1 - tsm->sessions;
2502             }
2503
2504           pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2505
2506           /* verify speculative enqueues, maybe switch current next frame */
2507           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2508                                            to_next, n_left_to_next,
2509                                            bi0, bi1, next0, next1);
2510         }
2511
2512       while (n_left_from > 0 && n_left_to_next > 0)
2513         {
2514           u32 bi0;
2515           vlib_buffer_t *b0;
2516           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2517           u16 old_port0, new_port0;
2518           ip4_header_t *ip0;
2519           udp_header_t *udp0;
2520           tcp_header_t *tcp0;
2521           icmp46_header_t * icmp0;
2522           snat_session_t *s0 = 0;
2523           clib_bihash_kv_16_8_t kv0, value0;
2524           ip_csum_t sum0;
2525           snat_session_key_t e_key0, l_key0;
2526           u8 is_lb0;
2527           twice_nat_type_t twice_nat0;
2528
2529           /* speculatively enqueue b0 to the current next frame */
2530           bi0 = from[0];
2531           to_next[0] = bi0;
2532           from += 1;
2533           to_next += 1;
2534           n_left_from -= 1;
2535           n_left_to_next -= 1;
2536
2537           b0 = vlib_get_buffer (vm, bi0);
2538           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2539           vnet_buffer (b0)->snat.flags = 0;
2540           ip0 = vlib_buffer_get_current (b0);
2541
2542           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2543           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2544                                                                sw_if_index0);
2545
2546           if (PREDICT_FALSE(ip0->ttl == 1))
2547             {
2548               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2549               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2550                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2551                                            0);
2552               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2553               goto trace0;
2554             }
2555
2556           udp0 = ip4_next_header (ip0);
2557           tcp0 = (tcp_header_t *) udp0;
2558           icmp0 = (icmp46_header_t *) udp0;
2559           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2560
2561           if (is_slow_path)
2562             {
2563               if (PREDICT_FALSE (proto0 == ~0))
2564                 {
2565                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2566                                                      thread_index, now, vm, node);
2567                   if (!sm->forwarding_enabled)
2568                     {
2569                       if (!s0)
2570                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2571                       goto trace0;
2572                     }
2573                 }
2574
2575               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2576                 {
2577                   next0 = icmp_out2in_slow_path
2578                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2579                      next0, now, thread_index, &s0);
2580                   goto trace0;
2581                 }
2582             }
2583           else
2584             {
2585               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2586                 {
2587                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2588                   goto trace0;
2589                 }
2590
2591               if (ip4_is_fragment (ip0))
2592                 {
2593                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2594                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2595                   goto trace0;
2596                 }
2597             }
2598
2599           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2600                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2601
2602           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2603             {
2604               if (is_slow_path)
2605                 {
2606                   /* Try to match static mapping by external address and port,
2607                      destination address and port in packet */
2608                   e_key0.addr = ip0->dst_address;
2609                   e_key0.port = udp0->dst_port;
2610                   e_key0.protocol = proto0;
2611                   e_key0.fib_index = rx_fib_index0;
2612                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2613                       &twice_nat0, &is_lb0))
2614                     {
2615                       /*
2616                        * Send DHCP packets to the ipv4 stack, or we won't
2617                        * be able to use dhcp client on the outside interface
2618                        */
2619                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2620                           && (udp0->dst_port ==
2621                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2622                         {
2623                           vnet_feature_next (&next0, b0);
2624                           goto trace0;
2625                         }
2626
2627                       if (!sm->forwarding_enabled)
2628                         {
2629                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2630                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2631                         }
2632                       else
2633                         {
2634                           if (next_src_nat(sm, ip0, ip0->protocol,
2635                                            udp0->src_port, udp0->dst_port,
2636                                            thread_index, rx_fib_index0))
2637                             {
2638                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2639                               goto trace0;
2640                             }
2641                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2642                                                 thread_index);
2643                         }
2644                       goto trace0;
2645                     }
2646
2647                   /* Create session initiated by host from external network */
2648                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2649                                                             e_key0, node,
2650                                                             thread_index,
2651                                                             twice_nat0, is_lb0);
2652
2653                   if (!s0)
2654                     {
2655                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2656                       goto trace0;
2657                     }
2658                 }
2659               else
2660                 {
2661                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2662                   goto trace0;
2663                 }
2664             }
2665           else
2666             {
2667               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2668             }
2669
2670           old_addr0 = ip0->dst_address.as_u32;
2671           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2672           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2673
2674           sum0 = ip0->checksum;
2675           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2676                                  dst_address);
2677           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2678             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2679                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2680                                    src_address);
2681           ip0->checksum = ip_csum_fold (sum0);
2682
2683           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2684             {
2685               old_port0 = tcp0->dst_port;
2686               new_port0 = tcp0->dst_port = s0->in2out.port;
2687
2688               sum0 = tcp0->checksum;
2689               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2690                                      dst_address);
2691               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2692                                      length);
2693               if (is_twice_nat_session (s0))
2694                 {
2695                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2696                                          s0->ext_host_nat_addr.as_u32,
2697                                          ip4_header_t, dst_address);
2698                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2699                                          s0->ext_host_nat_port, ip4_header_t,
2700                                          length);
2701                   tcp0->src_port = s0->ext_host_nat_port;
2702                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2703                 }
2704               tcp0->checksum = ip_csum_fold(sum0);
2705               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2706                 goto trace0;
2707             }
2708           else
2709             {
2710               udp0->dst_port = s0->in2out.port;
2711               if (is_twice_nat_session (s0))
2712                 {
2713                   udp0->src_port = s0->ext_host_nat_port;
2714                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2715                 }
2716               udp0->checksum = 0;
2717             }
2718
2719           /* Accounting */
2720           nat44_session_update_counters (s0, now,
2721                                          vlib_buffer_length_in_chain (vm, b0));
2722           /* Per-user LRU list maintenance */
2723           nat44_session_update_lru (sm, s0, thread_index);
2724
2725         trace0:
2726           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2727                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2728             {
2729               nat44_ed_out2in_trace_t *t =
2730                 vlib_add_trace (vm, node, b0, sizeof (*t));
2731               t->is_slow_path = is_slow_path;
2732               t->sw_if_index = sw_if_index0;
2733               t->next_index = next0;
2734               t->session_index = ~0;
2735               if (s0)
2736                 t->session_index = s0 - tsm->sessions;
2737             }
2738
2739           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2740
2741           /* verify speculative enqueue, maybe switch current next frame */
2742           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2743                                            to_next, n_left_to_next,
2744                                            bi0, next0);
2745         }
2746
2747       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2748     }
2749
2750   vlib_node_increment_counter (vm, stats_node_index,
2751                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2752                                pkts_processed);
2753   return frame->n_vectors;
2754 }
2755
2756 static uword
2757 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2758                               vlib_node_runtime_t * node,
2759                               vlib_frame_t * frame)
2760 {
2761   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2762 }
2763
2764 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2765   .function = nat44_ed_out2in_fast_path_fn,
2766   .name = "nat44-ed-out2in",
2767   .vector_size = sizeof (u32),
2768   .format_trace = format_nat44_ed_out2in_trace,
2769   .type = VLIB_NODE_TYPE_INTERNAL,
2770
2771   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2772   .error_strings = snat_out2in_error_strings,
2773
2774   .runtime_data_bytes = sizeof (snat_runtime_t),
2775
2776   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2777
2778   /* edit / add dispositions here */
2779   .next_nodes = {
2780     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2781     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2782     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2783     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2784     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2785   },
2786 };
2787
2788 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2789
2790 static uword
2791 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2792                               vlib_node_runtime_t * node,
2793                               vlib_frame_t * frame)
2794 {
2795   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2796 }
2797
2798 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2799   .function = nat44_ed_out2in_slow_path_fn,
2800   .name = "nat44-ed-out2in-slowpath",
2801   .vector_size = sizeof (u32),
2802   .format_trace = format_nat44_ed_out2in_trace,
2803   .type = VLIB_NODE_TYPE_INTERNAL,
2804
2805   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2806   .error_strings = snat_out2in_error_strings,
2807
2808   .runtime_data_bytes = sizeof (snat_runtime_t),
2809
2810   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2811
2812   /* edit / add dispositions here */
2813   .next_nodes = {
2814     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2815     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2816     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2817     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2818     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2819   },
2820 };
2821
2822 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2823                               nat44_ed_out2in_slow_path_fn);
2824
2825 /**************************/
2826 /*** deterministic mode ***/
2827 /**************************/
2828 static uword
2829 snat_det_out2in_node_fn (vlib_main_t * vm,
2830                          vlib_node_runtime_t * node,
2831                          vlib_frame_t * frame)
2832 {
2833   u32 n_left_from, * from, * to_next;
2834   snat_out2in_next_t next_index;
2835   u32 pkts_processed = 0;
2836   snat_main_t * sm = &snat_main;
2837   u32 thread_index = vm->thread_index;
2838
2839   from = vlib_frame_vector_args (frame);
2840   n_left_from = frame->n_vectors;
2841   next_index = node->cached_next_index;
2842
2843   while (n_left_from > 0)
2844     {
2845       u32 n_left_to_next;
2846
2847       vlib_get_next_frame (vm, node, next_index,
2848                            to_next, n_left_to_next);
2849
2850       while (n_left_from >= 4 && n_left_to_next >= 2)
2851         {
2852           u32 bi0, bi1;
2853           vlib_buffer_t * b0, * b1;
2854           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2855           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2856           u32 sw_if_index0, sw_if_index1;
2857           ip4_header_t * ip0, * ip1;
2858           ip_csum_t sum0, sum1;
2859           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2860           u16 new_port0, old_port0, old_port1, new_port1;
2861           udp_header_t * udp0, * udp1;
2862           tcp_header_t * tcp0, * tcp1;
2863           u32 proto0, proto1;
2864           snat_det_out_key_t key0, key1;
2865           snat_det_map_t * dm0, * dm1;
2866           snat_det_session_t * ses0 = 0, * ses1 = 0;
2867           u32 rx_fib_index0, rx_fib_index1;
2868           icmp46_header_t * icmp0, * icmp1;
2869
2870           /* Prefetch next iteration. */
2871           {
2872             vlib_buffer_t * p2, * p3;
2873
2874             p2 = vlib_get_buffer (vm, from[2]);
2875             p3 = vlib_get_buffer (vm, from[3]);
2876
2877             vlib_prefetch_buffer_header (p2, LOAD);
2878             vlib_prefetch_buffer_header (p3, LOAD);
2879
2880             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2881             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2882           }
2883
2884           /* speculatively enqueue b0 and b1 to the current next frame */
2885           to_next[0] = bi0 = from[0];
2886           to_next[1] = bi1 = from[1];
2887           from += 2;
2888           to_next += 2;
2889           n_left_from -= 2;
2890           n_left_to_next -= 2;
2891
2892           b0 = vlib_get_buffer (vm, bi0);
2893           b1 = vlib_get_buffer (vm, bi1);
2894
2895           ip0 = vlib_buffer_get_current (b0);
2896           udp0 = ip4_next_header (ip0);
2897           tcp0 = (tcp_header_t *) udp0;
2898
2899           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2900
2901           if (PREDICT_FALSE(ip0->ttl == 1))
2902             {
2903               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2904               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2905                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2906                                            0);
2907               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2908               goto trace0;
2909             }
2910
2911           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2912
2913           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2914             {
2915               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2916               icmp0 = (icmp46_header_t *) udp0;
2917
2918               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2919                                   rx_fib_index0, node, next0, thread_index,
2920                                   &ses0, &dm0);
2921               goto trace0;
2922             }
2923
2924           key0.ext_host_addr = ip0->src_address;
2925           key0.ext_host_port = tcp0->src;
2926           key0.out_port = tcp0->dst;
2927
2928           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2929           if (PREDICT_FALSE(!dm0))
2930             {
2931               nat_log_info ("unknown dst address:  %U",
2932                             format_ip4_address, &ip0->dst_address);
2933               next0 = SNAT_OUT2IN_NEXT_DROP;
2934               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2935               goto trace0;
2936             }
2937
2938           snat_det_reverse(dm0, &ip0->dst_address,
2939                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2940
2941           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2942           if (PREDICT_FALSE(!ses0))
2943             {
2944               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2945                             format_ip4_address, &ip0->src_address,
2946                             clib_net_to_host_u16 (tcp0->src),
2947                             format_ip4_address, &ip0->dst_address,
2948                             clib_net_to_host_u16 (tcp0->dst),
2949                             format_ip4_address, &new_addr0);
2950               next0 = SNAT_OUT2IN_NEXT_DROP;
2951               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2952               goto trace0;
2953             }
2954           new_port0 = ses0->in_port;
2955
2956           old_addr0 = ip0->dst_address;
2957           ip0->dst_address = new_addr0;
2958           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2959
2960           sum0 = ip0->checksum;
2961           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2962                                  ip4_header_t,
2963                                  dst_address /* changed member */);
2964           ip0->checksum = ip_csum_fold (sum0);
2965
2966           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2967             {
2968               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2969                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2970               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2971                 snat_det_ses_close(dm0, ses0);
2972
2973               old_port0 = tcp0->dst;
2974               tcp0->dst = new_port0;
2975
2976               sum0 = tcp0->checksum;
2977               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2978                                      ip4_header_t,
2979                                      dst_address /* changed member */);
2980
2981               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2982                                      ip4_header_t /* cheat */,
2983                                      length /* changed member */);
2984               tcp0->checksum = ip_csum_fold(sum0);
2985             }
2986           else
2987             {
2988               old_port0 = udp0->dst_port;
2989               udp0->dst_port = new_port0;
2990               udp0->checksum = 0;
2991             }
2992
2993         trace0:
2994
2995           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2996                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2997             {
2998               snat_out2in_trace_t *t =
2999                  vlib_add_trace (vm, node, b0, sizeof (*t));
3000               t->sw_if_index = sw_if_index0;
3001               t->next_index = next0;
3002               t->session_index = ~0;
3003               if (ses0)
3004                 t->session_index = ses0 - dm0->sessions;
3005             }
3006
3007           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3008
3009           b1 = vlib_get_buffer (vm, bi1);
3010
3011           ip1 = vlib_buffer_get_current (b1);
3012           udp1 = ip4_next_header (ip1);
3013           tcp1 = (tcp_header_t *) udp1;
3014
3015           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3016
3017           if (PREDICT_FALSE(ip1->ttl == 1))
3018             {
3019               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3020               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3021                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3022                                            0);
3023               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3024               goto trace1;
3025             }
3026
3027           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3028
3029           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3030             {
3031               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3032               icmp1 = (icmp46_header_t *) udp1;
3033
3034               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3035                                   rx_fib_index1, node, next1, thread_index,
3036                                   &ses1, &dm1);
3037               goto trace1;
3038             }
3039
3040           key1.ext_host_addr = ip1->src_address;
3041           key1.ext_host_port = tcp1->src;
3042           key1.out_port = tcp1->dst;
3043
3044           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3045           if (PREDICT_FALSE(!dm1))
3046             {
3047               nat_log_info ("unknown dst address:  %U",
3048                             format_ip4_address, &ip1->dst_address);
3049               next1 = SNAT_OUT2IN_NEXT_DROP;
3050               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3051               goto trace1;
3052             }
3053
3054           snat_det_reverse(dm1, &ip1->dst_address,
3055                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
3056
3057           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3058           if (PREDICT_FALSE(!ses1))
3059             {
3060               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3061                             format_ip4_address, &ip1->src_address,
3062                             clib_net_to_host_u16 (tcp1->src),
3063                             format_ip4_address, &ip1->dst_address,
3064                             clib_net_to_host_u16 (tcp1->dst),
3065                             format_ip4_address, &new_addr1);
3066               next1 = SNAT_OUT2IN_NEXT_DROP;
3067               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3068               goto trace1;
3069             }
3070           new_port1 = ses1->in_port;
3071
3072           old_addr1 = ip1->dst_address;
3073           ip1->dst_address = new_addr1;
3074           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3075
3076           sum1 = ip1->checksum;
3077           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3078                                  ip4_header_t,
3079                                  dst_address /* changed member */);
3080           ip1->checksum = ip_csum_fold (sum1);
3081
3082           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3083             {
3084               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3085                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3086               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3087                 snat_det_ses_close(dm1, ses1);
3088
3089               old_port1 = tcp1->dst;
3090               tcp1->dst = new_port1;
3091
3092               sum1 = tcp1->checksum;
3093               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3094                                      ip4_header_t,
3095                                      dst_address /* changed member */);
3096
3097               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3098                                      ip4_header_t /* cheat */,
3099                                      length /* changed member */);
3100               tcp1->checksum = ip_csum_fold(sum1);
3101             }
3102           else
3103             {
3104               old_port1 = udp1->dst_port;
3105               udp1->dst_port = new_port1;
3106               udp1->checksum = 0;
3107             }
3108
3109         trace1:
3110
3111           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3112                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3113             {
3114               snat_out2in_trace_t *t =
3115                  vlib_add_trace (vm, node, b1, sizeof (*t));
3116               t->sw_if_index = sw_if_index1;
3117               t->next_index = next1;
3118               t->session_index = ~0;
3119               if (ses1)
3120                 t->session_index = ses1 - dm1->sessions;
3121             }
3122
3123           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3124
3125           /* verify speculative enqueues, maybe switch current next frame */
3126           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3127                                            to_next, n_left_to_next,
3128                                            bi0, bi1, next0, next1);
3129          }
3130
3131       while (n_left_from > 0 && n_left_to_next > 0)
3132         {
3133           u32 bi0;
3134           vlib_buffer_t * b0;
3135           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3136           u32 sw_if_index0;
3137           ip4_header_t * ip0;
3138           ip_csum_t sum0;
3139           ip4_address_t new_addr0, old_addr0;
3140           u16 new_port0, old_port0;
3141           udp_header_t * udp0;
3142           tcp_header_t * tcp0;
3143           u32 proto0;
3144           snat_det_out_key_t key0;
3145           snat_det_map_t * dm0;
3146           snat_det_session_t * ses0 = 0;
3147           u32 rx_fib_index0;
3148           icmp46_header_t * icmp0;
3149
3150           /* speculatively enqueue b0 to the current next frame */
3151           bi0 = from[0];
3152           to_next[0] = bi0;
3153           from += 1;
3154           to_next += 1;
3155           n_left_from -= 1;
3156           n_left_to_next -= 1;
3157
3158           b0 = vlib_get_buffer (vm, bi0);
3159
3160           ip0 = vlib_buffer_get_current (b0);
3161           udp0 = ip4_next_header (ip0);
3162           tcp0 = (tcp_header_t *) udp0;
3163
3164           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3165
3166           if (PREDICT_FALSE(ip0->ttl == 1))
3167             {
3168               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3169               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3170                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3171                                            0);
3172               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3173               goto trace00;
3174             }
3175
3176           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3177
3178           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3179             {
3180               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3181               icmp0 = (icmp46_header_t *) udp0;
3182
3183               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3184                                   rx_fib_index0, node, next0, thread_index,
3185                                   &ses0, &dm0);
3186               goto trace00;
3187             }
3188
3189           key0.ext_host_addr = ip0->src_address;
3190           key0.ext_host_port = tcp0->src;
3191           key0.out_port = tcp0->dst;
3192
3193           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3194           if (PREDICT_FALSE(!dm0))
3195             {
3196               nat_log_info ("unknown dst address:  %U",
3197                             format_ip4_address, &ip0->dst_address);
3198               next0 = SNAT_OUT2IN_NEXT_DROP;
3199               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3200               goto trace00;
3201             }
3202
3203           snat_det_reverse(dm0, &ip0->dst_address,
3204                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
3205
3206           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3207           if (PREDICT_FALSE(!ses0))
3208             {
3209               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3210                             format_ip4_address, &ip0->src_address,
3211                             clib_net_to_host_u16 (tcp0->src),
3212                             format_ip4_address, &ip0->dst_address,
3213                             clib_net_to_host_u16 (tcp0->dst),
3214                             format_ip4_address, &new_addr0);
3215               next0 = SNAT_OUT2IN_NEXT_DROP;
3216               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3217               goto trace00;
3218             }
3219           new_port0 = ses0->in_port;
3220
3221           old_addr0 = ip0->dst_address;
3222           ip0->dst_address = new_addr0;
3223           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3224
3225           sum0 = ip0->checksum;
3226           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3227                                  ip4_header_t,
3228                                  dst_address /* changed member */);
3229           ip0->checksum = ip_csum_fold (sum0);
3230
3231           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3232             {
3233               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3234                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3235               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3236                 snat_det_ses_close(dm0, ses0);
3237
3238               old_port0 = tcp0->dst;
3239               tcp0->dst = new_port0;
3240
3241               sum0 = tcp0->checksum;
3242               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3243                                      ip4_header_t,
3244                                      dst_address /* changed member */);
3245
3246               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3247                                      ip4_header_t /* cheat */,
3248                                      length /* changed member */);
3249               tcp0->checksum = ip_csum_fold(sum0);
3250             }
3251           else
3252             {
3253               old_port0 = udp0->dst_port;
3254               udp0->dst_port = new_port0;
3255               udp0->checksum = 0;
3256             }
3257
3258         trace00:
3259
3260           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3261                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3262             {
3263               snat_out2in_trace_t *t =
3264                  vlib_add_trace (vm, node, b0, sizeof (*t));
3265               t->sw_if_index = sw_if_index0;
3266               t->next_index = next0;
3267               t->session_index = ~0;
3268               if (ses0)
3269                 t->session_index = ses0 - dm0->sessions;
3270             }
3271
3272           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3273
3274           /* verify speculative enqueue, maybe switch current next frame */
3275           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3276                                            to_next, n_left_to_next,
3277                                            bi0, next0);
3278         }
3279
3280       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3281     }
3282
3283   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3284                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3285                                pkts_processed);
3286   return frame->n_vectors;
3287 }
3288
3289 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3290   .function = snat_det_out2in_node_fn,
3291   .name = "nat44-det-out2in",
3292   .vector_size = sizeof (u32),
3293   .format_trace = format_snat_out2in_trace,
3294   .type = VLIB_NODE_TYPE_INTERNAL,
3295
3296   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3297   .error_strings = snat_out2in_error_strings,
3298
3299   .runtime_data_bytes = sizeof (snat_runtime_t),
3300
3301   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3302
3303   /* edit / add dispositions here */
3304   .next_nodes = {
3305     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3306     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3307     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3308     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3309   },
3310 };
3311 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3312
3313 /**
3314  * Get address and port values to be used for ICMP packet translation
3315  * and create session if needed
3316  *
3317  * @param[in,out] sm             NAT main
3318  * @param[in,out] node           NAT node runtime
3319  * @param[in] thread_index       thread index
3320  * @param[in,out] b0             buffer containing packet to be translated
3321  * @param[out] p_proto           protocol used for matching
3322  * @param[out] p_value           address and port after NAT translation
3323  * @param[out] p_dont_translate  if packet should not be translated
3324  * @param d                      optional parameter
3325  * @param e                      optional parameter
3326  */
3327 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3328                           u32 thread_index, vlib_buffer_t *b0,
3329                           ip4_header_t *ip0, u8 *p_proto,
3330                           snat_session_key_t *p_value,
3331                           u8 *p_dont_translate, void *d, void *e)
3332 {
3333   icmp46_header_t *icmp0;
3334   u32 sw_if_index0;
3335   u8 protocol;
3336   snat_det_out_key_t key0;
3337   u8 dont_translate = 0;
3338   u32 next0 = ~0;
3339   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3340   ip4_header_t *inner_ip0;
3341   void *l4_header = 0;
3342   icmp46_header_t *inner_icmp0;
3343   snat_det_map_t * dm0 = 0;
3344   ip4_address_t new_addr0 = {{0}};
3345   snat_det_session_t * ses0 = 0;
3346   ip4_address_t out_addr;
3347
3348   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3349   echo0 = (icmp_echo_header_t *)(icmp0+1);
3350   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3351
3352   if (!icmp_is_error_message (icmp0))
3353     {
3354       protocol = SNAT_PROTOCOL_ICMP;
3355       key0.ext_host_addr = ip0->src_address;
3356       key0.ext_host_port = 0;
3357       key0.out_port = echo0->identifier;
3358       out_addr = ip0->dst_address;
3359     }
3360   else
3361     {
3362       inner_ip0 = (ip4_header_t *)(echo0+1);
3363       l4_header = ip4_next_header (inner_ip0);
3364       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3365       key0.ext_host_addr = inner_ip0->dst_address;
3366       out_addr = inner_ip0->src_address;
3367       switch (protocol)
3368         {
3369         case SNAT_PROTOCOL_ICMP:
3370           inner_icmp0 = (icmp46_header_t*)l4_header;
3371           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3372           key0.ext_host_port = 0;
3373           key0.out_port = inner_echo0->identifier;
3374           break;
3375         case SNAT_PROTOCOL_UDP:
3376         case SNAT_PROTOCOL_TCP:
3377           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3378           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3379           break;
3380         default:
3381           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3382           next0 = SNAT_OUT2IN_NEXT_DROP;
3383           goto out;
3384         }
3385     }
3386
3387   dm0 = snat_det_map_by_out(sm, &out_addr);
3388   if (PREDICT_FALSE(!dm0))
3389     {
3390       /* Don't NAT packet aimed at the intfc address */
3391       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3392                                           ip0->dst_address.as_u32)))
3393         {
3394           dont_translate = 1;
3395           goto out;
3396         }
3397       nat_log_info ("unknown dst address:  %U",
3398                     format_ip4_address, &ip0->dst_address);
3399       goto out;
3400     }
3401
3402   snat_det_reverse(dm0, &ip0->dst_address,
3403                    clib_net_to_host_u16(key0.out_port), &new_addr0);
3404
3405   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3406   if (PREDICT_FALSE(!ses0))
3407     {
3408       /* Don't NAT packet aimed at the intfc address */
3409       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3410                                           ip0->dst_address.as_u32)))
3411         {
3412           dont_translate = 1;
3413           goto out;
3414         }
3415       nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3416                     format_ip4_address, &key0.ext_host_addr,
3417                     clib_net_to_host_u16 (key0.ext_host_port),
3418                     format_ip4_address, &out_addr,
3419                     clib_net_to_host_u16 (key0.out_port),
3420                     format_ip4_address, &new_addr0);
3421       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3422       next0 = SNAT_OUT2IN_NEXT_DROP;
3423       goto out;
3424     }
3425
3426   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3427                     !icmp_is_error_message (icmp0)))
3428     {
3429       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3430       next0 = SNAT_OUT2IN_NEXT_DROP;
3431       goto out;
3432     }
3433
3434   goto out;
3435
3436 out:
3437   *p_proto = protocol;
3438   if (ses0)
3439     {
3440       p_value->addr = new_addr0;
3441       p_value->fib_index = sm->inside_fib_index;
3442       p_value->port = ses0->in_port;
3443     }
3444   *p_dont_translate = dont_translate;
3445   if (d)
3446     *(snat_det_session_t**)d = ses0;
3447   if (e)
3448     *(snat_det_map_t**)e = dm0;
3449   return next0;
3450 }
3451
3452 /**********************/
3453 /*** worker handoff ***/
3454 /**********************/
3455 static uword
3456 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3457                                vlib_node_runtime_t * node,
3458                                vlib_frame_t * frame)
3459 {
3460   snat_main_t *sm = &snat_main;
3461   vlib_thread_main_t *tm = vlib_get_thread_main ();
3462   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3463   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3464   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3465     = 0;
3466   vlib_frame_queue_elt_t *hf = 0;
3467   vlib_frame_queue_t *fq;
3468   vlib_frame_t *f = 0;
3469   int i;
3470   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3471   u32 next_worker_index = 0;
3472   u32 current_worker_index = ~0;
3473   u32 thread_index = vm->thread_index;
3474   vlib_frame_t *d = 0;
3475
3476   ASSERT (vec_len (sm->workers));
3477
3478   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3479     {
3480       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3481
3482       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3483                                tm->n_vlib_mains - 1,
3484                                (vlib_frame_queue_t *) (~0));
3485     }
3486
3487   from = vlib_frame_vector_args (frame);
3488   n_left_from = frame->n_vectors;
3489
3490   while (n_left_from > 0)
3491     {
3492       u32 bi0;
3493       vlib_buffer_t *b0;
3494       u32 sw_if_index0;
3495       u32 rx_fib_index0;
3496       ip4_header_t * ip0;
3497       u8 do_handoff;
3498
3499       bi0 = from[0];
3500       from += 1;
3501       n_left_from -= 1;
3502
3503       b0 = vlib_get_buffer (vm, bi0);
3504
3505       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3506       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3507
3508       ip0 = vlib_buffer_get_current (b0);
3509
3510       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3511
3512       if (PREDICT_FALSE (next_worker_index != thread_index))
3513         {
3514           do_handoff = 1;
3515
3516           if (next_worker_index != current_worker_index)
3517             {
3518               fq = is_vlib_frame_queue_congested (
3519                 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3520                 congested_handoff_queue_by_worker_index);
3521
3522               if (fq)
3523                 {
3524                   /* if this is 1st frame */
3525                   if (!d)
3526                     {
3527                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
3528                       to_next_drop = vlib_frame_vector_args (d);
3529                     }
3530
3531                   to_next_drop[0] = bi0;
3532                   to_next_drop += 1;
3533                   d->n_vectors++;
3534                   b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3535                   goto trace0;
3536                 }
3537
3538               if (hf)
3539                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3540
3541               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3542                                                       next_worker_index,
3543                                                       handoff_queue_elt_by_worker_index);
3544
3545               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3546               to_next_worker = &hf->buffer_index[hf->n_vectors];
3547               current_worker_index = next_worker_index;
3548             }
3549
3550           /* enqueue to correct worker thread */
3551           to_next_worker[0] = bi0;
3552           to_next_worker++;
3553           n_left_to_next_worker--;
3554
3555           if (n_left_to_next_worker == 0)
3556             {
3557               hf->n_vectors = VLIB_FRAME_SIZE;
3558               vlib_put_frame_queue_elt (hf);
3559               current_worker_index = ~0;
3560               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3561               hf = 0;
3562             }
3563         }
3564       else
3565         {
3566           do_handoff = 0;
3567           /* if this is 1st frame */
3568           if (!f)
3569             {
3570               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3571               to_next = vlib_frame_vector_args (f);
3572             }
3573
3574           to_next[0] = bi0;
3575           to_next += 1;
3576           f->n_vectors++;
3577         }
3578
3579 trace0:
3580       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3581                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3582         {
3583           snat_out2in_worker_handoff_trace_t *t =
3584             vlib_add_trace (vm, node, b0, sizeof (*t));
3585           t->next_worker_index = next_worker_index;
3586           t->do_handoff = do_handoff;
3587         }
3588     }
3589
3590   if (f)
3591     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3592
3593   if (d)
3594     vlib_put_frame_to_node (vm, sm->error_node_index, d);
3595
3596   if (hf)
3597     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3598
3599   /* Ship frames to the worker nodes */
3600   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3601     {
3602       if (handoff_queue_elt_by_worker_index[i])
3603         {
3604           hf = handoff_queue_elt_by_worker_index[i];
3605           /*
3606            * It works better to let the handoff node
3607            * rate-adapt, always ship the handoff queue element.
3608            */
3609           if (1 || hf->n_vectors == hf->last_n_vectors)
3610             {
3611               vlib_put_frame_queue_elt (hf);
3612               handoff_queue_elt_by_worker_index[i] = 0;
3613             }
3614           else
3615             hf->last_n_vectors = hf->n_vectors;
3616         }
3617       congested_handoff_queue_by_worker_index[i] =
3618         (vlib_frame_queue_t *) (~0);
3619     }
3620   hf = 0;
3621   current_worker_index = ~0;
3622   return frame->n_vectors;
3623 }
3624
3625 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3626   .function = snat_out2in_worker_handoff_fn,
3627   .name = "nat44-out2in-worker-handoff",
3628   .vector_size = sizeof (u32),
3629   .format_trace = format_snat_out2in_worker_handoff_trace,
3630   .type = VLIB_NODE_TYPE_INTERNAL,
3631
3632   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3633   .error_strings = snat_out2in_error_strings,
3634
3635   .n_next_nodes = 1,
3636
3637   .next_nodes = {
3638     [0] = "error-drop",
3639   },
3640 };
3641
3642 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3643
3644 static uword
3645 snat_out2in_fast_node_fn (vlib_main_t * vm,
3646                           vlib_node_runtime_t * node,
3647                           vlib_frame_t * frame)
3648 {
3649   u32 n_left_from, * from, * to_next;
3650   snat_out2in_next_t next_index;
3651   u32 pkts_processed = 0;
3652   snat_main_t * sm = &snat_main;
3653
3654   from = vlib_frame_vector_args (frame);
3655   n_left_from = frame->n_vectors;
3656   next_index = node->cached_next_index;
3657
3658   while (n_left_from > 0)
3659     {
3660       u32 n_left_to_next;
3661
3662       vlib_get_next_frame (vm, node, next_index,
3663                            to_next, n_left_to_next);
3664
3665       while (n_left_from > 0 && n_left_to_next > 0)
3666         {
3667           u32 bi0;
3668           vlib_buffer_t * b0;
3669           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3670           u32 sw_if_index0;
3671           ip4_header_t * ip0;
3672           ip_csum_t sum0;
3673           u32 new_addr0, old_addr0;
3674           u16 new_port0, old_port0;
3675           udp_header_t * udp0;
3676           tcp_header_t * tcp0;
3677           icmp46_header_t * icmp0;
3678           snat_session_key_t key0, sm0;
3679           u32 proto0;
3680           u32 rx_fib_index0;
3681
3682           /* speculatively enqueue b0 to the current next frame */
3683           bi0 = from[0];
3684           to_next[0] = bi0;
3685           from += 1;
3686           to_next += 1;
3687           n_left_from -= 1;
3688           n_left_to_next -= 1;
3689
3690           b0 = vlib_get_buffer (vm, bi0);
3691
3692           ip0 = vlib_buffer_get_current (b0);
3693           udp0 = ip4_next_header (ip0);
3694           tcp0 = (tcp_header_t *) udp0;
3695           icmp0 = (icmp46_header_t *) udp0;
3696
3697           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3698           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3699
3700           vnet_feature_next (&next0, b0);
3701
3702           if (PREDICT_FALSE(ip0->ttl == 1))
3703             {
3704               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3705               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3706                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3707                                            0);
3708               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3709               goto trace00;
3710             }
3711
3712           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3713
3714           if (PREDICT_FALSE (proto0 == ~0))
3715               goto trace00;
3716
3717           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3718             {
3719               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3720                                   rx_fib_index0, node, next0, ~0, 0, 0);
3721               goto trace00;
3722             }
3723
3724           key0.addr = ip0->dst_address;
3725           key0.port = udp0->dst_port;
3726           key0.fib_index = rx_fib_index0;
3727
3728           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3729             {
3730               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3731               goto trace00;
3732             }
3733
3734           new_addr0 = sm0.addr.as_u32;
3735           new_port0 = sm0.port;
3736           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3737           old_addr0 = ip0->dst_address.as_u32;
3738           ip0->dst_address.as_u32 = new_addr0;
3739
3740           sum0 = ip0->checksum;
3741           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3742                                  ip4_header_t,
3743                                  dst_address /* changed member */);
3744           ip0->checksum = ip_csum_fold (sum0);
3745
3746           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3747             {
3748                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3749                 {
3750                   old_port0 = tcp0->dst_port;
3751                   tcp0->dst_port = new_port0;
3752
3753                   sum0 = tcp0->checksum;
3754                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3755                                          ip4_header_t,
3756                                          dst_address /* changed member */);
3757
3758                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3759                                          ip4_header_t /* cheat */,
3760                                          length /* changed member */);
3761                   tcp0->checksum = ip_csum_fold(sum0);
3762                 }
3763               else
3764                 {
3765                   old_port0 = udp0->dst_port;
3766                   udp0->dst_port = new_port0;
3767                   udp0->checksum = 0;
3768                 }
3769             }
3770           else
3771             {
3772               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3773                 {
3774                   sum0 = tcp0->checksum;
3775                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3776                                          ip4_header_t,
3777                                          dst_address /* changed member */);
3778
3779                   tcp0->checksum = ip_csum_fold(sum0);
3780                 }
3781             }
3782
3783         trace00:
3784
3785           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3786                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3787             {
3788               snat_out2in_trace_t *t =
3789                  vlib_add_trace (vm, node, b0, sizeof (*t));
3790               t->sw_if_index = sw_if_index0;
3791               t->next_index = next0;
3792             }
3793
3794           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3795
3796           /* verify speculative enqueue, maybe switch current next frame */
3797           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3798                                            to_next, n_left_to_next,
3799                                            bi0, next0);
3800         }
3801
3802       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3803     }
3804
3805   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3806                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3807                                pkts_processed);
3808   return frame->n_vectors;
3809 }
3810
3811 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3812   .function = snat_out2in_fast_node_fn,
3813   .name = "nat44-out2in-fast",
3814   .vector_size = sizeof (u32),
3815   .format_trace = format_snat_out2in_fast_trace,
3816   .type = VLIB_NODE_TYPE_INTERNAL,
3817
3818   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3819   .error_strings = snat_out2in_error_strings,
3820
3821   .runtime_data_bytes = sizeof (snat_runtime_t),
3822
3823   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3824
3825   /* edit / add dispositions here */
3826   .next_nodes = {
3827     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3828     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3829     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3830     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3831   },
3832 };
3833 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);