src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29 #include <nat/nat_inlines.h>
  30
  31 #include <vppinfra/hash.h>
  32 #include <vppinfra/error.h>
  33 #include <vppinfra/elog.h>
  34
  35 typedef struct {
  36   u32 sw_if_index;
  37   u32 next_index;
  38   u32 session_index;
  39 } snat_out2in_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_out2in_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  52
  53   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  54               t->sw_if_index, t->next_index, t->session_index);
  55   return s;
  56 }
  57
  58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  63
  64   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  65               t->sw_if_index, t->next_index);
  66   return s;
  67 }
  68
  69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  70 {
  71   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  72   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  73   snat_out2in_worker_handoff_trace_t * t =
  74     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  75   char * m;
  76
  77   m = t->do_handoff ? "next worker" : "same worker";
  78   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  79
  80   return s;
  81 }
  82
  83 typedef struct {
  84   u32 sw_if_index;
  85   u32 next_index;
  86   u8 cached;
  87 } nat44_out2in_reass_trace_t;
  88
  89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  90 {
  91   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  92   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  93   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  94
  95   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  96               t->sw_if_index, t->next_index,
  97               t->cached ? "cached" : "translated");
  98
  99   return s;
 100 }
 101
 102 vlib_node_registration_t snat_out2in_node;
 103 vlib_node_registration_t snat_out2in_fast_node;
 104 vlib_node_registration_t snat_out2in_worker_handoff_node;
 105 vlib_node_registration_t snat_det_out2in_node;
 106 vlib_node_registration_t nat44_out2in_reass_node;
 107 vlib_node_registration_t nat44_ed_out2in_node;
 108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
 109
 110 #define foreach_snat_out2in_error                       \
 111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 112 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 113 _(OUT_OF_PORTS, "Out of ports")                         \
 114 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 115 _(NO_TRANSLATION, "No translation")                     \
 116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 117 _(DROP_FRAGMENT, "Drop fragment")                       \
 118 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 120 _(FQ_CONGESTED, "Handoff frame queue congested")
 121
 122 typedef enum {
 123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 124   foreach_snat_out2in_error
 125 #undef _
 126   SNAT_OUT2IN_N_ERROR,
 127 } snat_out2in_error_t;
 128
 129 static char * snat_out2in_error_strings[] = {
 130 #define _(sym,string) string,
 131   foreach_snat_out2in_error
 132 #undef _
 133 };
 134
 135 typedef enum {
 136   SNAT_OUT2IN_NEXT_DROP,
 137   SNAT_OUT2IN_NEXT_LOOKUP,
 138   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 139   SNAT_OUT2IN_NEXT_REASS,
 140   SNAT_OUT2IN_N_NEXT,
 141 } snat_out2in_next_t;
 142
 143 /**
 144  * @brief Create session for static mapping.
 145  *
 146  * Create NAT session initiated by host from external network with static
 147  * mapping.
 148  *
 149  * @param sm     NAT main.
 150  * @param b0     Vlib buffer.
 151  * @param in2out In2out NAT44 session key.
 152  * @param out2in Out2in NAT44 session key.
 153  * @param node   Vlib node.
 154  *
 155  * @returns SNAT session if successfully created otherwise 0.
 156  */
 157 static inline snat_session_t *
 158 create_session_for_static_mapping (snat_main_t *sm,
 159                                    vlib_buffer_t *b0,
 160                                    snat_session_key_t in2out,
 161                                    snat_session_key_t out2in,
 162                                    vlib_node_runtime_t * node,
 163                                    u32 thread_index)
 164 {
 165   snat_user_t *u;
 166   snat_session_t *s;
 167   clib_bihash_kv_8_8_t kv0;
 168   ip4_header_t *ip0;
 169   udp_header_t *udp0;
 170
 171   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 172     {
 173       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 174       nat_log_notice ("maximum sessions exceeded");
 175       return 0;
 176     }
 177
 178   ip0 = vlib_buffer_get_current (b0);
 179   udp0 = ip4_next_header (ip0);
 180
 181   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 182   if (!u)
 183     {
 184       nat_log_warn ("create NAT user failed");
 185       return 0;
 186     }
 187
 188   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 189   if (!s)
 190     {
 191       nat44_delete_user_with_no_session (sm, u, thread_index);
 192       nat_log_warn ("create NAT session failed");
 193       return 0;
 194     }
 195
 196   s->outside_address_index = ~0;
 197   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 198   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 199   s->ext_host_port = udp0->src_port;
 200   user_session_increment (sm, u, 1 /* static */);
 201   s->in2out = in2out;
 202   s->out2in = out2in;
 203   s->in2out.protocol = out2in.protocol;
 204
 205   /* Add to translation hashes */
 206   kv0.key = s->in2out.as_u64;
 207   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 208   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 209                                1 /* is_add */))
 210       nat_log_notice ("in2out key add failed");
 211
 212   kv0.key = s->out2in.as_u64;
 213
 214   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 215                                1 /* is_add */))
 216       nat_log_notice ("out2in key add failed");
 217
 218   /* log NAT event */
 219   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 220                                       s->out2in.addr.as_u32,
 221                                       s->in2out.protocol,
 222                                       s->in2out.port,
 223                                       s->out2in.port,
 224                                       s->in2out.fib_index);
 225    return s;
 226 }
 227
 228 static_always_inline
 229 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 230                                  snat_session_key_t *p_key0)
 231 {
 232   icmp46_header_t *icmp0;
 233   snat_session_key_t key0;
 234   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 235   ip4_header_t *inner_ip0;
 236   void *l4_header = 0;
 237   icmp46_header_t *inner_icmp0;
 238
 239   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 240   echo0 = (icmp_echo_header_t *)(icmp0+1);
 241
 242   if (!icmp_is_error_message (icmp0))
 243     {
 244       key0.protocol = SNAT_PROTOCOL_ICMP;
 245       key0.addr = ip0->dst_address;
 246       key0.port = echo0->identifier;
 247     }
 248   else
 249     {
 250       inner_ip0 = (ip4_header_t *)(echo0+1);
 251       l4_header = ip4_next_header (inner_ip0);
 252       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 253       key0.addr = inner_ip0->src_address;
 254       switch (key0.protocol)
 255         {
 256         case SNAT_PROTOCOL_ICMP:
 257           inner_icmp0 = (icmp46_header_t*)l4_header;
 258           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 259           key0.port = inner_echo0->identifier;
 260           break;
 261         case SNAT_PROTOCOL_UDP:
 262         case SNAT_PROTOCOL_TCP:
 263           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 264           break;
 265         default:
 266           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 267         }
 268     }
 269   *p_key0 = key0;
 270   return -1; /* success */
 271 }
 272
 273 /**
 274  * Get address and port values to be used for ICMP packet translation
 275  * and create session if needed
 276  *
 277  * @param[in,out] sm             NAT main
 278  * @param[in,out] node           NAT node runtime
 279  * @param[in] thread_index       thread index
 280  * @param[in,out] b0             buffer containing packet to be translated
 281  * @param[out] p_proto           protocol used for matching
 282  * @param[out] p_value           address and port after NAT translation
 283  * @param[out] p_dont_translate  if packet should not be translated
 284  * @param d                      optional parameter
 285  * @param e                      optional parameter
 286  */
 287 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 288                            u32 thread_index, vlib_buffer_t *b0,
 289                            ip4_header_t *ip0, u8 *p_proto,
 290                            snat_session_key_t *p_value,
 291                            u8 *p_dont_translate, void *d, void *e)
 292 {
 293   icmp46_header_t *icmp0;
 294   u32 sw_if_index0;
 295   u32 rx_fib_index0;
 296   snat_session_key_t key0;
 297   snat_session_key_t sm0;
 298   snat_session_t *s0 = 0;
 299   u8 dont_translate = 0;
 300   clib_bihash_kv_8_8_t kv0, value0;
 301   u8 is_addr_only;
 302   u32 next0 = ~0;
 303   int err;
 304
 305   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 306   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 307   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 308
 309   key0.protocol = 0;
 310
 311   err = icmp_get_key (ip0, &key0);
 312   if (err != -1)
 313     {
 314       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 315       next0 = SNAT_OUT2IN_NEXT_DROP;
 316       goto out;
 317     }
 318   key0.fib_index = rx_fib_index0;
 319
 320   kv0.key = key0.as_u64;
 321
 322   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 323                               &value0))
 324     {
 325       /* Try to match static mapping by external address and port,
 326          destination address and port in packet */
 327       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 328         {
 329           if (!sm->forwarding_enabled)
 330             {
 331               /* Don't NAT packet aimed at the intfc address */
 332               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 333                                                   ip0->dst_address.as_u32)))
 334                 {
 335                   dont_translate = 1;
 336                   goto out;
 337                 }
 338               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 339               next0 = SNAT_OUT2IN_NEXT_DROP;
 340               goto out;
 341             }
 342           else
 343             {
 344               dont_translate = 1;
 345               goto out;
 346             }
 347         }
 348
 349       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 350                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 351         {
 352           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 353           next0 = SNAT_OUT2IN_NEXT_DROP;
 354           goto out;
 355         }
 356
 357       /* Create session initiated by host from external network */
 358       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 359                                              node, thread_index);
 360
 361       if (!s0)
 362         {
 363           next0 = SNAT_OUT2IN_NEXT_DROP;
 364           goto out;
 365         }
 366     }
 367   else
 368     {
 369       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 370                         icmp0->type != ICMP4_echo_request &&
 371                         !icmp_is_error_message (icmp0)))
 372         {
 373           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 374           next0 = SNAT_OUT2IN_NEXT_DROP;
 375           goto out;
 376         }
 377
 378       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 379                               value0.value);
 380     }
 381
 382 out:
 383   *p_proto = key0.protocol;
 384   if (s0)
 385     *p_value = s0->in2out;
 386   *p_dont_translate = dont_translate;
 387   if (d)
 388     *(snat_session_t**)d = s0;
 389   return next0;
 390 }
 391
 392 /**
 393  * Get address and port values to be used for ICMP packet translation
 394  *
 395  * @param[in] sm                 NAT main
 396  * @param[in,out] node           NAT node runtime
 397  * @param[in] thread_index       thread index
 398  * @param[in,out] b0             buffer containing packet to be translated
 399  * @param[out] p_proto           protocol used for matching
 400  * @param[out] p_value           address and port after NAT translation
 401  * @param[out] p_dont_translate  if packet should not be translated
 402  * @param d                      optional parameter
 403  * @param e                      optional parameter
 404  */
 405 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 406                            u32 thread_index, vlib_buffer_t *b0,
 407                            ip4_header_t *ip0, u8 *p_proto,
 408                            snat_session_key_t *p_value,
 409                            u8 *p_dont_translate, void *d, void *e)
 410 {
 411   icmp46_header_t *icmp0;
 412   u32 sw_if_index0;
 413   u32 rx_fib_index0;
 414   snat_session_key_t key0;
 415   snat_session_key_t sm0;
 416   u8 dont_translate = 0;
 417   u8 is_addr_only;
 418   u32 next0 = ~0;
 419   int err;
 420
 421   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 422   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 423   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 424
 425   err = icmp_get_key (ip0, &key0);
 426   if (err != -1)
 427     {
 428       b0->error = node->errors[err];
 429       next0 = SNAT_OUT2IN_NEXT_DROP;
 430       goto out2;
 431     }
 432   key0.fib_index = rx_fib_index0;
 433
 434   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 435     {
 436       /* Don't NAT packet aimed at the intfc address */
 437       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 438         {
 439           dont_translate = 1;
 440           goto out;
 441         }
 442       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 443       next0 = SNAT_OUT2IN_NEXT_DROP;
 444       goto out;
 445     }
 446
 447   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 448                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 449                     !icmp_is_error_message (icmp0)))
 450     {
 451       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 452       next0 = SNAT_OUT2IN_NEXT_DROP;
 453       goto out;
 454     }
 455
 456 out:
 457   *p_value = sm0;
 458 out2:
 459   *p_proto = key0.protocol;
 460   *p_dont_translate = dont_translate;
 461   return next0;
 462 }
 463
 464 static inline u32 icmp_out2in (snat_main_t *sm,
 465                                vlib_buffer_t * b0,
 466                                ip4_header_t * ip0,
 467                                icmp46_header_t * icmp0,
 468                                u32 sw_if_index0,
 469                                u32 rx_fib_index0,
 470                                vlib_node_runtime_t * node,
 471                                u32 next0,
 472                                u32 thread_index,
 473                                void *d,
 474                                void *e)
 475 {
 476   snat_session_key_t sm0;
 477   u8 protocol;
 478   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 479   ip4_header_t *inner_ip0 = 0;
 480   void *l4_header = 0;
 481   icmp46_header_t *inner_icmp0;
 482   u8 dont_translate;
 483   u32 new_addr0, old_addr0;
 484   u16 old_id0, new_id0;
 485   ip_csum_t sum0;
 486   u16 checksum0;
 487   u32 next0_tmp;
 488
 489   echo0 = (icmp_echo_header_t *)(icmp0+1);
 490
 491   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 492                                        &protocol, &sm0, &dont_translate, d, e);
 493   if (next0_tmp != ~0)
 494     next0 = next0_tmp;
 495   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 496     goto out;
 497
 498   sum0 = ip_incremental_checksum (0, icmp0,
 499                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 500   checksum0 = ~ip_csum_fold (sum0);
 501   if (checksum0 != 0 && checksum0 != 0xffff)
 502     {
 503       next0 = SNAT_OUT2IN_NEXT_DROP;
 504       goto out;
 505     }
 506
 507   old_addr0 = ip0->dst_address.as_u32;
 508   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 509   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 510
 511   sum0 = ip0->checksum;
 512   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 513                          dst_address /* changed member */);
 514   ip0->checksum = ip_csum_fold (sum0);
 515
 516   if (icmp0->checksum == 0)
 517     icmp0->checksum = 0xffff;
 518
 519   if (!icmp_is_error_message (icmp0))
 520     {
 521       new_id0 = sm0.port;
 522       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 523         {
 524           old_id0 = echo0->identifier;
 525           new_id0 = sm0.port;
 526           echo0->identifier = new_id0;
 527
 528           sum0 = icmp0->checksum;
 529           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 530                                  identifier /* changed member */);
 531           icmp0->checksum = ip_csum_fold (sum0);
 532         }
 533     }
 534   else
 535     {
 536       inner_ip0 = (ip4_header_t *)(echo0+1);
 537       l4_header = ip4_next_header (inner_ip0);
 538
 539       if (!ip4_header_checksum_is_valid (inner_ip0))
 540         {
 541           next0 = SNAT_OUT2IN_NEXT_DROP;
 542           goto out;
 543         }
 544
 545       old_addr0 = inner_ip0->src_address.as_u32;
 546       inner_ip0->src_address = sm0.addr;
 547       new_addr0 = inner_ip0->src_address.as_u32;
 548
 549       sum0 = icmp0->checksum;
 550       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 551                              src_address /* changed member */);
 552       icmp0->checksum = ip_csum_fold (sum0);
 553
 554       switch (protocol)
 555         {
 556         case SNAT_PROTOCOL_ICMP:
 557           inner_icmp0 = (icmp46_header_t*)l4_header;
 558           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 559
 560           old_id0 = inner_echo0->identifier;
 561           new_id0 = sm0.port;
 562           inner_echo0->identifier = new_id0;
 563
 564           sum0 = icmp0->checksum;
 565           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 566                                  identifier);
 567           icmp0->checksum = ip_csum_fold (sum0);
 568           break;
 569         case SNAT_PROTOCOL_UDP:
 570         case SNAT_PROTOCOL_TCP:
 571           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 572           new_id0 = sm0.port;
 573           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 574
 575           sum0 = icmp0->checksum;
 576           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 577                                  src_port);
 578           icmp0->checksum = ip_csum_fold (sum0);
 579           break;
 580         default:
 581           ASSERT(0);
 582         }
 583     }
 584
 585 out:
 586   return next0;
 587 }
 588
 589
 590 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 591                                          vlib_buffer_t * b0,
 592                                          ip4_header_t * ip0,
 593                                          icmp46_header_t * icmp0,
 594                                          u32 sw_if_index0,
 595                                          u32 rx_fib_index0,
 596                                          vlib_node_runtime_t * node,
 597                                          u32 next0, f64 now,
 598                                          u32 thread_index,
 599                                          snat_session_t ** p_s0)
 600 {
 601   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 602                       next0, thread_index, p_s0, 0);
 603   snat_session_t * s0 = *p_s0;
 604   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 605     {
 606       /* Accounting */
 607       nat44_session_update_counters (s0, now,
 608                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
 609       /* Per-user LRU list maintenance */
 610       nat44_session_update_lru (sm, s0, thread_index);
 611     }
 612   return next0;
 613 }
 614
 615 static int
 616 nat_out2in_sm_unknown_proto (snat_main_t *sm,
 617                              vlib_buffer_t * b,
 618                              ip4_header_t * ip,
 619                              u32 rx_fib_index)
 620 {
 621   clib_bihash_kv_8_8_t kv, value;
 622   snat_static_mapping_t *m;
 623   snat_session_key_t m_key;
 624   u32 old_addr, new_addr;
 625   ip_csum_t sum;
 626
 627   m_key.addr = ip->dst_address;
 628   m_key.port = 0;
 629   m_key.protocol = 0;
 630   m_key.fib_index = 0;
 631   kv.key = m_key.as_u64;
 632   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 633     return 1;
 634
 635   m = pool_elt_at_index (sm->static_mappings, value.value);
 636
 637   old_addr = ip->dst_address.as_u32;
 638   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 639   sum = ip->checksum;
 640   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 641   ip->checksum = ip_csum_fold (sum);
 642
 643   vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 644   return 0;
 645 }
 646
 647 static uword
 648 snat_out2in_node_fn (vlib_main_t * vm,
 649                   vlib_node_runtime_t * node,
 650                   vlib_frame_t * frame)
 651 {
 652   u32 n_left_from, * from, * to_next;
 653   snat_out2in_next_t next_index;
 654   u32 pkts_processed = 0;
 655   snat_main_t * sm = &snat_main;
 656   f64 now = vlib_time_now (vm);
 657   u32 thread_index = vm->thread_index;
 658
 659   from = vlib_frame_vector_args (frame);
 660   n_left_from = frame->n_vectors;
 661   next_index = node->cached_next_index;
 662
 663   while (n_left_from > 0)
 664     {
 665       u32 n_left_to_next;
 666
 667       vlib_get_next_frame (vm, node, next_index,
 668                            to_next, n_left_to_next);
 669
 670       while (n_left_from >= 4 && n_left_to_next >= 2)
 671         {
 672           u32 bi0, bi1;
 673           vlib_buffer_t * b0, * b1;
 674           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 675           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 676           u32 sw_if_index0, sw_if_index1;
 677           ip4_header_t * ip0, *ip1;
 678           ip_csum_t sum0, sum1;
 679           u32 new_addr0, old_addr0;
 680           u16 new_port0, old_port0;
 681           u32 new_addr1, old_addr1;
 682           u16 new_port1, old_port1;
 683           udp_header_t * udp0, * udp1;
 684           tcp_header_t * tcp0, * tcp1;
 685           icmp46_header_t * icmp0, * icmp1;
 686           snat_session_key_t key0, key1, sm0, sm1;
 687           u32 rx_fib_index0, rx_fib_index1;
 688           u32 proto0, proto1;
 689           snat_session_t * s0 = 0, * s1 = 0;
 690           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 691
 692           /* Prefetch next iteration. */
 693           {
 694             vlib_buffer_t * p2, * p3;
 695
 696             p2 = vlib_get_buffer (vm, from[2]);
 697             p3 = vlib_get_buffer (vm, from[3]);
 698
 699             vlib_prefetch_buffer_header (p2, LOAD);
 700             vlib_prefetch_buffer_header (p3, LOAD);
 701
 702             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 703             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 704           }
 705
 706           /* speculatively enqueue b0 and b1 to the current next frame */
 707           to_next[0] = bi0 = from[0];
 708           to_next[1] = bi1 = from[1];
 709           from += 2;
 710           to_next += 2;
 711           n_left_from -= 2;
 712           n_left_to_next -= 2;
 713
 714           b0 = vlib_get_buffer (vm, bi0);
 715           b1 = vlib_get_buffer (vm, bi1);
 716
 717           vnet_buffer (b0)->snat.flags = 0;
 718           vnet_buffer (b1)->snat.flags = 0;
 719
 720           ip0 = vlib_buffer_get_current (b0);
 721           udp0 = ip4_next_header (ip0);
 722           tcp0 = (tcp_header_t *) udp0;
 723           icmp0 = (icmp46_header_t *) udp0;
 724
 725           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 726           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 727                                    sw_if_index0);
 728
 729           if (PREDICT_FALSE(ip0->ttl == 1))
 730             {
 731               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 732               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 733                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 734                                            0);
 735               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 736               goto trace0;
 737             }
 738
 739           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 740
 741           if (PREDICT_FALSE (proto0 == ~0))
 742             {
 743               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
 744                 {
 745                   if (!sm->forwarding_enabled)
 746                     {
 747                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 748                       next0 = SNAT_OUT2IN_NEXT_DROP;
 749                     }
 750                 }
 751               goto trace0;
 752             }
 753
 754           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 755             {
 756               next0 = icmp_out2in_slow_path
 757                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 758                  next0, now, thread_index, &s0);
 759               goto trace0;
 760             }
 761
 762           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
 763             {
 764               next0 = SNAT_OUT2IN_NEXT_REASS;
 765               goto trace0;
 766             }
 767
 768           key0.addr = ip0->dst_address;
 769           key0.port = udp0->dst_port;
 770           key0.protocol = proto0;
 771           key0.fib_index = rx_fib_index0;
 772
 773           kv0.key = key0.as_u64;
 774
 775           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 776                                       &kv0, &value0))
 777             {
 778               /* Try to match static mapping by external address and port,
 779                  destination address and port in packet */
 780               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 781                 {
 782                   /*
 783                    * Send DHCP packets to the ipv4 stack, or we won't
 784                    * be able to use dhcp client on the outside interface
 785                    */
 786                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 787                       && (udp0->dst_port ==
 788                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 789                     {
 790                       vnet_feature_next (&next0, b0);
 791                       goto trace0;
 792                     }
 793
 794                   if (!sm->forwarding_enabled)
 795                     {
 796                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 797                       next0 = SNAT_OUT2IN_NEXT_DROP;
 798                     }
 799                   goto trace0;
 800                 }
 801
 802               /* Create session initiated by host from external network */
 803               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 804                                                      thread_index);
 805               if (!s0)
 806                 {
 807                   next0 = SNAT_OUT2IN_NEXT_DROP;
 808                   goto trace0;
 809                 }
 810             }
 811           else
 812             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 813                                     value0.value);
 814
 815           old_addr0 = ip0->dst_address.as_u32;
 816           ip0->dst_address = s0->in2out.addr;
 817           new_addr0 = ip0->dst_address.as_u32;
 818           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 819
 820           sum0 = ip0->checksum;
 821           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 822                                  ip4_header_t,
 823                                  dst_address /* changed member */);
 824           ip0->checksum = ip_csum_fold (sum0);
 825
 826           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 827             {
 828               old_port0 = tcp0->dst_port;
 829               tcp0->dst_port = s0->in2out.port;
 830               new_port0 = tcp0->dst_port;
 831
 832               sum0 = tcp0->checksum;
 833               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 834                                      ip4_header_t,
 835                                      dst_address /* changed member */);
 836
 837               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 838                                      ip4_header_t /* cheat */,
 839                                      length /* changed member */);
 840               tcp0->checksum = ip_csum_fold(sum0);
 841             }
 842           else
 843             {
 844               old_port0 = udp0->dst_port;
 845               udp0->dst_port = s0->in2out.port;
 846               udp0->checksum = 0;
 847             }
 848
 849           /* Accounting */
 850           nat44_session_update_counters (s0, now,
 851                                          vlib_buffer_length_in_chain (vm, b0));
 852           /* Per-user LRU list maintenance */
 853           nat44_session_update_lru (sm, s0, thread_index);
 854         trace0:
 855
 856           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 857                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 858             {
 859               snat_out2in_trace_t *t =
 860                  vlib_add_trace (vm, node, b0, sizeof (*t));
 861               t->sw_if_index = sw_if_index0;
 862               t->next_index = next0;
 863               t->session_index = ~0;
 864               if (s0)
 865                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 866             }
 867
 868           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 869
 870
 871           ip1 = vlib_buffer_get_current (b1);
 872           udp1 = ip4_next_header (ip1);
 873           tcp1 = (tcp_header_t *) udp1;
 874           icmp1 = (icmp46_header_t *) udp1;
 875
 876           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 877           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 878                                    sw_if_index1);
 879
 880           if (PREDICT_FALSE(ip1->ttl == 1))
 881             {
 882               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 883               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 884                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 885                                            0);
 886               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 887               goto trace1;
 888             }
 889
 890           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 891
 892           if (PREDICT_FALSE (proto1 == ~0))
 893             {
 894               if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
 895                 {
 896                   if (!sm->forwarding_enabled)
 897                     {
 898                       b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 899                       next1 = SNAT_OUT2IN_NEXT_DROP;
 900                     }
 901                 }
 902               goto trace1;
 903             }
 904
 905           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 906             {
 907               next1 = icmp_out2in_slow_path
 908                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 909                  next1, now, thread_index, &s1);
 910               goto trace1;
 911             }
 912
 913           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
 914             {
 915               next1 = SNAT_OUT2IN_NEXT_REASS;
 916               goto trace1;
 917             }
 918
 919           key1.addr = ip1->dst_address;
 920           key1.port = udp1->dst_port;
 921           key1.protocol = proto1;
 922           key1.fib_index = rx_fib_index1;
 923
 924           kv1.key = key1.as_u64;
 925
 926           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 927                                       &kv1, &value1))
 928             {
 929               /* Try to match static mapping by external address and port,
 930                  destination address and port in packet */
 931               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
 932                 {
 933                   /*
 934                    * Send DHCP packets to the ipv4 stack, or we won't
 935                    * be able to use dhcp client on the outside interface
 936                    */
 937                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
 938                       && (udp1->dst_port ==
 939                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 940                     {
 941                       vnet_feature_next (&next1, b1);
 942                       goto trace1;
 943                     }
 944
 945                   if (!sm->forwarding_enabled)
 946                     {
 947                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 948                       next1 = SNAT_OUT2IN_NEXT_DROP;
 949                     }
 950                   goto trace1;
 951                 }
 952
 953               /* Create session initiated by host from external network */
 954               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 955                                                      thread_index);
 956               if (!s1)
 957                 {
 958                   next1 = SNAT_OUT2IN_NEXT_DROP;
 959                   goto trace1;
 960                 }
 961             }
 962           else
 963             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 964                                     value1.value);
 965
 966           old_addr1 = ip1->dst_address.as_u32;
 967           ip1->dst_address = s1->in2out.addr;
 968           new_addr1 = ip1->dst_address.as_u32;
 969           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 970
 971           sum1 = ip1->checksum;
 972           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 973                                  ip4_header_t,
 974                                  dst_address /* changed member */);
 975           ip1->checksum = ip_csum_fold (sum1);
 976
 977           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 978             {
 979               old_port1 = tcp1->dst_port;
 980               tcp1->dst_port = s1->in2out.port;
 981               new_port1 = tcp1->dst_port;
 982
 983               sum1 = tcp1->checksum;
 984               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 985                                      ip4_header_t,
 986                                      dst_address /* changed member */);
 987
 988               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 989                                      ip4_header_t /* cheat */,
 990                                      length /* changed member */);
 991               tcp1->checksum = ip_csum_fold(sum1);
 992             }
 993           else
 994             {
 995               old_port1 = udp1->dst_port;
 996               udp1->dst_port = s1->in2out.port;
 997               udp1->checksum = 0;
 998             }
 999
1000           /* Accounting */
1001           nat44_session_update_counters (s1, now,
1002                                          vlib_buffer_length_in_chain (vm, b1));
1003           /* Per-user LRU list maintenance */
1004           nat44_session_update_lru (sm, s1, thread_index);
1005         trace1:
1006
1007           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1008                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1009             {
1010               snat_out2in_trace_t *t =
1011                  vlib_add_trace (vm, node, b1, sizeof (*t));
1012               t->sw_if_index = sw_if_index1;
1013               t->next_index = next1;
1014               t->session_index = ~0;
1015               if (s1)
1016                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1017             }
1018
1019           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1020
1021           /* verify speculative enqueues, maybe switch current next frame */
1022           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1023                                            to_next, n_left_to_next,
1024                                            bi0, bi1, next0, next1);
1025         }
1026
1027       while (n_left_from > 0 && n_left_to_next > 0)
1028         {
1029           u32 bi0;
1030           vlib_buffer_t * b0;
1031           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1032           u32 sw_if_index0;
1033           ip4_header_t * ip0;
1034           ip_csum_t sum0;
1035           u32 new_addr0, old_addr0;
1036           u16 new_port0, old_port0;
1037           udp_header_t * udp0;
1038           tcp_header_t * tcp0;
1039           icmp46_header_t * icmp0;
1040           snat_session_key_t key0, sm0;
1041           u32 rx_fib_index0;
1042           u32 proto0;
1043           snat_session_t * s0 = 0;
1044           clib_bihash_kv_8_8_t kv0, value0;
1045
1046           /* speculatively enqueue b0 to the current next frame */
1047           bi0 = from[0];
1048           to_next[0] = bi0;
1049           from += 1;
1050           to_next += 1;
1051           n_left_from -= 1;
1052           n_left_to_next -= 1;
1053
1054           b0 = vlib_get_buffer (vm, bi0);
1055
1056           vnet_buffer (b0)->snat.flags = 0;
1057
1058           ip0 = vlib_buffer_get_current (b0);
1059           udp0 = ip4_next_header (ip0);
1060           tcp0 = (tcp_header_t *) udp0;
1061           icmp0 = (icmp46_header_t *) udp0;
1062
1063           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1064           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1065                                    sw_if_index0);
1066
1067           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1068
1069           if (PREDICT_FALSE (proto0 == ~0))
1070             {
1071               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1072                 {
1073                   if (!sm->forwarding_enabled)
1074                     {
1075                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1076                       next0 = SNAT_OUT2IN_NEXT_DROP;
1077                     }
1078                 }
1079               goto trace00;
1080             }
1081
1082           if (PREDICT_FALSE(ip0->ttl == 1))
1083             {
1084               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1085               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1086                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1087                                            0);
1088               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1089               goto trace00;
1090             }
1091
1092           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1093             {
1094               next0 = icmp_out2in_slow_path
1095                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1096                  next0, now, thread_index, &s0);
1097               goto trace00;
1098             }
1099
1100           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1101             {
1102               next0 = SNAT_OUT2IN_NEXT_REASS;
1103               goto trace00;
1104             }
1105
1106           key0.addr = ip0->dst_address;
1107           key0.port = udp0->dst_port;
1108           key0.protocol = proto0;
1109           key0.fib_index = rx_fib_index0;
1110
1111           kv0.key = key0.as_u64;
1112
1113           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1114                                       &kv0, &value0))
1115             {
1116               /* Try to match static mapping by external address and port,
1117                  destination address and port in packet */
1118               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1119                 {
1120                   /*
1121                    * Send DHCP packets to the ipv4 stack, or we won't
1122                    * be able to use dhcp client on the outside interface
1123                    */
1124                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1125                       && (udp0->dst_port ==
1126                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1127                     {
1128                       vnet_feature_next (&next0, b0);
1129                       goto trace00;
1130                     }
1131
1132                   if (!sm->forwarding_enabled)
1133                     {
1134                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1135                       next0 = SNAT_OUT2IN_NEXT_DROP;
1136                     }
1137                   goto trace00;
1138                 }
1139
1140               /* Create session initiated by host from external network */
1141               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1142                                                      thread_index);
1143               if (!s0)
1144                 {
1145                   next0 = SNAT_OUT2IN_NEXT_DROP;
1146                   goto trace00;
1147                 }
1148             }
1149           else
1150             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1151                                     value0.value);
1152
1153           old_addr0 = ip0->dst_address.as_u32;
1154           ip0->dst_address = s0->in2out.addr;
1155           new_addr0 = ip0->dst_address.as_u32;
1156           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1157
1158           sum0 = ip0->checksum;
1159           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1160                                  ip4_header_t,
1161                                  dst_address /* changed member */);
1162           ip0->checksum = ip_csum_fold (sum0);
1163
1164           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1165             {
1166               old_port0 = tcp0->dst_port;
1167               tcp0->dst_port = s0->in2out.port;
1168               new_port0 = tcp0->dst_port;
1169
1170               sum0 = tcp0->checksum;
1171               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1172                                      ip4_header_t,
1173                                      dst_address /* changed member */);
1174
1175               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1176                                      ip4_header_t /* cheat */,
1177                                      length /* changed member */);
1178               tcp0->checksum = ip_csum_fold(sum0);
1179             }
1180           else
1181             {
1182               old_port0 = udp0->dst_port;
1183               udp0->dst_port = s0->in2out.port;
1184               udp0->checksum = 0;
1185             }
1186
1187           /* Accounting */
1188           nat44_session_update_counters (s0, now,
1189                                          vlib_buffer_length_in_chain (vm, b0));
1190           /* Per-user LRU list maintenance */
1191           nat44_session_update_lru (sm, s0, thread_index);
1192         trace00:
1193
1194           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1195                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1196             {
1197               snat_out2in_trace_t *t =
1198                  vlib_add_trace (vm, node, b0, sizeof (*t));
1199               t->sw_if_index = sw_if_index0;
1200               t->next_index = next0;
1201               t->session_index = ~0;
1202               if (s0)
1203                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1204             }
1205
1206           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1207
1208           /* verify speculative enqueue, maybe switch current next frame */
1209           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1210                                            to_next, n_left_to_next,
1211                                            bi0, next0);
1212         }
1213
1214       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1215     }
1216
1217   vlib_node_increment_counter (vm, snat_out2in_node.index,
1218                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1219                                pkts_processed);
1220   return frame->n_vectors;
1221 }
1222
1223 VLIB_REGISTER_NODE (snat_out2in_node) = {
1224   .function = snat_out2in_node_fn,
1225   .name = "nat44-out2in",
1226   .vector_size = sizeof (u32),
1227   .format_trace = format_snat_out2in_trace,
1228   .type = VLIB_NODE_TYPE_INTERNAL,
1229
1230   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1231   .error_strings = snat_out2in_error_strings,
1232
1233   .runtime_data_bytes = sizeof (snat_runtime_t),
1234
1235   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1236
1237   /* edit / add dispositions here */
1238   .next_nodes = {
1239     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1240     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1241     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1242     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1243   },
1244 };
1245 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1246
1247 static uword
1248 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1249                             vlib_node_runtime_t * node,
1250                             vlib_frame_t * frame)
1251 {
1252   u32 n_left_from, *from, *to_next;
1253   snat_out2in_next_t next_index;
1254   u32 pkts_processed = 0;
1255   snat_main_t *sm = &snat_main;
1256   f64 now = vlib_time_now (vm);
1257   u32 thread_index = vm->thread_index;
1258   snat_main_per_thread_data_t *per_thread_data =
1259     &sm->per_thread_data[thread_index];
1260   u32 *fragments_to_drop = 0;
1261   u32 *fragments_to_loopback = 0;
1262
1263   from = vlib_frame_vector_args (frame);
1264   n_left_from = frame->n_vectors;
1265   next_index = node->cached_next_index;
1266
1267   while (n_left_from > 0)
1268     {
1269       u32 n_left_to_next;
1270
1271       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1272
1273       while (n_left_from > 0 && n_left_to_next > 0)
1274        {
1275           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1276           vlib_buffer_t *b0;
1277           u32 next0;
1278           u8 cached0 = 0;
1279           ip4_header_t *ip0;
1280           nat_reass_ip4_t *reass0;
1281           udp_header_t * udp0;
1282           tcp_header_t * tcp0;
1283           snat_session_key_t key0, sm0;
1284           clib_bihash_kv_8_8_t kv0, value0;
1285           snat_session_t * s0 = 0;
1286           u16 old_port0, new_port0;
1287           ip_csum_t sum0;
1288
1289           /* speculatively enqueue b0 to the current next frame */
1290           bi0 = from[0];
1291           to_next[0] = bi0;
1292           from += 1;
1293           to_next += 1;
1294           n_left_from -= 1;
1295           n_left_to_next -= 1;
1296
1297           b0 = vlib_get_buffer (vm, bi0);
1298           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1299
1300           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1301           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1302                                                                sw_if_index0);
1303
1304           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1305             {
1306               next0 = SNAT_OUT2IN_NEXT_DROP;
1307               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1308               goto trace0;
1309             }
1310
1311           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1312           udp0 = ip4_next_header (ip0);
1313           tcp0 = (tcp_header_t *) udp0;
1314           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1315
1316           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1317                                                  ip0->dst_address,
1318                                                  ip0->fragment_id,
1319                                                  ip0->protocol,
1320                                                  1,
1321                                                  &fragments_to_drop);
1322
1323           if (PREDICT_FALSE (!reass0))
1324             {
1325               next0 = SNAT_OUT2IN_NEXT_DROP;
1326               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1327               nat_log_notice ("maximum reassemblies exceeded");
1328               goto trace0;
1329             }
1330
1331           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1332             {
1333               key0.addr = ip0->dst_address;
1334               key0.port = udp0->dst_port;
1335               key0.protocol = proto0;
1336               key0.fib_index = rx_fib_index0;
1337               kv0.key = key0.as_u64;
1338
1339               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1340                 {
1341                   /* Try to match static mapping by external address and port,
1342                      destination address and port in packet */
1343                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1344                     {
1345                       /*
1346                        * Send DHCP packets to the ipv4 stack, or we won't
1347                        * be able to use dhcp client on the outside interface
1348                        */
1349                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1350                           && (udp0->dst_port
1351                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1352                         {
1353                           vnet_feature_next (&next0, b0);
1354                           goto trace0;
1355                         }
1356
1357                       if (!sm->forwarding_enabled)
1358                         {
1359                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1360                           next0 = SNAT_OUT2IN_NEXT_DROP;
1361                         }
1362                       goto trace0;
1363                     }
1364
1365                   /* Create session initiated by host from external network */
1366                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1367                                                          thread_index);
1368                   if (!s0)
1369                     {
1370                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1371                       next0 = SNAT_OUT2IN_NEXT_DROP;
1372                       goto trace0;
1373                     }
1374                   reass0->sess_index = s0 - per_thread_data->sessions;
1375                   reass0->thread_index = thread_index;
1376                 }
1377               else
1378                 {
1379                   s0 = pool_elt_at_index (per_thread_data->sessions,
1380                                           value0.value);
1381                   reass0->sess_index = value0.value;
1382                 }
1383               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1384             }
1385           else
1386             {
1387               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1388                 {
1389                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1390                     {
1391                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1392                       nat_log_notice ("maximum fragments per reassembly exceeded");
1393                       next0 = SNAT_OUT2IN_NEXT_DROP;
1394                       goto trace0;
1395                     }
1396                   cached0 = 1;
1397                   goto trace0;
1398                 }
1399               s0 = pool_elt_at_index (per_thread_data->sessions,
1400                                       reass0->sess_index);
1401             }
1402
1403           old_addr0 = ip0->dst_address.as_u32;
1404           ip0->dst_address = s0->in2out.addr;
1405           new_addr0 = ip0->dst_address.as_u32;
1406           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1407
1408           sum0 = ip0->checksum;
1409           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1410                                  ip4_header_t,
1411                                  dst_address /* changed member */);
1412           ip0->checksum = ip_csum_fold (sum0);
1413
1414           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1415             {
1416               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1417                 {
1418                   old_port0 = tcp0->dst_port;
1419                   tcp0->dst_port = s0->in2out.port;
1420                   new_port0 = tcp0->dst_port;
1421
1422                   sum0 = tcp0->checksum;
1423                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1424                                          ip4_header_t,
1425                                          dst_address /* changed member */);
1426
1427                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1428                                          ip4_header_t /* cheat */,
1429                                          length /* changed member */);
1430                   tcp0->checksum = ip_csum_fold(sum0);
1431                 }
1432               else
1433                 {
1434                   old_port0 = udp0->dst_port;
1435                   udp0->dst_port = s0->in2out.port;
1436                   udp0->checksum = 0;
1437                 }
1438             }
1439
1440           /* Accounting */
1441           nat44_session_update_counters (s0, now,
1442                                          vlib_buffer_length_in_chain (vm, b0));
1443           /* Per-user LRU list maintenance */
1444           nat44_session_update_lru (sm, s0, thread_index);
1445
1446         trace0:
1447           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1448                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1449             {
1450               nat44_out2in_reass_trace_t *t =
1451                  vlib_add_trace (vm, node, b0, sizeof (*t));
1452               t->cached = cached0;
1453               t->sw_if_index = sw_if_index0;
1454               t->next_index = next0;
1455             }
1456
1457           if (cached0)
1458             {
1459               n_left_to_next++;
1460               to_next--;
1461             }
1462           else
1463             {
1464               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1465
1466               /* verify speculative enqueue, maybe switch current next frame */
1467               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1468                                                to_next, n_left_to_next,
1469                                                bi0, next0);
1470             }
1471
1472           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1473             {
1474               from = vlib_frame_vector_args (frame);
1475               u32 len = vec_len (fragments_to_loopback);
1476               if (len <= VLIB_FRAME_SIZE)
1477                 {
1478                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1479                   n_left_from = len;
1480                   vec_reset_length (fragments_to_loopback);
1481                 }
1482               else
1483                 {
1484                   clib_memcpy (from,
1485                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1486                                sizeof (u32) * VLIB_FRAME_SIZE);
1487                   n_left_from = VLIB_FRAME_SIZE;
1488                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1489                 }
1490             }
1491        }
1492
1493       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1494     }
1495
1496   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1497                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1498                                pkts_processed);
1499
1500   nat_send_all_to_node (vm, fragments_to_drop, node,
1501                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1502                         SNAT_OUT2IN_NEXT_DROP);
1503
1504   vec_free (fragments_to_drop);
1505   vec_free (fragments_to_loopback);
1506   return frame->n_vectors;
1507 }
1508
1509 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1510   .function = nat44_out2in_reass_node_fn,
1511   .name = "nat44-out2in-reass",
1512   .vector_size = sizeof (u32),
1513   .format_trace = format_nat44_out2in_reass_trace,
1514   .type = VLIB_NODE_TYPE_INTERNAL,
1515
1516   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1517   .error_strings = snat_out2in_error_strings,
1518
1519   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1520
1521   /* edit / add dispositions here */
1522   .next_nodes = {
1523     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1524     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1525     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1526     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1527   },
1528 };
1529 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1530                               nat44_out2in_reass_node_fn);
1531
1532 /*******************************/
1533 /*** endpoint-dependent mode ***/
1534 /*******************************/
1535 typedef enum {
1536   NAT44_ED_OUT2IN_NEXT_DROP,
1537   NAT44_ED_OUT2IN_NEXT_LOOKUP,
1538   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1539   NAT44_ED_OUT2IN_NEXT_IN2OUT,
1540   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1541   NAT44_ED_OUT2IN_N_NEXT,
1542 } nat44_ed_out2in_next_t;
1543
1544 typedef struct {
1545   u32 sw_if_index;
1546   u32 next_index;
1547   u32 session_index;
1548   u32 is_slow_path;
1549 } nat44_ed_out2in_trace_t;
1550
1551 static u8 *
1552 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1553 {
1554   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1555   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1556   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1557   char * tag;
1558
1559   tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1560
1561   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1562               t->sw_if_index, t->next_index, t->session_index);
1563
1564   return s;
1565 }
1566
1567 static snat_session_t *
1568 create_session_for_static_mapping_ed (snat_main_t * sm,
1569                                       vlib_buffer_t *b,
1570                                       snat_session_key_t l_key,
1571                                       snat_session_key_t e_key,
1572                                       vlib_node_runtime_t * node,
1573                                       u32 thread_index,
1574                                       twice_nat_type_t twice_nat,
1575                                       u8 is_lb)
1576 {
1577   snat_session_t *s;
1578   snat_user_t *u;
1579   ip4_header_t *ip;
1580   udp_header_t *udp;
1581   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1582   clib_bihash_kv_16_8_t kv;
1583   snat_session_key_t eh_key;
1584   u32 address_index;
1585
1586   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1587     {
1588       b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1589       nat_log_notice ("maximum sessions exceeded");
1590       return 0;
1591     }
1592
1593   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1594   if (!u)
1595     {
1596       nat_log_warn ("create NAT user failed");
1597       return 0;
1598     }
1599
1600   s = nat_session_alloc_or_recycle (sm, u, thread_index);
1601   if (!s)
1602     {
1603       nat44_delete_user_with_no_session (sm, u, thread_index);
1604       nat_log_warn ("create NAT session failed");
1605       return 0;
1606     }
1607
1608   ip = vlib_buffer_get_current (b);
1609   udp = ip4_next_header (ip);
1610
1611   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1612   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1613   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1614   if (is_lb)
1615     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1616   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1617   s->outside_address_index = ~0;
1618   s->out2in = e_key;
1619   s->in2out = l_key;
1620   s->in2out.protocol = s->out2in.protocol;
1621   user_session_increment (sm, u, 1);
1622
1623   /* Add to lookup tables */
1624   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1625               e_key.fib_index, e_key.port, s->ext_host_port);
1626   kv.value = s - tsm->sessions;
1627   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
1628     nat_log_notice ("out2in-ed key add failed");
1629
1630   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1631       ip->src_address.as_u32 == l_key.addr.as_u32))
1632     {
1633       eh_key.protocol = e_key.protocol;
1634       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1635                                                thread_index, &eh_key,
1636                                                &address_index,
1637                                                sm->port_per_thread,
1638                                                tsm->snat_thread_index))
1639         {
1640           b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1641           nat44_delete_session (sm, s, thread_index);
1642           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1643             nat_log_notice ("out2in-ed key del failed");
1644           return 0;
1645         }
1646       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1647       s->ext_host_nat_port = eh_key.port;
1648       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1649       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1650                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
1651     }
1652   else
1653     {
1654       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1655                   l_key.fib_index, l_key.port, s->ext_host_port);
1656     }
1657   kv.value = s - tsm->sessions;
1658   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1659     nat_log_notice ("in2out-ed key add failed");
1660
1661   return s;
1662 }
1663
1664 static_always_inline int
1665 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1666 {
1667   icmp46_header_t *icmp0;
1668   nat_ed_ses_key_t key0;
1669   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1670   ip4_header_t *inner_ip0;
1671   void *l4_header = 0;
1672   icmp46_header_t *inner_icmp0;
1673
1674   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1675   echo0 = (icmp_echo_header_t *)(icmp0+1);
1676
1677   if (!icmp_is_error_message (icmp0))
1678     {
1679       key0.proto = IP_PROTOCOL_ICMP;
1680       key0.l_addr = ip0->dst_address;
1681       key0.r_addr = ip0->src_address;
1682       key0.l_port = echo0->identifier;
1683       key0.r_port = 0;
1684     }
1685   else
1686     {
1687       inner_ip0 = (ip4_header_t *)(echo0+1);
1688       l4_header = ip4_next_header (inner_ip0);
1689       key0.proto = inner_ip0->protocol;
1690       key0.l_addr = inner_ip0->src_address;
1691       key0.r_addr = inner_ip0->dst_address;
1692       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1693         {
1694         case SNAT_PROTOCOL_ICMP:
1695           inner_icmp0 = (icmp46_header_t*)l4_header;
1696           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1697           key0.l_port = inner_echo0->identifier;
1698           key0.r_port = 0;
1699           break;
1700         case SNAT_PROTOCOL_UDP:
1701         case SNAT_PROTOCOL_TCP:
1702           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1703           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1704           break;
1705         default:
1706           return -1;
1707         }
1708     }
1709   *p_key0 = key0;
1710   return 0;
1711 }
1712
1713 static int
1714 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1715               u16 dst_port, u32 thread_index)
1716 {
1717   clib_bihash_kv_16_8_t kv, value;
1718   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1719
1720   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1721               sm->inside_fib_index, src_port, dst_port);
1722   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1723     return 1;
1724
1725   return 0;
1726 }
1727
1728 static void
1729 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1730                       u32 thread_index)
1731 {
1732   nat_ed_ses_key_t key;
1733   clib_bihash_kv_16_8_t kv, value;
1734   udp_header_t *udp;
1735   snat_user_t *u;
1736   snat_session_t *s = 0;
1737   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1738   f64 now = vlib_time_now (sm->vlib_main);
1739
1740   if (ip->protocol == IP_PROTOCOL_ICMP)
1741     {
1742       if (icmp_get_ed_key (ip, &key))
1743         return;
1744     }
1745   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1746     {
1747       udp = ip4_next_header(ip);
1748       key.r_addr = ip->src_address;
1749       key.l_addr = ip->dst_address;
1750       key.proto = ip->protocol;
1751       key.l_port = udp->dst_port;
1752       key.r_port = udp->src_port;
1753     }
1754   else
1755     {
1756       key.r_addr = ip->src_address;
1757       key.l_addr = ip->dst_address;
1758       key.proto = ip->protocol;
1759       key.l_port = key.r_port = 0;
1760     }
1761   key.fib_index = 0;
1762   kv.key[0] = key.as_u64[0];
1763   kv.key[1] = key.as_u64[1];
1764
1765   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1766     {
1767       s = pool_elt_at_index (tsm->sessions, value.value);
1768     }
1769   else
1770     {
1771       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1772         return;
1773
1774       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1775                                   thread_index);
1776       if (!u)
1777         {
1778           nat_log_warn ("create NAT user failed");
1779           return;
1780         }
1781
1782       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1783       if (!s)
1784         {
1785           nat44_delete_user_with_no_session (sm, u, thread_index);
1786           nat_log_warn ("create NAT session failed");
1787           return;
1788         }
1789
1790       s->ext_host_addr = key.r_addr;
1791       s->ext_host_port = key.r_port;
1792       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1793       s->outside_address_index = ~0;
1794       s->out2in.addr = key.l_addr;
1795       s->out2in.port = key.l_port;
1796       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1797       s->out2in.fib_index = 0;
1798       s->in2out = s->out2in;
1799       user_session_increment (sm, u, 0);
1800
1801       kv.value = s - tsm->sessions;
1802       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1803         nat_log_notice ("in2out_ed key add failed");
1804     }
1805
1806   if (ip->protocol == IP_PROTOCOL_TCP)
1807     {
1808       tcp_header_t *tcp = ip4_next_header(ip);
1809       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1810         return;
1811     }
1812
1813   /* Per-user LRU list maintenance */
1814   nat44_session_update_lru (sm, s, thread_index);
1815   /* Accounting */
1816   nat44_session_update_counters (s, now, 0);
1817 }
1818
1819 u32
1820 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1821                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1822                       u8 * p_proto, snat_session_key_t * p_value,
1823                       u8 * p_dont_translate, void * d, void * e)
1824 {
1825   u32 next = ~0, sw_if_index, rx_fib_index;
1826   icmp46_header_t *icmp;
1827   nat_ed_ses_key_t key;
1828   clib_bihash_kv_16_8_t kv, value;
1829   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1830   snat_session_t *s = 0;
1831   u8 dont_translate = 0, is_addr_only;
1832   snat_session_key_t e_key, l_key;
1833
1834   icmp = (icmp46_header_t *) ip4_next_header (ip);
1835   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1836   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1837
1838   if (icmp_get_ed_key (ip, &key))
1839     {
1840       b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1841       next = SNAT_OUT2IN_NEXT_DROP;
1842       goto out;
1843     }
1844   key.fib_index = rx_fib_index;
1845   kv.key[0] = key.as_u64[0];
1846   kv.key[1] = key.as_u64[1];
1847
1848   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
1849     {
1850       /* Try to match static mapping */
1851       e_key.addr = ip->dst_address;
1852       e_key.port = key.l_port;
1853       e_key.protocol = ip_proto_to_snat_proto (key.proto);
1854       e_key.fib_index = rx_fib_index;
1855       if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
1856         {
1857           if (!sm->forwarding_enabled)
1858             {
1859               /* Don't NAT packet aimed at the intfc address */
1860               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
1861                                                   ip->dst_address.as_u32)))
1862                 {
1863                   dont_translate = 1;
1864                   goto out;
1865                 }
1866               b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1867               next = NAT44_ED_OUT2IN_NEXT_DROP;
1868               goto out;
1869             }
1870           else
1871             {
1872               dont_translate = 1;
1873               if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port, thread_index))
1874                 {
1875                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1876                   goto out;
1877                 }
1878               create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
1879               goto out;
1880             }
1881         }
1882
1883       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1884                         (icmp->type != ICMP4_echo_request || !is_addr_only)))
1885         {
1886           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1887           next = NAT44_ED_OUT2IN_NEXT_DROP;
1888           goto out;
1889         }
1890
1891       /* Create session initiated by host from external network */
1892       s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
1893                                                thread_index, 0, 0);
1894
1895       if (!s)
1896         {
1897           next = NAT44_ED_OUT2IN_NEXT_DROP;
1898           goto out;
1899         }
1900     }
1901   else
1902     {
1903       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1904                         icmp->type != ICMP4_echo_request &&
1905                         !icmp_is_error_message (icmp)))
1906         {
1907           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1908           next = SNAT_OUT2IN_NEXT_DROP;
1909           goto out;
1910         }
1911
1912       s = pool_elt_at_index (tsm->sessions, value.value);
1913     }
1914
1915   *p_proto = ip_proto_to_snat_proto (key.proto);
1916 out:
1917   if (s)
1918     *p_value = s->in2out;
1919   *p_dont_translate = dont_translate;
1920   if (d)
1921     *(snat_session_t**)d = s;
1922   return next;
1923 }
1924
1925 static snat_session_t *
1926 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
1927                                vlib_buffer_t * b,
1928                                ip4_header_t * ip,
1929                                u32 rx_fib_index,
1930                                u32 thread_index,
1931                                f64 now,
1932                                vlib_main_t * vm,
1933                                vlib_node_runtime_t * node)
1934 {
1935   clib_bihash_kv_8_8_t kv, value;
1936   clib_bihash_kv_16_8_t s_kv, s_value;
1937   snat_static_mapping_t *m;
1938   u32 old_addr, new_addr;
1939   ip_csum_t sum;
1940   snat_session_t * s;
1941   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1942   snat_user_t *u;
1943
1944   old_addr = ip->dst_address.as_u32;
1945
1946   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
1947               rx_fib_index, 0, 0);
1948
1949   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
1950     {
1951       s = pool_elt_at_index (tsm->sessions, s_value.value);
1952       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1953     }
1954   else
1955     {
1956       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1957         {
1958           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1959           nat_log_notice ("maximum sessions exceeded");
1960           return 0;
1961         }
1962
1963       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1964       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1965         {
1966           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1967           return 0;
1968         }
1969
1970       m = pool_elt_at_index (sm->static_mappings, value.value);
1971
1972       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1973
1974       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
1975                                   thread_index);
1976       if (!u)
1977         {
1978           nat_log_warn ("create NAT user failed");
1979           return 0;
1980         }
1981
1982       /* Create a new session */
1983       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1984       if (!s)
1985         {
1986           nat44_delete_user_with_no_session (sm, u, thread_index);
1987           nat_log_warn ("create NAT session failed");
1988           return 0;
1989         }
1990
1991       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1992       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1993       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1994       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1995       s->outside_address_index = ~0;
1996       s->out2in.addr.as_u32 = old_addr;
1997       s->out2in.fib_index = rx_fib_index;
1998       s->in2out.addr.as_u32 = new_addr;
1999       s->in2out.fib_index = m->fib_index;
2000       s->in2out.port = s->out2in.port = ip->protocol;
2001       user_session_increment (sm, u, 1);
2002
2003       /* Add to lookup tables */
2004       s_kv.value = s - tsm->sessions;
2005       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2006         nat_log_notice ("out2in key add failed");
2007
2008       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2009                   m->fib_index, 0, 0);
2010       s_kv.value = s - tsm->sessions;
2011       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2012         nat_log_notice ("in2out key add failed");
2013    }
2014
2015   /* Update IP checksum */
2016   sum = ip->checksum;
2017   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2018   ip->checksum = ip_csum_fold (sum);
2019
2020   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2021
2022   /* Accounting */
2023   nat44_session_update_counters (s, now,
2024                                  vlib_buffer_length_in_chain (vm, b));
2025   /* Per-user LRU list maintenance */
2026   nat44_session_update_lru (sm, s, thread_index);
2027
2028   return s;
2029 }
2030
2031 static inline uword
2032 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2033                                 vlib_node_runtime_t * node,
2034                                 vlib_frame_t * frame, int is_slow_path)
2035 {
2036   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2037   nat44_ed_out2in_next_t next_index;
2038   snat_main_t *sm = &snat_main;
2039   f64 now = vlib_time_now (vm);
2040   u32 thread_index = vm->thread_index;
2041   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2042
2043   stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2044     nat44_ed_out2in_node.index;
2045
2046   from = vlib_frame_vector_args (frame);
2047   n_left_from = frame->n_vectors;
2048   next_index = node->cached_next_index;
2049
2050   while (n_left_from > 0)
2051     {
2052       u32 n_left_to_next;
2053
2054       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2055
2056       while (n_left_from >= 4 && n_left_to_next >= 2)
2057         {
2058           u32 bi0, bi1;
2059           vlib_buffer_t *b0, *b1;
2060           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2061           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2062           u16 old_port0, new_port0, old_port1, new_port1;
2063           ip4_header_t *ip0, *ip1;
2064           udp_header_t *udp0, *udp1;
2065           tcp_header_t *tcp0, *tcp1;
2066           icmp46_header_t *icmp0, *icmp1;
2067           snat_session_t *s0 = 0, *s1 = 0;
2068           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2069           ip_csum_t sum0, sum1;
2070           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2071           u8 is_lb0, is_lb1;
2072           twice_nat_type_t twice_nat0, twice_nat1;
2073
2074           /* Prefetch next iteration. */
2075           {
2076             vlib_buffer_t * p2, * p3;
2077
2078             p2 = vlib_get_buffer (vm, from[2]);
2079             p3 = vlib_get_buffer (vm, from[3]);
2080
2081             vlib_prefetch_buffer_header (p2, LOAD);
2082             vlib_prefetch_buffer_header (p3, LOAD);
2083
2084             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2085             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2086           }
2087
2088           /* speculatively enqueue b0 and b1 to the current next frame */
2089           to_next[0] = bi0 = from[0];
2090           to_next[1] = bi1 = from[1];
2091           from += 2;
2092           to_next += 2;
2093           n_left_from -= 2;
2094           n_left_to_next -= 2;
2095
2096           b0 = vlib_get_buffer (vm, bi0);
2097           b1 = vlib_get_buffer (vm, bi1);
2098
2099           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2100           vnet_buffer (b0)->snat.flags = 0;
2101           ip0 = vlib_buffer_get_current (b0);
2102
2103           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2104           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2105                                                                sw_if_index0);
2106
2107           if (PREDICT_FALSE(ip0->ttl == 1))
2108             {
2109               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2110               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2111                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2112                                            0);
2113               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2114               goto trace00;
2115             }
2116
2117           udp0 = ip4_next_header (ip0);
2118           tcp0 = (tcp_header_t *) udp0;
2119           icmp0 = (icmp46_header_t *) udp0;
2120           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2121
2122           if (is_slow_path)
2123             {
2124               if (PREDICT_FALSE (proto0 == ~0))
2125                 {
2126                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2127                                                      thread_index, now, vm, node);
2128                   if (!sm->forwarding_enabled)
2129                     {
2130                       if (!s0)
2131                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2132                       goto trace00;
2133                     }
2134                 }
2135
2136               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2137                 {
2138                   next0 = icmp_out2in_slow_path
2139                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2140                      next0, now, thread_index, &s0);
2141                   goto trace00;
2142                 }
2143             }
2144           else
2145             {
2146               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2147                 {
2148                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2149                   goto trace00;
2150                 }
2151
2152               if (ip4_is_fragment (ip0))
2153                 {
2154                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2155                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2156                   goto trace00;
2157                 }
2158             }
2159
2160           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2161                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2162
2163           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2164             {
2165               if (is_slow_path)
2166                 {
2167                   /* Try to match static mapping by external address and port,
2168                      destination address and port in packet */
2169                   e_key0.addr = ip0->dst_address;
2170                   e_key0.port = udp0->dst_port;
2171                   e_key0.protocol = proto0;
2172                   e_key0.fib_index = rx_fib_index0;
2173                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2174                       &twice_nat0, &is_lb0))
2175                     {
2176                       /*
2177                        * Send DHCP packets to the ipv4 stack, or we won't
2178                        * be able to use dhcp client on the outside interface
2179                        */
2180                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2181                           && (udp0->dst_port ==
2182                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2183                         {
2184                           vnet_feature_next (&next0, b0);
2185                           goto trace00;
2186                         }
2187
2188                       if (!sm->forwarding_enabled)
2189                         {
2190                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2191                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2192                         }
2193                       else
2194                         {
2195                           if (next_src_nat(sm, ip0, ip0->protocol,
2196                                            udp0->src_port, udp0->dst_port,
2197                                            thread_index))
2198                             {
2199                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2200                               goto trace00;
2201                             }
2202                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2203                                                 thread_index);
2204                         }
2205                       goto trace00;
2206                     }
2207
2208                   /* Create session initiated by host from external network */
2209                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2210                                                             e_key0, node,
2211                                                             thread_index,
2212                                                             twice_nat0, is_lb0);
2213
2214                   if (!s0)
2215                     {
2216                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2217                       goto trace00;
2218                     }
2219                 }
2220               else
2221                 {
2222                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2223                   goto trace00;
2224                 }
2225             }
2226           else
2227             {
2228               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2229             }
2230
2231           old_addr0 = ip0->dst_address.as_u32;
2232           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2233           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2234
2235           sum0 = ip0->checksum;
2236           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2237                                  dst_address);
2238           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2239             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2240                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2241                                    src_address);
2242           ip0->checksum = ip_csum_fold (sum0);
2243
2244           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2245             {
2246               old_port0 = tcp0->dst_port;
2247               new_port0 = tcp0->dst_port = s0->in2out.port;
2248
2249               sum0 = tcp0->checksum;
2250               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2251                                      dst_address);
2252               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2253                                      length);
2254               if (is_twice_nat_session (s0))
2255                 {
2256                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2257                                          s0->ext_host_nat_addr.as_u32,
2258                                          ip4_header_t, dst_address);
2259                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2260                                          s0->ext_host_nat_port, ip4_header_t,
2261                                          length);
2262                   tcp0->src_port = s0->ext_host_nat_port;
2263                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2264                 }
2265               tcp0->checksum = ip_csum_fold(sum0);
2266               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2267                 goto trace00;
2268             }
2269           else
2270             {
2271               udp0->dst_port = s0->in2out.port;
2272               if (is_twice_nat_session (s0))
2273                 {
2274                   udp0->src_port = s0->ext_host_nat_port;
2275                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2276                 }
2277               udp0->checksum = 0;
2278             }
2279
2280           /* Accounting */
2281           nat44_session_update_counters (s0, now,
2282                                          vlib_buffer_length_in_chain (vm, b0));
2283           /* Per-user LRU list maintenance */
2284           nat44_session_update_lru (sm, s0, thread_index);
2285
2286         trace00:
2287           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2288                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2289             {
2290               nat44_ed_out2in_trace_t *t =
2291                 vlib_add_trace (vm, node, b0, sizeof (*t));
2292               t->is_slow_path = is_slow_path;
2293               t->sw_if_index = sw_if_index0;
2294               t->next_index = next0;
2295               t->session_index = ~0;
2296               if (s0)
2297                 t->session_index = s0 - tsm->sessions;
2298             }
2299
2300           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2301
2302           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2303           vnet_buffer (b1)->snat.flags = 0;
2304           ip1 = vlib_buffer_get_current (b1);
2305
2306           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2307           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2308                                                                sw_if_index1);
2309
2310           if (PREDICT_FALSE(ip1->ttl == 1))
2311             {
2312               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2313               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2314                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2315                                            0);
2316               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2317               goto trace01;
2318             }
2319
2320           udp1 = ip4_next_header (ip1);
2321           tcp1 = (tcp_header_t *) udp1;
2322           icmp1 = (icmp46_header_t *) udp1;
2323           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2324
2325           if (is_slow_path)
2326             {
2327               if (PREDICT_FALSE (proto1 == ~0))
2328                 {
2329                   s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2330                                                      thread_index, now, vm, node);
2331                   if (!sm->forwarding_enabled)
2332                     {
2333                       if (!s1)
2334                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2335                       goto trace01;
2336                     }
2337                 }
2338
2339               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2340                 {
2341                   next1 = icmp_out2in_slow_path
2342                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2343                      next1, now, thread_index, &s1);
2344                   goto trace01;
2345                 }
2346             }
2347           else
2348             {
2349               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2350                 {
2351                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2352                   goto trace01;
2353                 }
2354
2355               if (ip4_is_fragment (ip1))
2356                 {
2357                   b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2358                   next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2359                   goto trace01;
2360                 }
2361             }
2362
2363           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2364                       rx_fib_index1, udp1->dst_port, udp1->src_port);
2365
2366           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2367             {
2368               if (is_slow_path)
2369                 {
2370                   /* Try to match static mapping by external address and port,
2371                      destination address and port in packet */
2372                   e_key1.addr = ip1->dst_address;
2373                   e_key1.port = udp1->dst_port;
2374                   e_key1.protocol = proto1;
2375                   e_key1.fib_index = rx_fib_index1;
2376                   if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2377                       &twice_nat1, &is_lb1))
2378                     {
2379                       /*
2380                        * Send DHCP packets to the ipv4 stack, or we won't
2381                        * be able to use dhcp client on the outside interface
2382                        */
2383                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2384                           && (udp1->dst_port ==
2385                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2386                         {
2387                           vnet_feature_next (&next1, b1);
2388                           goto trace01;
2389                         }
2390
2391                       if (!sm->forwarding_enabled)
2392                         {
2393                           b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2394                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2395                         }
2396                       else
2397                         {
2398                           if (next_src_nat(sm, ip1, ip1->protocol,
2399                                            udp1->src_port, udp1->dst_port,
2400                                            thread_index))
2401                             {
2402                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2403                               goto trace01;
2404                             }
2405                           create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2406                                                 thread_index);
2407                         }
2408                       goto trace01;
2409                     }
2410
2411                   /* Create session initiated by host from external network */
2412                   s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2413                                                             e_key1, node,
2414                                                             thread_index,
2415                                                             twice_nat1, is_lb1);
2416
2417                   if (!s1)
2418                     {
2419                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2420                       goto trace01;
2421                     }
2422                 }
2423               else
2424                 {
2425                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2426                   goto trace01;
2427                 }
2428             }
2429           else
2430             {
2431               s1 = pool_elt_at_index (tsm->sessions, value1.value);
2432             }
2433
2434           old_addr1 = ip1->dst_address.as_u32;
2435           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2436           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2437
2438           sum1 = ip1->checksum;
2439           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2440                                  dst_address);
2441           if (PREDICT_FALSE (is_twice_nat_session (s1)))
2442             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2443                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
2444                                    src_address);
2445           ip1->checksum = ip_csum_fold (sum1);
2446
2447           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2448             {
2449               old_port1 = tcp1->dst_port;
2450               new_port1 = tcp1->dst_port = s1->in2out.port;
2451
2452               sum1 = tcp1->checksum;
2453               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2454                                      dst_address);
2455               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2456                                      length);
2457               if (is_twice_nat_session (s1))
2458                 {
2459                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2460                                          s1->ext_host_nat_addr.as_u32,
2461                                          ip4_header_t, dst_address);
2462                   sum1 = ip_csum_update (sum1, tcp1->src_port,
2463                                          s1->ext_host_nat_port, ip4_header_t,
2464                                          length);
2465                   tcp1->src_port = s1->ext_host_nat_port;
2466                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2467                 }
2468               tcp1->checksum = ip_csum_fold(sum1);
2469               if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2470                 goto trace01;
2471             }
2472           else
2473             {
2474               udp1->dst_port = s1->in2out.port;
2475               if (is_twice_nat_session (s1))
2476                 {
2477                   udp1->src_port = s1->ext_host_nat_port;
2478                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2479                 }
2480               udp1->checksum = 0;
2481             }
2482
2483           /* Accounting */
2484           nat44_session_update_counters (s1, now,
2485                                          vlib_buffer_length_in_chain (vm, b1));
2486           /* Per-user LRU list maintenance */
2487           nat44_session_update_lru (sm, s1, thread_index);
2488
2489         trace01:
2490           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2491                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2492             {
2493               nat44_ed_out2in_trace_t *t =
2494                 vlib_add_trace (vm, node, b1, sizeof (*t));
2495               t->is_slow_path = is_slow_path;
2496               t->sw_if_index = sw_if_index1;
2497               t->next_index = next1;
2498               t->session_index = ~0;
2499               if (s1)
2500                 t->session_index = s1 - tsm->sessions;
2501             }
2502
2503           pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2504
2505           /* verify speculative enqueues, maybe switch current next frame */
2506           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2507                                            to_next, n_left_to_next,
2508                                            bi0, bi1, next0, next1);
2509         }
2510
2511       while (n_left_from > 0 && n_left_to_next > 0)
2512         {
2513           u32 bi0;
2514           vlib_buffer_t *b0;
2515           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2516           u16 old_port0, new_port0;
2517           ip4_header_t *ip0;
2518           udp_header_t *udp0;
2519           tcp_header_t *tcp0;
2520           icmp46_header_t * icmp0;
2521           snat_session_t *s0 = 0;
2522           clib_bihash_kv_16_8_t kv0, value0;
2523           ip_csum_t sum0;
2524           snat_session_key_t e_key0, l_key0;
2525           u8 is_lb0;
2526           twice_nat_type_t twice_nat0;
2527
2528           /* speculatively enqueue b0 to the current next frame */
2529           bi0 = from[0];
2530           to_next[0] = bi0;
2531           from += 1;
2532           to_next += 1;
2533           n_left_from -= 1;
2534           n_left_to_next -= 1;
2535
2536           b0 = vlib_get_buffer (vm, bi0);
2537           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2538           vnet_buffer (b0)->snat.flags = 0;
2539           ip0 = vlib_buffer_get_current (b0);
2540
2541           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2542           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2543                                                                sw_if_index0);
2544
2545           if (PREDICT_FALSE(ip0->ttl == 1))
2546             {
2547               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2548               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2549                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2550                                            0);
2551               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2552               goto trace0;
2553             }
2554
2555           udp0 = ip4_next_header (ip0);
2556           tcp0 = (tcp_header_t *) udp0;
2557           icmp0 = (icmp46_header_t *) udp0;
2558           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2559
2560           if (is_slow_path)
2561             {
2562               if (PREDICT_FALSE (proto0 == ~0))
2563                 {
2564                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2565                                                      thread_index, now, vm, node);
2566                   if (!sm->forwarding_enabled)
2567                     {
2568                       if (!s0)
2569                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2570                       goto trace0;
2571                     }
2572                 }
2573
2574               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2575                 {
2576                   next0 = icmp_out2in_slow_path
2577                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2578                      next0, now, thread_index, &s0);
2579                   goto trace0;
2580                 }
2581             }
2582           else
2583             {
2584               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2585                 {
2586                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2587                   goto trace0;
2588                 }
2589
2590               if (ip4_is_fragment (ip0))
2591                 {
2592                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2593                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2594                   goto trace0;
2595                 }
2596             }
2597
2598           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2599                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2600
2601           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2602             {
2603               if (is_slow_path)
2604                 {
2605                   /* Try to match static mapping by external address and port,
2606                      destination address and port in packet */
2607                   e_key0.addr = ip0->dst_address;
2608                   e_key0.port = udp0->dst_port;
2609                   e_key0.protocol = proto0;
2610                   e_key0.fib_index = rx_fib_index0;
2611                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2612                       &twice_nat0, &is_lb0))
2613                     {
2614                       /*
2615                        * Send DHCP packets to the ipv4 stack, or we won't
2616                        * be able to use dhcp client on the outside interface
2617                        */
2618                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2619                           && (udp0->dst_port ==
2620                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2621                         {
2622                           vnet_feature_next (&next0, b0);
2623                           goto trace0;
2624                         }
2625
2626                       if (!sm->forwarding_enabled)
2627                         {
2628                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2629                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2630                         }
2631                       else
2632                         {
2633                           if (next_src_nat(sm, ip0, ip0->protocol,
2634                                            udp0->src_port, udp0->dst_port,
2635                                            thread_index))
2636                             {
2637                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2638                               goto trace0;
2639                             }
2640                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2641                                                 thread_index);
2642                         }
2643                       goto trace0;
2644                     }
2645
2646                   /* Create session initiated by host from external network */
2647                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2648                                                             e_key0, node,
2649                                                             thread_index,
2650                                                             twice_nat0, is_lb0);
2651
2652                   if (!s0)
2653                     {
2654                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2655                       goto trace0;
2656                     }
2657                 }
2658               else
2659                 {
2660                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2661                   goto trace0;
2662                 }
2663             }
2664           else
2665             {
2666               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2667             }
2668
2669           old_addr0 = ip0->dst_address.as_u32;
2670           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2671           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2672
2673           sum0 = ip0->checksum;
2674           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2675                                  dst_address);
2676           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2677             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2678                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2679                                    src_address);
2680           ip0->checksum = ip_csum_fold (sum0);
2681
2682           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2683             {
2684               old_port0 = tcp0->dst_port;
2685               new_port0 = tcp0->dst_port = s0->in2out.port;
2686
2687               sum0 = tcp0->checksum;
2688               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2689                                      dst_address);
2690               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2691                                      length);
2692               if (is_twice_nat_session (s0))
2693                 {
2694                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2695                                          s0->ext_host_nat_addr.as_u32,
2696                                          ip4_header_t, dst_address);
2697                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2698                                          s0->ext_host_nat_port, ip4_header_t,
2699                                          length);
2700                   tcp0->src_port = s0->ext_host_nat_port;
2701                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2702                 }
2703               tcp0->checksum = ip_csum_fold(sum0);
2704               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2705                 goto trace0;
2706             }
2707           else
2708             {
2709               udp0->dst_port = s0->in2out.port;
2710               if (is_twice_nat_session (s0))
2711                 {
2712                   udp0->src_port = s0->ext_host_nat_port;
2713                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2714                 }
2715               udp0->checksum = 0;
2716             }
2717
2718           /* Accounting */
2719           nat44_session_update_counters (s0, now,
2720                                          vlib_buffer_length_in_chain (vm, b0));
2721           /* Per-user LRU list maintenance */
2722           nat44_session_update_lru (sm, s0, thread_index);
2723
2724         trace0:
2725           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2726                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2727             {
2728               nat44_ed_out2in_trace_t *t =
2729                 vlib_add_trace (vm, node, b0, sizeof (*t));
2730               t->is_slow_path = is_slow_path;
2731               t->sw_if_index = sw_if_index0;
2732               t->next_index = next0;
2733               t->session_index = ~0;
2734               if (s0)
2735                 t->session_index = s0 - tsm->sessions;
2736             }
2737
2738           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2739
2740           /* verify speculative enqueue, maybe switch current next frame */
2741           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2742                                            to_next, n_left_to_next,
2743                                            bi0, next0);
2744         }
2745
2746       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2747     }
2748
2749   vlib_node_increment_counter (vm, stats_node_index,
2750                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2751                                pkts_processed);
2752   return frame->n_vectors;
2753 }
2754
2755 static uword
2756 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2757                               vlib_node_runtime_t * node,
2758                               vlib_frame_t * frame)
2759 {
2760   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2761 }
2762
2763 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2764   .function = nat44_ed_out2in_fast_path_fn,
2765   .name = "nat44-ed-out2in",
2766   .vector_size = sizeof (u32),
2767   .format_trace = format_nat44_ed_out2in_trace,
2768   .type = VLIB_NODE_TYPE_INTERNAL,
2769
2770   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2771   .error_strings = snat_out2in_error_strings,
2772
2773   .runtime_data_bytes = sizeof (snat_runtime_t),
2774
2775   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2776
2777   /* edit / add dispositions here */
2778   .next_nodes = {
2779     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2780     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2781     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2782     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2783     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2784   },
2785 };
2786
2787 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2788
2789 static uword
2790 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2791                               vlib_node_runtime_t * node,
2792                               vlib_frame_t * frame)
2793 {
2794   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2795 }
2796
2797 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2798   .function = nat44_ed_out2in_slow_path_fn,
2799   .name = "nat44-ed-out2in-slowpath",
2800   .vector_size = sizeof (u32),
2801   .format_trace = format_nat44_ed_out2in_trace,
2802   .type = VLIB_NODE_TYPE_INTERNAL,
2803
2804   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2805   .error_strings = snat_out2in_error_strings,
2806
2807   .runtime_data_bytes = sizeof (snat_runtime_t),
2808
2809   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2810
2811   /* edit / add dispositions here */
2812   .next_nodes = {
2813     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2814     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2815     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2816     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2817     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2818   },
2819 };
2820
2821 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2822                               nat44_ed_out2in_slow_path_fn);
2823
2824 /**************************/
2825 /*** deterministic mode ***/
2826 /**************************/
2827 static uword
2828 snat_det_out2in_node_fn (vlib_main_t * vm,
2829                          vlib_node_runtime_t * node,
2830                          vlib_frame_t * frame)
2831 {
2832   u32 n_left_from, * from, * to_next;
2833   snat_out2in_next_t next_index;
2834   u32 pkts_processed = 0;
2835   snat_main_t * sm = &snat_main;
2836   u32 thread_index = vm->thread_index;
2837
2838   from = vlib_frame_vector_args (frame);
2839   n_left_from = frame->n_vectors;
2840   next_index = node->cached_next_index;
2841
2842   while (n_left_from > 0)
2843     {
2844       u32 n_left_to_next;
2845
2846       vlib_get_next_frame (vm, node, next_index,
2847                            to_next, n_left_to_next);
2848
2849       while (n_left_from >= 4 && n_left_to_next >= 2)
2850         {
2851           u32 bi0, bi1;
2852           vlib_buffer_t * b0, * b1;
2853           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2854           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2855           u32 sw_if_index0, sw_if_index1;
2856           ip4_header_t * ip0, * ip1;
2857           ip_csum_t sum0, sum1;
2858           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2859           u16 new_port0, old_port0, old_port1, new_port1;
2860           udp_header_t * udp0, * udp1;
2861           tcp_header_t * tcp0, * tcp1;
2862           u32 proto0, proto1;
2863           snat_det_out_key_t key0, key1;
2864           snat_det_map_t * dm0, * dm1;
2865           snat_det_session_t * ses0 = 0, * ses1 = 0;
2866           u32 rx_fib_index0, rx_fib_index1;
2867           icmp46_header_t * icmp0, * icmp1;
2868
2869           /* Prefetch next iteration. */
2870           {
2871             vlib_buffer_t * p2, * p3;
2872
2873             p2 = vlib_get_buffer (vm, from[2]);
2874             p3 = vlib_get_buffer (vm, from[3]);
2875
2876             vlib_prefetch_buffer_header (p2, LOAD);
2877             vlib_prefetch_buffer_header (p3, LOAD);
2878
2879             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2880             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2881           }
2882
2883           /* speculatively enqueue b0 and b1 to the current next frame */
2884           to_next[0] = bi0 = from[0];
2885           to_next[1] = bi1 = from[1];
2886           from += 2;
2887           to_next += 2;
2888           n_left_from -= 2;
2889           n_left_to_next -= 2;
2890
2891           b0 = vlib_get_buffer (vm, bi0);
2892           b1 = vlib_get_buffer (vm, bi1);
2893
2894           ip0 = vlib_buffer_get_current (b0);
2895           udp0 = ip4_next_header (ip0);
2896           tcp0 = (tcp_header_t *) udp0;
2897
2898           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2899
2900           if (PREDICT_FALSE(ip0->ttl == 1))
2901             {
2902               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2903               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2904                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2905                                            0);
2906               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2907               goto trace0;
2908             }
2909
2910           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2911
2912           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2913             {
2914               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2915               icmp0 = (icmp46_header_t *) udp0;
2916
2917               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2918                                   rx_fib_index0, node, next0, thread_index,
2919                                   &ses0, &dm0);
2920               goto trace0;
2921             }
2922
2923           key0.ext_host_addr = ip0->src_address;
2924           key0.ext_host_port = tcp0->src;
2925           key0.out_port = tcp0->dst;
2926
2927           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2928           if (PREDICT_FALSE(!dm0))
2929             {
2930               nat_log_info ("unknown dst address:  %U",
2931                             format_ip4_address, &ip0->dst_address);
2932               next0 = SNAT_OUT2IN_NEXT_DROP;
2933               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2934               goto trace0;
2935             }
2936
2937           snat_det_reverse(dm0, &ip0->dst_address,
2938                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2939
2940           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2941           if (PREDICT_FALSE(!ses0))
2942             {
2943               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2944                             format_ip4_address, &ip0->src_address,
2945                             clib_net_to_host_u16 (tcp0->src),
2946                             format_ip4_address, &ip0->dst_address,
2947                             clib_net_to_host_u16 (tcp0->dst),
2948                             format_ip4_address, &new_addr0);
2949               next0 = SNAT_OUT2IN_NEXT_DROP;
2950               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2951               goto trace0;
2952             }
2953           new_port0 = ses0->in_port;
2954
2955           old_addr0 = ip0->dst_address;
2956           ip0->dst_address = new_addr0;
2957           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2958
2959           sum0 = ip0->checksum;
2960           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2961                                  ip4_header_t,
2962                                  dst_address /* changed member */);
2963           ip0->checksum = ip_csum_fold (sum0);
2964
2965           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2966             {
2967               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2968                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2969               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2970                 snat_det_ses_close(dm0, ses0);
2971
2972               old_port0 = tcp0->dst;
2973               tcp0->dst = new_port0;
2974
2975               sum0 = tcp0->checksum;
2976               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2977                                      ip4_header_t,
2978                                      dst_address /* changed member */);
2979
2980               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2981                                      ip4_header_t /* cheat */,
2982                                      length /* changed member */);
2983               tcp0->checksum = ip_csum_fold(sum0);
2984             }
2985           else
2986             {
2987               old_port0 = udp0->dst_port;
2988               udp0->dst_port = new_port0;
2989               udp0->checksum = 0;
2990             }
2991
2992         trace0:
2993
2994           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2995                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2996             {
2997               snat_out2in_trace_t *t =
2998                  vlib_add_trace (vm, node, b0, sizeof (*t));
2999               t->sw_if_index = sw_if_index0;
3000               t->next_index = next0;
3001               t->session_index = ~0;
3002               if (ses0)
3003                 t->session_index = ses0 - dm0->sessions;
3004             }
3005
3006           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3007
3008           b1 = vlib_get_buffer (vm, bi1);
3009
3010           ip1 = vlib_buffer_get_current (b1);
3011           udp1 = ip4_next_header (ip1);
3012           tcp1 = (tcp_header_t *) udp1;
3013
3014           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3015
3016           if (PREDICT_FALSE(ip1->ttl == 1))
3017             {
3018               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3019               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3020                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3021                                            0);
3022               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3023               goto trace1;
3024             }
3025
3026           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3027
3028           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3029             {
3030               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3031               icmp1 = (icmp46_header_t *) udp1;
3032
3033               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3034                                   rx_fib_index1, node, next1, thread_index,
3035                                   &ses1, &dm1);
3036               goto trace1;
3037             }
3038
3039           key1.ext_host_addr = ip1->src_address;
3040           key1.ext_host_port = tcp1->src;
3041           key1.out_port = tcp1->dst;
3042
3043           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3044           if (PREDICT_FALSE(!dm1))
3045             {
3046               nat_log_info ("unknown dst address:  %U",
3047                             format_ip4_address, &ip1->dst_address);
3048               next1 = SNAT_OUT2IN_NEXT_DROP;
3049               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3050               goto trace1;
3051             }
3052
3053           snat_det_reverse(dm1, &ip1->dst_address,
3054                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
3055
3056           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3057           if (PREDICT_FALSE(!ses1))
3058             {
3059               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3060                             format_ip4_address, &ip1->src_address,
3061                             clib_net_to_host_u16 (tcp1->src),
3062                             format_ip4_address, &ip1->dst_address,
3063                             clib_net_to_host_u16 (tcp1->dst),
3064                             format_ip4_address, &new_addr1);
3065               next1 = SNAT_OUT2IN_NEXT_DROP;
3066               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3067               goto trace1;
3068             }
3069           new_port1 = ses1->in_port;
3070
3071           old_addr1 = ip1->dst_address;
3072           ip1->dst_address = new_addr1;
3073           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3074
3075           sum1 = ip1->checksum;
3076           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3077                                  ip4_header_t,
3078                                  dst_address /* changed member */);
3079           ip1->checksum = ip_csum_fold (sum1);
3080
3081           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3082             {
3083               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3084                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3085               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3086                 snat_det_ses_close(dm1, ses1);
3087
3088               old_port1 = tcp1->dst;
3089               tcp1->dst = new_port1;
3090
3091               sum1 = tcp1->checksum;
3092               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3093                                      ip4_header_t,
3094                                      dst_address /* changed member */);
3095
3096               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3097                                      ip4_header_t /* cheat */,
3098                                      length /* changed member */);
3099               tcp1->checksum = ip_csum_fold(sum1);
3100             }
3101           else
3102             {
3103               old_port1 = udp1->dst_port;
3104               udp1->dst_port = new_port1;
3105               udp1->checksum = 0;
3106             }
3107
3108         trace1:
3109
3110           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3111                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3112             {
3113               snat_out2in_trace_t *t =
3114                  vlib_add_trace (vm, node, b1, sizeof (*t));
3115               t->sw_if_index = sw_if_index1;
3116               t->next_index = next1;
3117               t->session_index = ~0;
3118               if (ses1)
3119                 t->session_index = ses1 - dm1->sessions;
3120             }
3121
3122           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3123
3124           /* verify speculative enqueues, maybe switch current next frame */
3125           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3126                                            to_next, n_left_to_next,
3127                                            bi0, bi1, next0, next1);
3128          }
3129
3130       while (n_left_from > 0 && n_left_to_next > 0)
3131         {
3132           u32 bi0;
3133           vlib_buffer_t * b0;
3134           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3135           u32 sw_if_index0;
3136           ip4_header_t * ip0;
3137           ip_csum_t sum0;
3138           ip4_address_t new_addr0, old_addr0;
3139           u16 new_port0, old_port0;
3140           udp_header_t * udp0;
3141           tcp_header_t * tcp0;
3142           u32 proto0;
3143           snat_det_out_key_t key0;
3144           snat_det_map_t * dm0;
3145           snat_det_session_t * ses0 = 0;
3146           u32 rx_fib_index0;
3147           icmp46_header_t * icmp0;
3148
3149           /* speculatively enqueue b0 to the current next frame */
3150           bi0 = from[0];
3151           to_next[0] = bi0;
3152           from += 1;
3153           to_next += 1;
3154           n_left_from -= 1;
3155           n_left_to_next -= 1;
3156
3157           b0 = vlib_get_buffer (vm, bi0);
3158
3159           ip0 = vlib_buffer_get_current (b0);
3160           udp0 = ip4_next_header (ip0);
3161           tcp0 = (tcp_header_t *) udp0;
3162
3163           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3164
3165           if (PREDICT_FALSE(ip0->ttl == 1))
3166             {
3167               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3168               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3169                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3170                                            0);
3171               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3172               goto trace00;
3173             }
3174
3175           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3176
3177           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3178             {
3179               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3180               icmp0 = (icmp46_header_t *) udp0;
3181
3182               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3183                                   rx_fib_index0, node, next0, thread_index,
3184                                   &ses0, &dm0);
3185               goto trace00;
3186             }
3187
3188           key0.ext_host_addr = ip0->src_address;
3189           key0.ext_host_port = tcp0->src;
3190           key0.out_port = tcp0->dst;
3191
3192           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3193           if (PREDICT_FALSE(!dm0))
3194             {
3195               nat_log_info ("unknown dst address:  %U",
3196                             format_ip4_address, &ip0->dst_address);
3197               next0 = SNAT_OUT2IN_NEXT_DROP;
3198               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3199               goto trace00;
3200             }
3201
3202           snat_det_reverse(dm0, &ip0->dst_address,
3203                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
3204
3205           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3206           if (PREDICT_FALSE(!ses0))
3207             {
3208               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3209                             format_ip4_address, &ip0->src_address,
3210                             clib_net_to_host_u16 (tcp0->src),
3211                             format_ip4_address, &ip0->dst_address,
3212                             clib_net_to_host_u16 (tcp0->dst),
3213                             format_ip4_address, &new_addr0);
3214               next0 = SNAT_OUT2IN_NEXT_DROP;
3215               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3216               goto trace00;
3217             }
3218           new_port0 = ses0->in_port;
3219
3220           old_addr0 = ip0->dst_address;
3221           ip0->dst_address = new_addr0;
3222           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3223
3224           sum0 = ip0->checksum;
3225           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3226                                  ip4_header_t,
3227                                  dst_address /* changed member */);
3228           ip0->checksum = ip_csum_fold (sum0);
3229
3230           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3231             {
3232               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3233                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3234               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3235                 snat_det_ses_close(dm0, ses0);
3236
3237               old_port0 = tcp0->dst;
3238               tcp0->dst = new_port0;
3239
3240               sum0 = tcp0->checksum;
3241               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3242                                      ip4_header_t,
3243                                      dst_address /* changed member */);
3244
3245               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3246                                      ip4_header_t /* cheat */,
3247                                      length /* changed member */);
3248               tcp0->checksum = ip_csum_fold(sum0);
3249             }
3250           else
3251             {
3252               old_port0 = udp0->dst_port;
3253               udp0->dst_port = new_port0;
3254               udp0->checksum = 0;
3255             }
3256
3257         trace00:
3258
3259           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3260                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3261             {
3262               snat_out2in_trace_t *t =
3263                  vlib_add_trace (vm, node, b0, sizeof (*t));
3264               t->sw_if_index = sw_if_index0;
3265               t->next_index = next0;
3266               t->session_index = ~0;
3267               if (ses0)
3268                 t->session_index = ses0 - dm0->sessions;
3269             }
3270
3271           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3272
3273           /* verify speculative enqueue, maybe switch current next frame */
3274           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3275                                            to_next, n_left_to_next,
3276                                            bi0, next0);
3277         }
3278
3279       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3280     }
3281
3282   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3283                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3284                                pkts_processed);
3285   return frame->n_vectors;
3286 }
3287
3288 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3289   .function = snat_det_out2in_node_fn,
3290   .name = "nat44-det-out2in",
3291   .vector_size = sizeof (u32),
3292   .format_trace = format_snat_out2in_trace,
3293   .type = VLIB_NODE_TYPE_INTERNAL,
3294
3295   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3296   .error_strings = snat_out2in_error_strings,
3297
3298   .runtime_data_bytes = sizeof (snat_runtime_t),
3299
3300   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3301
3302   /* edit / add dispositions here */
3303   .next_nodes = {
3304     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3305     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3306     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3307     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3308   },
3309 };
3310 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3311
3312 /**
3313  * Get address and port values to be used for ICMP packet translation
3314  * and create session if needed
3315  *
3316  * @param[in,out] sm             NAT main
3317  * @param[in,out] node           NAT node runtime
3318  * @param[in] thread_index       thread index
3319  * @param[in,out] b0             buffer containing packet to be translated
3320  * @param[out] p_proto           protocol used for matching
3321  * @param[out] p_value           address and port after NAT translation
3322  * @param[out] p_dont_translate  if packet should not be translated
3323  * @param d                      optional parameter
3324  * @param e                      optional parameter
3325  */
3326 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3327                           u32 thread_index, vlib_buffer_t *b0,
3328                           ip4_header_t *ip0, u8 *p_proto,
3329                           snat_session_key_t *p_value,
3330                           u8 *p_dont_translate, void *d, void *e)
3331 {
3332   icmp46_header_t *icmp0;
3333   u32 sw_if_index0;
3334   u8 protocol;
3335   snat_det_out_key_t key0;
3336   u8 dont_translate = 0;
3337   u32 next0 = ~0;
3338   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3339   ip4_header_t *inner_ip0;
3340   void *l4_header = 0;
3341   icmp46_header_t *inner_icmp0;
3342   snat_det_map_t * dm0 = 0;
3343   ip4_address_t new_addr0 = {{0}};
3344   snat_det_session_t * ses0 = 0;
3345   ip4_address_t out_addr;
3346
3347   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3348   echo0 = (icmp_echo_header_t *)(icmp0+1);
3349   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3350
3351   if (!icmp_is_error_message (icmp0))
3352     {
3353       protocol = SNAT_PROTOCOL_ICMP;
3354       key0.ext_host_addr = ip0->src_address;
3355       key0.ext_host_port = 0;
3356       key0.out_port = echo0->identifier;
3357       out_addr = ip0->dst_address;
3358     }
3359   else
3360     {
3361       inner_ip0 = (ip4_header_t *)(echo0+1);
3362       l4_header = ip4_next_header (inner_ip0);
3363       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3364       key0.ext_host_addr = inner_ip0->dst_address;
3365       out_addr = inner_ip0->src_address;
3366       switch (protocol)
3367         {
3368         case SNAT_PROTOCOL_ICMP:
3369           inner_icmp0 = (icmp46_header_t*)l4_header;
3370           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3371           key0.ext_host_port = 0;
3372           key0.out_port = inner_echo0->identifier;
3373           break;
3374         case SNAT_PROTOCOL_UDP:
3375         case SNAT_PROTOCOL_TCP:
3376           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3377           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3378           break;
3379         default:
3380           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3381           next0 = SNAT_OUT2IN_NEXT_DROP;
3382           goto out;
3383         }
3384     }
3385
3386   dm0 = snat_det_map_by_out(sm, &out_addr);
3387   if (PREDICT_FALSE(!dm0))
3388     {
3389       /* Don't NAT packet aimed at the intfc address */
3390       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3391                                           ip0->dst_address.as_u32)))
3392         {
3393           dont_translate = 1;
3394           goto out;
3395         }
3396       nat_log_info ("unknown dst address:  %U",
3397                     format_ip4_address, &ip0->dst_address);
3398       goto out;
3399     }
3400
3401   snat_det_reverse(dm0, &ip0->dst_address,
3402                    clib_net_to_host_u16(key0.out_port), &new_addr0);
3403
3404   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3405   if (PREDICT_FALSE(!ses0))
3406     {
3407       /* Don't NAT packet aimed at the intfc address */
3408       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3409                                           ip0->dst_address.as_u32)))
3410         {
3411           dont_translate = 1;
3412           goto out;
3413         }
3414       nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3415                     format_ip4_address, &key0.ext_host_addr,
3416                     clib_net_to_host_u16 (key0.ext_host_port),
3417                     format_ip4_address, &out_addr,
3418                     clib_net_to_host_u16 (key0.out_port),
3419                     format_ip4_address, &new_addr0);
3420       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3421       next0 = SNAT_OUT2IN_NEXT_DROP;
3422       goto out;
3423     }
3424
3425   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3426                     !icmp_is_error_message (icmp0)))
3427     {
3428       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3429       next0 = SNAT_OUT2IN_NEXT_DROP;
3430       goto out;
3431     }
3432
3433   goto out;
3434
3435 out:
3436   *p_proto = protocol;
3437   if (ses0)
3438     {
3439       p_value->addr = new_addr0;
3440       p_value->fib_index = sm->inside_fib_index;
3441       p_value->port = ses0->in_port;
3442     }
3443   *p_dont_translate = dont_translate;
3444   if (d)
3445     *(snat_det_session_t**)d = ses0;
3446   if (e)
3447     *(snat_det_map_t**)e = dm0;
3448   return next0;
3449 }
3450
3451 /**********************/
3452 /*** worker handoff ***/
3453 /**********************/
3454 static uword
3455 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3456                                vlib_node_runtime_t * node,
3457                                vlib_frame_t * frame)
3458 {
3459   snat_main_t *sm = &snat_main;
3460   vlib_thread_main_t *tm = vlib_get_thread_main ();
3461   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3462   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3463   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3464     = 0;
3465   vlib_frame_queue_elt_t *hf = 0;
3466   vlib_frame_queue_t *fq;
3467   vlib_frame_t *f = 0;
3468   int i;
3469   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3470   u32 next_worker_index = 0;
3471   u32 current_worker_index = ~0;
3472   u32 thread_index = vm->thread_index;
3473   vlib_frame_t *d = 0;
3474
3475   ASSERT (vec_len (sm->workers));
3476
3477   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3478     {
3479       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3480
3481       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3482                                tm->n_vlib_mains - 1,
3483                                (vlib_frame_queue_t *) (~0));
3484     }
3485
3486   from = vlib_frame_vector_args (frame);
3487   n_left_from = frame->n_vectors;
3488
3489   while (n_left_from > 0)
3490     {
3491       u32 bi0;
3492       vlib_buffer_t *b0;
3493       u32 sw_if_index0;
3494       u32 rx_fib_index0;
3495       ip4_header_t * ip0;
3496       u8 do_handoff;
3497
3498       bi0 = from[0];
3499       from += 1;
3500       n_left_from -= 1;
3501
3502       b0 = vlib_get_buffer (vm, bi0);
3503
3504       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3505       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3506
3507       ip0 = vlib_buffer_get_current (b0);
3508
3509       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3510
3511       if (PREDICT_FALSE (next_worker_index != thread_index))
3512         {
3513           do_handoff = 1;
3514
3515           if (next_worker_index != current_worker_index)
3516             {
3517               fq = is_vlib_frame_queue_congested (
3518                 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3519                 congested_handoff_queue_by_worker_index);
3520
3521               if (fq)
3522                 {
3523                   /* if this is 1st frame */
3524                   if (!d)
3525                     {
3526                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
3527                       to_next_drop = vlib_frame_vector_args (d);
3528                     }
3529
3530                   to_next_drop[0] = bi0;
3531                   to_next_drop += 1;
3532                   d->n_vectors++;
3533                   b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3534                   goto trace0;
3535                 }
3536
3537               if (hf)
3538                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3539
3540               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3541                                                       next_worker_index,
3542                                                       handoff_queue_elt_by_worker_index);
3543
3544               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3545               to_next_worker = &hf->buffer_index[hf->n_vectors];
3546               current_worker_index = next_worker_index;
3547             }
3548
3549           /* enqueue to correct worker thread */
3550           to_next_worker[0] = bi0;
3551           to_next_worker++;
3552           n_left_to_next_worker--;
3553
3554           if (n_left_to_next_worker == 0)
3555             {
3556               hf->n_vectors = VLIB_FRAME_SIZE;
3557               vlib_put_frame_queue_elt (hf);
3558               current_worker_index = ~0;
3559               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3560               hf = 0;
3561             }
3562         }
3563       else
3564         {
3565           do_handoff = 0;
3566           /* if this is 1st frame */
3567           if (!f)
3568             {
3569               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3570               to_next = vlib_frame_vector_args (f);
3571             }
3572
3573           to_next[0] = bi0;
3574           to_next += 1;
3575           f->n_vectors++;
3576         }
3577
3578 trace0:
3579       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3580                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3581         {
3582           snat_out2in_worker_handoff_trace_t *t =
3583             vlib_add_trace (vm, node, b0, sizeof (*t));
3584           t->next_worker_index = next_worker_index;
3585           t->do_handoff = do_handoff;
3586         }
3587     }
3588
3589   if (f)
3590     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3591
3592   if (d)
3593     vlib_put_frame_to_node (vm, sm->error_node_index, d);
3594
3595   if (hf)
3596     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3597
3598   /* Ship frames to the worker nodes */
3599   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3600     {
3601       if (handoff_queue_elt_by_worker_index[i])
3602         {
3603           hf = handoff_queue_elt_by_worker_index[i];
3604           /*
3605            * It works better to let the handoff node
3606            * rate-adapt, always ship the handoff queue element.
3607            */
3608           if (1 || hf->n_vectors == hf->last_n_vectors)
3609             {
3610               vlib_put_frame_queue_elt (hf);
3611               handoff_queue_elt_by_worker_index[i] = 0;
3612             }
3613           else
3614             hf->last_n_vectors = hf->n_vectors;
3615         }
3616       congested_handoff_queue_by_worker_index[i] =
3617         (vlib_frame_queue_t *) (~0);
3618     }
3619   hf = 0;
3620   current_worker_index = ~0;
3621   return frame->n_vectors;
3622 }
3623
3624 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3625   .function = snat_out2in_worker_handoff_fn,
3626   .name = "nat44-out2in-worker-handoff",
3627   .vector_size = sizeof (u32),
3628   .format_trace = format_snat_out2in_worker_handoff_trace,
3629   .type = VLIB_NODE_TYPE_INTERNAL,
3630
3631   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3632   .error_strings = snat_out2in_error_strings,
3633
3634   .n_next_nodes = 1,
3635
3636   .next_nodes = {
3637     [0] = "error-drop",
3638   },
3639 };
3640
3641 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3642
3643 static uword
3644 snat_out2in_fast_node_fn (vlib_main_t * vm,
3645                           vlib_node_runtime_t * node,
3646                           vlib_frame_t * frame)
3647 {
3648   u32 n_left_from, * from, * to_next;
3649   snat_out2in_next_t next_index;
3650   u32 pkts_processed = 0;
3651   snat_main_t * sm = &snat_main;
3652
3653   from = vlib_frame_vector_args (frame);
3654   n_left_from = frame->n_vectors;
3655   next_index = node->cached_next_index;
3656
3657   while (n_left_from > 0)
3658     {
3659       u32 n_left_to_next;
3660
3661       vlib_get_next_frame (vm, node, next_index,
3662                            to_next, n_left_to_next);
3663
3664       while (n_left_from > 0 && n_left_to_next > 0)
3665         {
3666           u32 bi0;
3667           vlib_buffer_t * b0;
3668           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3669           u32 sw_if_index0;
3670           ip4_header_t * ip0;
3671           ip_csum_t sum0;
3672           u32 new_addr0, old_addr0;
3673           u16 new_port0, old_port0;
3674           udp_header_t * udp0;
3675           tcp_header_t * tcp0;
3676           icmp46_header_t * icmp0;
3677           snat_session_key_t key0, sm0;
3678           u32 proto0;
3679           u32 rx_fib_index0;
3680
3681           /* speculatively enqueue b0 to the current next frame */
3682           bi0 = from[0];
3683           to_next[0] = bi0;
3684           from += 1;
3685           to_next += 1;
3686           n_left_from -= 1;
3687           n_left_to_next -= 1;
3688
3689           b0 = vlib_get_buffer (vm, bi0);
3690
3691           ip0 = vlib_buffer_get_current (b0);
3692           udp0 = ip4_next_header (ip0);
3693           tcp0 = (tcp_header_t *) udp0;
3694           icmp0 = (icmp46_header_t *) udp0;
3695
3696           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3697           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3698
3699           vnet_feature_next (&next0, b0);
3700
3701           if (PREDICT_FALSE(ip0->ttl == 1))
3702             {
3703               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3704               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3705                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3706                                            0);
3707               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3708               goto trace00;
3709             }
3710
3711           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3712
3713           if (PREDICT_FALSE (proto0 == ~0))
3714               goto trace00;
3715
3716           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3717             {
3718               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3719                                   rx_fib_index0, node, next0, ~0, 0, 0);
3720               goto trace00;
3721             }
3722
3723           key0.addr = ip0->dst_address;
3724           key0.port = udp0->dst_port;
3725           key0.fib_index = rx_fib_index0;
3726
3727           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3728             {
3729               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3730               goto trace00;
3731             }
3732
3733           new_addr0 = sm0.addr.as_u32;
3734           new_port0 = sm0.port;
3735           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3736           old_addr0 = ip0->dst_address.as_u32;
3737           ip0->dst_address.as_u32 = new_addr0;
3738
3739           sum0 = ip0->checksum;
3740           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3741                                  ip4_header_t,
3742                                  dst_address /* changed member */);
3743           ip0->checksum = ip_csum_fold (sum0);
3744
3745           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3746             {
3747                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3748                 {
3749                   old_port0 = tcp0->dst_port;
3750                   tcp0->dst_port = new_port0;
3751
3752                   sum0 = tcp0->checksum;
3753                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3754                                          ip4_header_t,
3755                                          dst_address /* changed member */);
3756
3757                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3758                                          ip4_header_t /* cheat */,
3759                                          length /* changed member */);
3760                   tcp0->checksum = ip_csum_fold(sum0);
3761                 }
3762               else
3763                 {
3764                   old_port0 = udp0->dst_port;
3765                   udp0->dst_port = new_port0;
3766                   udp0->checksum = 0;
3767                 }
3768             }
3769           else
3770             {
3771               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3772                 {
3773                   sum0 = tcp0->checksum;
3774                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3775                                          ip4_header_t,
3776                                          dst_address /* changed member */);
3777
3778                   tcp0->checksum = ip_csum_fold(sum0);
3779                 }
3780             }
3781
3782         trace00:
3783
3784           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3785                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3786             {
3787               snat_out2in_trace_t *t =
3788                  vlib_add_trace (vm, node, b0, sizeof (*t));
3789               t->sw_if_index = sw_if_index0;
3790               t->next_index = next0;
3791             }
3792
3793           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3794
3795           /* verify speculative enqueue, maybe switch current next frame */
3796           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3797                                            to_next, n_left_to_next,
3798                                            bi0, next0);
3799         }
3800
3801       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3802     }
3803
3804   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3805                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3806                                pkts_processed);
3807   return frame->n_vectors;
3808 }
3809
3810 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3811   .function = snat_out2in_fast_node_fn,
3812   .name = "nat44-out2in-fast",
3813   .vector_size = sizeof (u32),
3814   .format_trace = format_snat_out2in_fast_trace,
3815   .type = VLIB_NODE_TYPE_INTERNAL,
3816
3817   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3818   .error_strings = snat_out2in_error_strings,
3819
3820   .runtime_data_bytes = sizeof (snat_runtime_t),
3821
3822   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3823
3824   /* edit / add dispositions here */
3825   .next_nodes = {
3826     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3827     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3828     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3829     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3830   },
3831 };
3832 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);