src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29 #include <nat/nat_inlines.h>
  30
  31 #include <vppinfra/hash.h>
  32 #include <vppinfra/error.h>
  33 #include <vppinfra/elog.h>
  34
  35 typedef struct {
  36   u32 sw_if_index;
  37   u32 next_index;
  38   u32 session_index;
  39 } snat_out2in_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_out2in_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  52
  53   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  54               t->sw_if_index, t->next_index, t->session_index);
  55   return s;
  56 }
  57
  58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  63
  64   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  65               t->sw_if_index, t->next_index);
  66   return s;
  67 }
  68
  69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  70 {
  71   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  72   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  73   snat_out2in_worker_handoff_trace_t * t =
  74     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  75   char * m;
  76
  77   m = t->do_handoff ? "next worker" : "same worker";
  78   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  79
  80   return s;
  81 }
  82
  83 typedef struct {
  84   u32 sw_if_index;
  85   u32 next_index;
  86   u8 cached;
  87 } nat44_out2in_reass_trace_t;
  88
  89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  90 {
  91   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  92   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  93   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  94
  95   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  96               t->sw_if_index, t->next_index,
  97               t->cached ? "cached" : "translated");
  98
  99   return s;
 100 }
 101
 102 vlib_node_registration_t snat_out2in_node;
 103 vlib_node_registration_t snat_out2in_fast_node;
 104 vlib_node_registration_t snat_out2in_worker_handoff_node;
 105 vlib_node_registration_t snat_det_out2in_node;
 106 vlib_node_registration_t nat44_out2in_reass_node;
 107 vlib_node_registration_t nat44_ed_out2in_node;
 108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
 109
 110 #define foreach_snat_out2in_error                       \
 111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 112 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 113 _(OUT_OF_PORTS, "Out of ports")                         \
 114 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 115 _(NO_TRANSLATION, "No translation")                     \
 116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 117 _(DROP_FRAGMENT, "Drop fragment")                       \
 118 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 120 _(FQ_CONGESTED, "Handoff frame queue congested")
 121
 122 typedef enum {
 123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 124   foreach_snat_out2in_error
 125 #undef _
 126   SNAT_OUT2IN_N_ERROR,
 127 } snat_out2in_error_t;
 128
 129 static char * snat_out2in_error_strings[] = {
 130 #define _(sym,string) string,
 131   foreach_snat_out2in_error
 132 #undef _
 133 };
 134
 135 typedef enum {
 136   SNAT_OUT2IN_NEXT_DROP,
 137   SNAT_OUT2IN_NEXT_LOOKUP,
 138   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 139   SNAT_OUT2IN_NEXT_REASS,
 140   SNAT_OUT2IN_N_NEXT,
 141 } snat_out2in_next_t;
 142
 143 /**
 144  * @brief Create session for static mapping.
 145  *
 146  * Create NAT session initiated by host from external network with static
 147  * mapping.
 148  *
 149  * @param sm     NAT main.
 150  * @param b0     Vlib buffer.
 151  * @param in2out In2out NAT44 session key.
 152  * @param out2in Out2in NAT44 session key.
 153  * @param node   Vlib node.
 154  *
 155  * @returns SNAT session if successfully created otherwise 0.
 156  */
 157 static inline snat_session_t *
 158 create_session_for_static_mapping (snat_main_t *sm,
 159                                    vlib_buffer_t *b0,
 160                                    snat_session_key_t in2out,
 161                                    snat_session_key_t out2in,
 162                                    vlib_node_runtime_t * node,
 163                                    u32 thread_index)
 164 {
 165   snat_user_t *u;
 166   snat_session_t *s;
 167   clib_bihash_kv_8_8_t kv0;
 168   ip4_header_t *ip0;
 169   udp_header_t *udp0;
 170
 171   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 172     {
 173       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 174       nat_log_notice ("maximum sessions exceeded");
 175       return 0;
 176     }
 177
 178   ip0 = vlib_buffer_get_current (b0);
 179   udp0 = ip4_next_header (ip0);
 180
 181   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 182   if (!u)
 183     {
 184       nat_log_warn ("create NAT user failed");
 185       return 0;
 186     }
 187
 188   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 189   if (!s)
 190     {
 191       nat_log_warn ("create NAT session failed");
 192       return 0;
 193     }
 194
 195   s->outside_address_index = ~0;
 196   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 197   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 198   s->ext_host_port = udp0->src_port;
 199   user_session_increment (sm, u, 1 /* static */);
 200   s->in2out = in2out;
 201   s->out2in = out2in;
 202   s->in2out.protocol = out2in.protocol;
 203
 204   /* Add to translation hashes */
 205   kv0.key = s->in2out.as_u64;
 206   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 207   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 208                                1 /* is_add */))
 209       nat_log_notice ("in2out key add failed");
 210
 211   kv0.key = s->out2in.as_u64;
 212
 213   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 214                                1 /* is_add */))
 215       nat_log_notice ("out2in key add failed");
 216
 217   /* log NAT event */
 218   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 219                                       s->out2in.addr.as_u32,
 220                                       s->in2out.protocol,
 221                                       s->in2out.port,
 222                                       s->out2in.port,
 223                                       s->in2out.fib_index);
 224    return s;
 225 }
 226
 227 static_always_inline
 228 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 229                                  snat_session_key_t *p_key0)
 230 {
 231   icmp46_header_t *icmp0;
 232   snat_session_key_t key0;
 233   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 234   ip4_header_t *inner_ip0;
 235   void *l4_header = 0;
 236   icmp46_header_t *inner_icmp0;
 237
 238   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 239   echo0 = (icmp_echo_header_t *)(icmp0+1);
 240
 241   if (!icmp_is_error_message (icmp0))
 242     {
 243       key0.protocol = SNAT_PROTOCOL_ICMP;
 244       key0.addr = ip0->dst_address;
 245       key0.port = echo0->identifier;
 246     }
 247   else
 248     {
 249       inner_ip0 = (ip4_header_t *)(echo0+1);
 250       l4_header = ip4_next_header (inner_ip0);
 251       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 252       key0.addr = inner_ip0->src_address;
 253       switch (key0.protocol)
 254         {
 255         case SNAT_PROTOCOL_ICMP:
 256           inner_icmp0 = (icmp46_header_t*)l4_header;
 257           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 258           key0.port = inner_echo0->identifier;
 259           break;
 260         case SNAT_PROTOCOL_UDP:
 261         case SNAT_PROTOCOL_TCP:
 262           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 263           break;
 264         default:
 265           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 266         }
 267     }
 268   *p_key0 = key0;
 269   return -1; /* success */
 270 }
 271
 272 /**
 273  * Get address and port values to be used for ICMP packet translation
 274  * and create session if needed
 275  *
 276  * @param[in,out] sm             NAT main
 277  * @param[in,out] node           NAT node runtime
 278  * @param[in] thread_index       thread index
 279  * @param[in,out] b0             buffer containing packet to be translated
 280  * @param[out] p_proto           protocol used for matching
 281  * @param[out] p_value           address and port after NAT translation
 282  * @param[out] p_dont_translate  if packet should not be translated
 283  * @param d                      optional parameter
 284  * @param e                      optional parameter
 285  */
 286 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 287                            u32 thread_index, vlib_buffer_t *b0,
 288                            ip4_header_t *ip0, u8 *p_proto,
 289                            snat_session_key_t *p_value,
 290                            u8 *p_dont_translate, void *d, void *e)
 291 {
 292   icmp46_header_t *icmp0;
 293   u32 sw_if_index0;
 294   u32 rx_fib_index0;
 295   snat_session_key_t key0;
 296   snat_session_key_t sm0;
 297   snat_session_t *s0 = 0;
 298   u8 dont_translate = 0;
 299   clib_bihash_kv_8_8_t kv0, value0;
 300   u8 is_addr_only;
 301   u32 next0 = ~0;
 302   int err;
 303
 304   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 305   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 306   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 307
 308   key0.protocol = 0;
 309
 310   err = icmp_get_key (ip0, &key0);
 311   if (err != -1)
 312     {
 313       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 314       next0 = SNAT_OUT2IN_NEXT_DROP;
 315       goto out;
 316     }
 317   key0.fib_index = rx_fib_index0;
 318
 319   kv0.key = key0.as_u64;
 320
 321   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 322                               &value0))
 323     {
 324       /* Try to match static mapping by external address and port,
 325          destination address and port in packet */
 326       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 327         {
 328           if (!sm->forwarding_enabled)
 329             {
 330               /* Don't NAT packet aimed at the intfc address */
 331               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 332                                                   ip0->dst_address.as_u32)))
 333                 {
 334                   dont_translate = 1;
 335                   goto out;
 336                 }
 337               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 338               next0 = SNAT_OUT2IN_NEXT_DROP;
 339               goto out;
 340             }
 341           else
 342             {
 343               dont_translate = 1;
 344               goto out;
 345             }
 346         }
 347
 348       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 349                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 350         {
 351           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 352           next0 = SNAT_OUT2IN_NEXT_DROP;
 353           goto out;
 354         }
 355
 356       /* Create session initiated by host from external network */
 357       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 358                                              node, thread_index);
 359
 360       if (!s0)
 361         {
 362           next0 = SNAT_OUT2IN_NEXT_DROP;
 363           goto out;
 364         }
 365     }
 366   else
 367     {
 368       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 369                         icmp0->type != ICMP4_echo_request &&
 370                         !icmp_is_error_message (icmp0)))
 371         {
 372           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 373           next0 = SNAT_OUT2IN_NEXT_DROP;
 374           goto out;
 375         }
 376
 377       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 378                               value0.value);
 379     }
 380
 381 out:
 382   *p_proto = key0.protocol;
 383   if (s0)
 384     *p_value = s0->in2out;
 385   *p_dont_translate = dont_translate;
 386   if (d)
 387     *(snat_session_t**)d = s0;
 388   return next0;
 389 }
 390
 391 /**
 392  * Get address and port values to be used for ICMP packet translation
 393  *
 394  * @param[in] sm                 NAT main
 395  * @param[in,out] node           NAT node runtime
 396  * @param[in] thread_index       thread index
 397  * @param[in,out] b0             buffer containing packet to be translated
 398  * @param[out] p_proto           protocol used for matching
 399  * @param[out] p_value           address and port after NAT translation
 400  * @param[out] p_dont_translate  if packet should not be translated
 401  * @param d                      optional parameter
 402  * @param e                      optional parameter
 403  */
 404 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 405                            u32 thread_index, vlib_buffer_t *b0,
 406                            ip4_header_t *ip0, u8 *p_proto,
 407                            snat_session_key_t *p_value,
 408                            u8 *p_dont_translate, void *d, void *e)
 409 {
 410   icmp46_header_t *icmp0;
 411   u32 sw_if_index0;
 412   u32 rx_fib_index0;
 413   snat_session_key_t key0;
 414   snat_session_key_t sm0;
 415   u8 dont_translate = 0;
 416   u8 is_addr_only;
 417   u32 next0 = ~0;
 418   int err;
 419
 420   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 421   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 422   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 423
 424   err = icmp_get_key (ip0, &key0);
 425   if (err != -1)
 426     {
 427       b0->error = node->errors[err];
 428       next0 = SNAT_OUT2IN_NEXT_DROP;
 429       goto out2;
 430     }
 431   key0.fib_index = rx_fib_index0;
 432
 433   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 434     {
 435       /* Don't NAT packet aimed at the intfc address */
 436       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 437         {
 438           dont_translate = 1;
 439           goto out;
 440         }
 441       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 442       next0 = SNAT_OUT2IN_NEXT_DROP;
 443       goto out;
 444     }
 445
 446   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 447                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 448                     !icmp_is_error_message (icmp0)))
 449     {
 450       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 451       next0 = SNAT_OUT2IN_NEXT_DROP;
 452       goto out;
 453     }
 454
 455 out:
 456   *p_value = sm0;
 457 out2:
 458   *p_proto = key0.protocol;
 459   *p_dont_translate = dont_translate;
 460   return next0;
 461 }
 462
 463 static inline u32 icmp_out2in (snat_main_t *sm,
 464                                vlib_buffer_t * b0,
 465                                ip4_header_t * ip0,
 466                                icmp46_header_t * icmp0,
 467                                u32 sw_if_index0,
 468                                u32 rx_fib_index0,
 469                                vlib_node_runtime_t * node,
 470                                u32 next0,
 471                                u32 thread_index,
 472                                void *d,
 473                                void *e)
 474 {
 475   snat_session_key_t sm0;
 476   u8 protocol;
 477   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 478   ip4_header_t *inner_ip0 = 0;
 479   void *l4_header = 0;
 480   icmp46_header_t *inner_icmp0;
 481   u8 dont_translate;
 482   u32 new_addr0, old_addr0;
 483   u16 old_id0, new_id0;
 484   ip_csum_t sum0;
 485   u16 checksum0;
 486   u32 next0_tmp;
 487
 488   echo0 = (icmp_echo_header_t *)(icmp0+1);
 489
 490   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 491                                        &protocol, &sm0, &dont_translate, d, e);
 492   if (next0_tmp != ~0)
 493     next0 = next0_tmp;
 494   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 495     goto out;
 496
 497   sum0 = ip_incremental_checksum (0, icmp0,
 498                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 499   checksum0 = ~ip_csum_fold (sum0);
 500   if (checksum0 != 0 && checksum0 != 0xffff)
 501     {
 502       next0 = SNAT_OUT2IN_NEXT_DROP;
 503       goto out;
 504     }
 505
 506   old_addr0 = ip0->dst_address.as_u32;
 507   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 508   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 509
 510   sum0 = ip0->checksum;
 511   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 512                          dst_address /* changed member */);
 513   ip0->checksum = ip_csum_fold (sum0);
 514
 515   if (icmp0->checksum == 0)
 516     icmp0->checksum = 0xffff;
 517
 518   if (!icmp_is_error_message (icmp0))
 519     {
 520       new_id0 = sm0.port;
 521       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 522         {
 523           old_id0 = echo0->identifier;
 524           new_id0 = sm0.port;
 525           echo0->identifier = new_id0;
 526
 527           sum0 = icmp0->checksum;
 528           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 529                                  identifier /* changed member */);
 530           icmp0->checksum = ip_csum_fold (sum0);
 531         }
 532     }
 533   else
 534     {
 535       inner_ip0 = (ip4_header_t *)(echo0+1);
 536       l4_header = ip4_next_header (inner_ip0);
 537
 538       if (!ip4_header_checksum_is_valid (inner_ip0))
 539         {
 540           next0 = SNAT_OUT2IN_NEXT_DROP;
 541           goto out;
 542         }
 543
 544       old_addr0 = inner_ip0->src_address.as_u32;
 545       inner_ip0->src_address = sm0.addr;
 546       new_addr0 = inner_ip0->src_address.as_u32;
 547
 548       sum0 = icmp0->checksum;
 549       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 550                              src_address /* changed member */);
 551       icmp0->checksum = ip_csum_fold (sum0);
 552
 553       switch (protocol)
 554         {
 555         case SNAT_PROTOCOL_ICMP:
 556           inner_icmp0 = (icmp46_header_t*)l4_header;
 557           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 558
 559           old_id0 = inner_echo0->identifier;
 560           new_id0 = sm0.port;
 561           inner_echo0->identifier = new_id0;
 562
 563           sum0 = icmp0->checksum;
 564           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 565                                  identifier);
 566           icmp0->checksum = ip_csum_fold (sum0);
 567           break;
 568         case SNAT_PROTOCOL_UDP:
 569         case SNAT_PROTOCOL_TCP:
 570           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 571           new_id0 = sm0.port;
 572           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 573
 574           sum0 = icmp0->checksum;
 575           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 576                                  src_port);
 577           icmp0->checksum = ip_csum_fold (sum0);
 578           break;
 579         default:
 580           ASSERT(0);
 581         }
 582     }
 583
 584 out:
 585   return next0;
 586 }
 587
 588
 589 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 590                                          vlib_buffer_t * b0,
 591                                          ip4_header_t * ip0,
 592                                          icmp46_header_t * icmp0,
 593                                          u32 sw_if_index0,
 594                                          u32 rx_fib_index0,
 595                                          vlib_node_runtime_t * node,
 596                                          u32 next0, f64 now,
 597                                          u32 thread_index,
 598                                          snat_session_t ** p_s0)
 599 {
 600   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 601                       next0, thread_index, p_s0, 0);
 602   snat_session_t * s0 = *p_s0;
 603   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 604     {
 605       /* Accounting */
 606       nat44_session_update_counters (s0, now,
 607                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
 608       /* Per-user LRU list maintenance */
 609       nat44_session_update_lru (sm, s0, thread_index);
 610     }
 611   return next0;
 612 }
 613
 614 static int
 615 nat_out2in_sm_unknown_proto (snat_main_t *sm,
 616                              vlib_buffer_t * b,
 617                              ip4_header_t * ip,
 618                              u32 rx_fib_index)
 619 {
 620   clib_bihash_kv_8_8_t kv, value;
 621   snat_static_mapping_t *m;
 622   snat_session_key_t m_key;
 623   u32 old_addr, new_addr;
 624   ip_csum_t sum;
 625
 626   m_key.addr = ip->dst_address;
 627   m_key.port = 0;
 628   m_key.protocol = 0;
 629   m_key.fib_index = 0;
 630   kv.key = m_key.as_u64;
 631   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 632     return 1;
 633
 634   m = pool_elt_at_index (sm->static_mappings, value.value);
 635
 636   old_addr = ip->dst_address.as_u32;
 637   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 638   sum = ip->checksum;
 639   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 640   ip->checksum = ip_csum_fold (sum);
 641
 642   vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 643   return 0;
 644 }
 645
 646 static uword
 647 snat_out2in_node_fn (vlib_main_t * vm,
 648                   vlib_node_runtime_t * node,
 649                   vlib_frame_t * frame)
 650 {
 651   u32 n_left_from, * from, * to_next;
 652   snat_out2in_next_t next_index;
 653   u32 pkts_processed = 0;
 654   snat_main_t * sm = &snat_main;
 655   f64 now = vlib_time_now (vm);
 656   u32 thread_index = vm->thread_index;
 657
 658   from = vlib_frame_vector_args (frame);
 659   n_left_from = frame->n_vectors;
 660   next_index = node->cached_next_index;
 661
 662   while (n_left_from > 0)
 663     {
 664       u32 n_left_to_next;
 665
 666       vlib_get_next_frame (vm, node, next_index,
 667                            to_next, n_left_to_next);
 668
 669       while (n_left_from >= 4 && n_left_to_next >= 2)
 670         {
 671           u32 bi0, bi1;
 672           vlib_buffer_t * b0, * b1;
 673           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 674           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 675           u32 sw_if_index0, sw_if_index1;
 676           ip4_header_t * ip0, *ip1;
 677           ip_csum_t sum0, sum1;
 678           u32 new_addr0, old_addr0;
 679           u16 new_port0, old_port0;
 680           u32 new_addr1, old_addr1;
 681           u16 new_port1, old_port1;
 682           udp_header_t * udp0, * udp1;
 683           tcp_header_t * tcp0, * tcp1;
 684           icmp46_header_t * icmp0, * icmp1;
 685           snat_session_key_t key0, key1, sm0, sm1;
 686           u32 rx_fib_index0, rx_fib_index1;
 687           u32 proto0, proto1;
 688           snat_session_t * s0 = 0, * s1 = 0;
 689           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 690
 691           /* Prefetch next iteration. */
 692           {
 693             vlib_buffer_t * p2, * p3;
 694
 695             p2 = vlib_get_buffer (vm, from[2]);
 696             p3 = vlib_get_buffer (vm, from[3]);
 697
 698             vlib_prefetch_buffer_header (p2, LOAD);
 699             vlib_prefetch_buffer_header (p3, LOAD);
 700
 701             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 702             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 703           }
 704
 705           /* speculatively enqueue b0 and b1 to the current next frame */
 706           to_next[0] = bi0 = from[0];
 707           to_next[1] = bi1 = from[1];
 708           from += 2;
 709           to_next += 2;
 710           n_left_from -= 2;
 711           n_left_to_next -= 2;
 712
 713           b0 = vlib_get_buffer (vm, bi0);
 714           b1 = vlib_get_buffer (vm, bi1);
 715
 716           vnet_buffer (b0)->snat.flags = 0;
 717           vnet_buffer (b1)->snat.flags = 0;
 718
 719           ip0 = vlib_buffer_get_current (b0);
 720           udp0 = ip4_next_header (ip0);
 721           tcp0 = (tcp_header_t *) udp0;
 722           icmp0 = (icmp46_header_t *) udp0;
 723
 724           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 725           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 726                                    sw_if_index0);
 727
 728           if (PREDICT_FALSE(ip0->ttl == 1))
 729             {
 730               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 731               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 732                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 733                                            0);
 734               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 735               goto trace0;
 736             }
 737
 738           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 739
 740           if (PREDICT_FALSE (proto0 == ~0))
 741             {
 742               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
 743                 {
 744                   if (!sm->forwarding_enabled)
 745                     {
 746                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 747                       next0 = SNAT_OUT2IN_NEXT_DROP;
 748                     }
 749                 }
 750               goto trace0;
 751             }
 752
 753           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 754             {
 755               next0 = icmp_out2in_slow_path
 756                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 757                  next0, now, thread_index, &s0);
 758               goto trace0;
 759             }
 760
 761           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
 762             {
 763               next0 = SNAT_OUT2IN_NEXT_REASS;
 764               goto trace0;
 765             }
 766
 767           key0.addr = ip0->dst_address;
 768           key0.port = udp0->dst_port;
 769           key0.protocol = proto0;
 770           key0.fib_index = rx_fib_index0;
 771
 772           kv0.key = key0.as_u64;
 773
 774           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 775                                       &kv0, &value0))
 776             {
 777               /* Try to match static mapping by external address and port,
 778                  destination address and port in packet */
 779               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 780                 {
 781                   /*
 782                    * Send DHCP packets to the ipv4 stack, or we won't
 783                    * be able to use dhcp client on the outside interface
 784                    */
 785                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 786                       && (udp0->dst_port ==
 787                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 788                     {
 789                       vnet_feature_next (&next0, b0);
 790                       goto trace0;
 791                     }
 792
 793                   if (!sm->forwarding_enabled)
 794                     {
 795                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 796                       next0 = SNAT_OUT2IN_NEXT_DROP;
 797                       goto trace0;
 798                     }
 799                 }
 800
 801               /* Create session initiated by host from external network */
 802               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 803                                                      thread_index);
 804               if (!s0)
 805                 {
 806                   next0 = SNAT_OUT2IN_NEXT_DROP;
 807                   goto trace0;
 808                 }
 809             }
 810           else
 811             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 812                                     value0.value);
 813
 814           old_addr0 = ip0->dst_address.as_u32;
 815           ip0->dst_address = s0->in2out.addr;
 816           new_addr0 = ip0->dst_address.as_u32;
 817           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 818
 819           sum0 = ip0->checksum;
 820           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 821                                  ip4_header_t,
 822                                  dst_address /* changed member */);
 823           ip0->checksum = ip_csum_fold (sum0);
 824
 825           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 826             {
 827               old_port0 = tcp0->dst_port;
 828               tcp0->dst_port = s0->in2out.port;
 829               new_port0 = tcp0->dst_port;
 830
 831               sum0 = tcp0->checksum;
 832               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 833                                      ip4_header_t,
 834                                      dst_address /* changed member */);
 835
 836               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 837                                      ip4_header_t /* cheat */,
 838                                      length /* changed member */);
 839               tcp0->checksum = ip_csum_fold(sum0);
 840             }
 841           else
 842             {
 843               old_port0 = udp0->dst_port;
 844               udp0->dst_port = s0->in2out.port;
 845               udp0->checksum = 0;
 846             }
 847
 848           /* Accounting */
 849           nat44_session_update_counters (s0, now,
 850                                          vlib_buffer_length_in_chain (vm, b0));
 851           /* Per-user LRU list maintenance */
 852           nat44_session_update_lru (sm, s0, thread_index);
 853         trace0:
 854
 855           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 856                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 857             {
 858               snat_out2in_trace_t *t =
 859                  vlib_add_trace (vm, node, b0, sizeof (*t));
 860               t->sw_if_index = sw_if_index0;
 861               t->next_index = next0;
 862               t->session_index = ~0;
 863               if (s0)
 864                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 865             }
 866
 867           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 868
 869
 870           ip1 = vlib_buffer_get_current (b1);
 871           udp1 = ip4_next_header (ip1);
 872           tcp1 = (tcp_header_t *) udp1;
 873           icmp1 = (icmp46_header_t *) udp1;
 874
 875           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 876           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 877                                    sw_if_index1);
 878
 879           if (PREDICT_FALSE(ip1->ttl == 1))
 880             {
 881               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 882               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 883                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 884                                            0);
 885               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 886               goto trace1;
 887             }
 888
 889           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 890
 891           if (PREDICT_FALSE (proto1 == ~0))
 892             {
 893               if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
 894                 {
 895                   if (!sm->forwarding_enabled)
 896                     {
 897                       b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 898                       next1 = SNAT_OUT2IN_NEXT_DROP;
 899                     }
 900                 }
 901               goto trace1;
 902             }
 903
 904           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 905             {
 906               next1 = icmp_out2in_slow_path
 907                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 908                  next1, now, thread_index, &s1);
 909               goto trace1;
 910             }
 911
 912           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
 913             {
 914               next1 = SNAT_OUT2IN_NEXT_REASS;
 915               goto trace1;
 916             }
 917
 918           key1.addr = ip1->dst_address;
 919           key1.port = udp1->dst_port;
 920           key1.protocol = proto1;
 921           key1.fib_index = rx_fib_index1;
 922
 923           kv1.key = key1.as_u64;
 924
 925           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 926                                       &kv1, &value1))
 927             {
 928               /* Try to match static mapping by external address and port,
 929                  destination address and port in packet */
 930               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
 931                 {
 932                   /*
 933                    * Send DHCP packets to the ipv4 stack, or we won't
 934                    * be able to use dhcp client on the outside interface
 935                    */
 936                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
 937                       && (udp1->dst_port ==
 938                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 939                     {
 940                       vnet_feature_next (&next1, b1);
 941                       goto trace1;
 942                     }
 943
 944                   if (!sm->forwarding_enabled)
 945                     {
 946                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 947                       next1 = SNAT_OUT2IN_NEXT_DROP;
 948                       goto trace1;
 949                     }
 950                 }
 951
 952               /* Create session initiated by host from external network */
 953               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 954                                                      thread_index);
 955               if (!s1)
 956                 {
 957                   next1 = SNAT_OUT2IN_NEXT_DROP;
 958                   goto trace1;
 959                 }
 960             }
 961           else
 962             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 963                                     value1.value);
 964
 965           old_addr1 = ip1->dst_address.as_u32;
 966           ip1->dst_address = s1->in2out.addr;
 967           new_addr1 = ip1->dst_address.as_u32;
 968           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 969
 970           sum1 = ip1->checksum;
 971           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 972                                  ip4_header_t,
 973                                  dst_address /* changed member */);
 974           ip1->checksum = ip_csum_fold (sum1);
 975
 976           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 977             {
 978               old_port1 = tcp1->dst_port;
 979               tcp1->dst_port = s1->in2out.port;
 980               new_port1 = tcp1->dst_port;
 981
 982               sum1 = tcp1->checksum;
 983               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 984                                      ip4_header_t,
 985                                      dst_address /* changed member */);
 986
 987               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 988                                      ip4_header_t /* cheat */,
 989                                      length /* changed member */);
 990               tcp1->checksum = ip_csum_fold(sum1);
 991             }
 992           else
 993             {
 994               old_port1 = udp1->dst_port;
 995               udp1->dst_port = s1->in2out.port;
 996               udp1->checksum = 0;
 997             }
 998
 999           /* Accounting */
1000           nat44_session_update_counters (s1, now,
1001                                          vlib_buffer_length_in_chain (vm, b1));
1002           /* Per-user LRU list maintenance */
1003           nat44_session_update_lru (sm, s1, thread_index);
1004         trace1:
1005
1006           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1007                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1008             {
1009               snat_out2in_trace_t *t =
1010                  vlib_add_trace (vm, node, b1, sizeof (*t));
1011               t->sw_if_index = sw_if_index1;
1012               t->next_index = next1;
1013               t->session_index = ~0;
1014               if (s1)
1015                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1016             }
1017
1018           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1019
1020           /* verify speculative enqueues, maybe switch current next frame */
1021           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1022                                            to_next, n_left_to_next,
1023                                            bi0, bi1, next0, next1);
1024         }
1025
1026       while (n_left_from > 0 && n_left_to_next > 0)
1027         {
1028           u32 bi0;
1029           vlib_buffer_t * b0;
1030           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1031           u32 sw_if_index0;
1032           ip4_header_t * ip0;
1033           ip_csum_t sum0;
1034           u32 new_addr0, old_addr0;
1035           u16 new_port0, old_port0;
1036           udp_header_t * udp0;
1037           tcp_header_t * tcp0;
1038           icmp46_header_t * icmp0;
1039           snat_session_key_t key0, sm0;
1040           u32 rx_fib_index0;
1041           u32 proto0;
1042           snat_session_t * s0 = 0;
1043           clib_bihash_kv_8_8_t kv0, value0;
1044
1045           /* speculatively enqueue b0 to the current next frame */
1046           bi0 = from[0];
1047           to_next[0] = bi0;
1048           from += 1;
1049           to_next += 1;
1050           n_left_from -= 1;
1051           n_left_to_next -= 1;
1052
1053           b0 = vlib_get_buffer (vm, bi0);
1054
1055           vnet_buffer (b0)->snat.flags = 0;
1056
1057           ip0 = vlib_buffer_get_current (b0);
1058           udp0 = ip4_next_header (ip0);
1059           tcp0 = (tcp_header_t *) udp0;
1060           icmp0 = (icmp46_header_t *) udp0;
1061
1062           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1063           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1064                                    sw_if_index0);
1065
1066           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1067
1068           if (PREDICT_FALSE (proto0 == ~0))
1069             {
1070               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1071                 {
1072                   if (!sm->forwarding_enabled)
1073                     {
1074                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1075                       next0 = SNAT_OUT2IN_NEXT_DROP;
1076                     }
1077                 }
1078               goto trace00;
1079             }
1080
1081           if (PREDICT_FALSE(ip0->ttl == 1))
1082             {
1083               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1084               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1085                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1086                                            0);
1087               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1088               goto trace00;
1089             }
1090
1091           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1092             {
1093               next0 = icmp_out2in_slow_path
1094                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1095                  next0, now, thread_index, &s0);
1096               goto trace00;
1097             }
1098
1099           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1100             {
1101               next0 = SNAT_OUT2IN_NEXT_REASS;
1102               goto trace00;
1103             }
1104
1105           key0.addr = ip0->dst_address;
1106           key0.port = udp0->dst_port;
1107           key0.protocol = proto0;
1108           key0.fib_index = rx_fib_index0;
1109
1110           kv0.key = key0.as_u64;
1111
1112           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1113                                       &kv0, &value0))
1114             {
1115               /* Try to match static mapping by external address and port,
1116                  destination address and port in packet */
1117               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1118                 {
1119                   /*
1120                    * Send DHCP packets to the ipv4 stack, or we won't
1121                    * be able to use dhcp client on the outside interface
1122                    */
1123                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1124                       && (udp0->dst_port ==
1125                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1126                     {
1127                       vnet_feature_next (&next0, b0);
1128                       goto trace00;
1129                     }
1130
1131                   if (!sm->forwarding_enabled)
1132                     {
1133                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1134                       next0 = SNAT_OUT2IN_NEXT_DROP;
1135                       goto trace00;
1136                     }
1137                 }
1138
1139               /* Create session initiated by host from external network */
1140               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1141                                                      thread_index);
1142               if (!s0)
1143                 {
1144                   next0 = SNAT_OUT2IN_NEXT_DROP;
1145                   goto trace00;
1146                 }
1147             }
1148           else
1149             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1150                                     value0.value);
1151
1152           old_addr0 = ip0->dst_address.as_u32;
1153           ip0->dst_address = s0->in2out.addr;
1154           new_addr0 = ip0->dst_address.as_u32;
1155           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1156
1157           sum0 = ip0->checksum;
1158           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1159                                  ip4_header_t,
1160                                  dst_address /* changed member */);
1161           ip0->checksum = ip_csum_fold (sum0);
1162
1163           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1164             {
1165               old_port0 = tcp0->dst_port;
1166               tcp0->dst_port = s0->in2out.port;
1167               new_port0 = tcp0->dst_port;
1168
1169               sum0 = tcp0->checksum;
1170               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1171                                      ip4_header_t,
1172                                      dst_address /* changed member */);
1173
1174               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1175                                      ip4_header_t /* cheat */,
1176                                      length /* changed member */);
1177               tcp0->checksum = ip_csum_fold(sum0);
1178             }
1179           else
1180             {
1181               old_port0 = udp0->dst_port;
1182               udp0->dst_port = s0->in2out.port;
1183               udp0->checksum = 0;
1184             }
1185
1186           /* Accounting */
1187           nat44_session_update_counters (s0, now,
1188                                          vlib_buffer_length_in_chain (vm, b0));
1189           /* Per-user LRU list maintenance */
1190           nat44_session_update_lru (sm, s0, thread_index);
1191         trace00:
1192
1193           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1194                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1195             {
1196               snat_out2in_trace_t *t =
1197                  vlib_add_trace (vm, node, b0, sizeof (*t));
1198               t->sw_if_index = sw_if_index0;
1199               t->next_index = next0;
1200               t->session_index = ~0;
1201               if (s0)
1202                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1203             }
1204
1205           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1206
1207           /* verify speculative enqueue, maybe switch current next frame */
1208           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1209                                            to_next, n_left_to_next,
1210                                            bi0, next0);
1211         }
1212
1213       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1214     }
1215
1216   vlib_node_increment_counter (vm, snat_out2in_node.index,
1217                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1218                                pkts_processed);
1219   return frame->n_vectors;
1220 }
1221
1222 VLIB_REGISTER_NODE (snat_out2in_node) = {
1223   .function = snat_out2in_node_fn,
1224   .name = "nat44-out2in",
1225   .vector_size = sizeof (u32),
1226   .format_trace = format_snat_out2in_trace,
1227   .type = VLIB_NODE_TYPE_INTERNAL,
1228
1229   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1230   .error_strings = snat_out2in_error_strings,
1231
1232   .runtime_data_bytes = sizeof (snat_runtime_t),
1233
1234   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1235
1236   /* edit / add dispositions here */
1237   .next_nodes = {
1238     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1239     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1240     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1241     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1242   },
1243 };
1244 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1245
1246 static uword
1247 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1248                             vlib_node_runtime_t * node,
1249                             vlib_frame_t * frame)
1250 {
1251   u32 n_left_from, *from, *to_next;
1252   snat_out2in_next_t next_index;
1253   u32 pkts_processed = 0;
1254   snat_main_t *sm = &snat_main;
1255   f64 now = vlib_time_now (vm);
1256   u32 thread_index = vm->thread_index;
1257   snat_main_per_thread_data_t *per_thread_data =
1258     &sm->per_thread_data[thread_index];
1259   u32 *fragments_to_drop = 0;
1260   u32 *fragments_to_loopback = 0;
1261
1262   from = vlib_frame_vector_args (frame);
1263   n_left_from = frame->n_vectors;
1264   next_index = node->cached_next_index;
1265
1266   while (n_left_from > 0)
1267     {
1268       u32 n_left_to_next;
1269
1270       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1271
1272       while (n_left_from > 0 && n_left_to_next > 0)
1273        {
1274           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1275           vlib_buffer_t *b0;
1276           u32 next0;
1277           u8 cached0 = 0;
1278           ip4_header_t *ip0;
1279           nat_reass_ip4_t *reass0;
1280           udp_header_t * udp0;
1281           tcp_header_t * tcp0;
1282           snat_session_key_t key0, sm0;
1283           clib_bihash_kv_8_8_t kv0, value0;
1284           snat_session_t * s0 = 0;
1285           u16 old_port0, new_port0;
1286           ip_csum_t sum0;
1287
1288           /* speculatively enqueue b0 to the current next frame */
1289           bi0 = from[0];
1290           to_next[0] = bi0;
1291           from += 1;
1292           to_next += 1;
1293           n_left_from -= 1;
1294           n_left_to_next -= 1;
1295
1296           b0 = vlib_get_buffer (vm, bi0);
1297           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1298
1299           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1300           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1301                                                                sw_if_index0);
1302
1303           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1304             {
1305               next0 = SNAT_OUT2IN_NEXT_DROP;
1306               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1307               goto trace0;
1308             }
1309
1310           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1311           udp0 = ip4_next_header (ip0);
1312           tcp0 = (tcp_header_t *) udp0;
1313           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1314
1315           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1316                                                  ip0->dst_address,
1317                                                  ip0->fragment_id,
1318                                                  ip0->protocol,
1319                                                  1,
1320                                                  &fragments_to_drop);
1321
1322           if (PREDICT_FALSE (!reass0))
1323             {
1324               next0 = SNAT_OUT2IN_NEXT_DROP;
1325               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1326               nat_log_notice ("maximum reassemblies exceeded");
1327               goto trace0;
1328             }
1329
1330           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1331             {
1332               key0.addr = ip0->dst_address;
1333               key0.port = udp0->dst_port;
1334               key0.protocol = proto0;
1335               key0.fib_index = rx_fib_index0;
1336               kv0.key = key0.as_u64;
1337
1338               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1339                 {
1340                   /* Try to match static mapping by external address and port,
1341                      destination address and port in packet */
1342                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1343                     {
1344                       /*
1345                        * Send DHCP packets to the ipv4 stack, or we won't
1346                        * be able to use dhcp client on the outside interface
1347                        */
1348                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1349                           && (udp0->dst_port
1350                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1351                         {
1352                           vnet_feature_next (&next0, b0);
1353                           goto trace0;
1354                         }
1355
1356                       if (!sm->forwarding_enabled)
1357                         {
1358                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1359                           next0 = SNAT_OUT2IN_NEXT_DROP;
1360                         }
1361                       goto trace0;
1362                     }
1363
1364                   /* Create session initiated by host from external network */
1365                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1366                                                          thread_index);
1367                   if (!s0)
1368                     {
1369                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1370                       next0 = SNAT_OUT2IN_NEXT_DROP;
1371                       goto trace0;
1372                     }
1373                   reass0->sess_index = s0 - per_thread_data->sessions;
1374                   reass0->thread_index = thread_index;
1375                 }
1376               else
1377                 {
1378                   s0 = pool_elt_at_index (per_thread_data->sessions,
1379                                           value0.value);
1380                   reass0->sess_index = value0.value;
1381                 }
1382               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1383             }
1384           else
1385             {
1386               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1387                 {
1388                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1389                     {
1390                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1391                       nat_log_notice ("maximum fragments per reassembly exceeded");
1392                       next0 = SNAT_OUT2IN_NEXT_DROP;
1393                       goto trace0;
1394                     }
1395                   cached0 = 1;
1396                   goto trace0;
1397                 }
1398               s0 = pool_elt_at_index (per_thread_data->sessions,
1399                                       reass0->sess_index);
1400             }
1401
1402           old_addr0 = ip0->dst_address.as_u32;
1403           ip0->dst_address = s0->in2out.addr;
1404           new_addr0 = ip0->dst_address.as_u32;
1405           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1406
1407           sum0 = ip0->checksum;
1408           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1409                                  ip4_header_t,
1410                                  dst_address /* changed member */);
1411           ip0->checksum = ip_csum_fold (sum0);
1412
1413           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1414             {
1415               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1416                 {
1417                   old_port0 = tcp0->dst_port;
1418                   tcp0->dst_port = s0->in2out.port;
1419                   new_port0 = tcp0->dst_port;
1420
1421                   sum0 = tcp0->checksum;
1422                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1423                                          ip4_header_t,
1424                                          dst_address /* changed member */);
1425
1426                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1427                                          ip4_header_t /* cheat */,
1428                                          length /* changed member */);
1429                   tcp0->checksum = ip_csum_fold(sum0);
1430                 }
1431               else
1432                 {
1433                   old_port0 = udp0->dst_port;
1434                   udp0->dst_port = s0->in2out.port;
1435                   udp0->checksum = 0;
1436                 }
1437             }
1438
1439           /* Accounting */
1440           nat44_session_update_counters (s0, now,
1441                                          vlib_buffer_length_in_chain (vm, b0));
1442           /* Per-user LRU list maintenance */
1443           nat44_session_update_lru (sm, s0, thread_index);
1444
1445         trace0:
1446           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1447                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1448             {
1449               nat44_out2in_reass_trace_t *t =
1450                  vlib_add_trace (vm, node, b0, sizeof (*t));
1451               t->cached = cached0;
1452               t->sw_if_index = sw_if_index0;
1453               t->next_index = next0;
1454             }
1455
1456           if (cached0)
1457             {
1458               n_left_to_next++;
1459               to_next--;
1460             }
1461           else
1462             {
1463               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1464
1465               /* verify speculative enqueue, maybe switch current next frame */
1466               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1467                                                to_next, n_left_to_next,
1468                                                bi0, next0);
1469             }
1470
1471           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1472             {
1473               from = vlib_frame_vector_args (frame);
1474               u32 len = vec_len (fragments_to_loopback);
1475               if (len <= VLIB_FRAME_SIZE)
1476                 {
1477                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1478                   n_left_from = len;
1479                   vec_reset_length (fragments_to_loopback);
1480                 }
1481               else
1482                 {
1483                   clib_memcpy (from,
1484                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1485                                sizeof (u32) * VLIB_FRAME_SIZE);
1486                   n_left_from = VLIB_FRAME_SIZE;
1487                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1488                 }
1489             }
1490        }
1491
1492       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1493     }
1494
1495   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1496                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1497                                pkts_processed);
1498
1499   nat_send_all_to_node (vm, fragments_to_drop, node,
1500                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1501                         SNAT_OUT2IN_NEXT_DROP);
1502
1503   vec_free (fragments_to_drop);
1504   vec_free (fragments_to_loopback);
1505   return frame->n_vectors;
1506 }
1507
1508 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1509   .function = nat44_out2in_reass_node_fn,
1510   .name = "nat44-out2in-reass",
1511   .vector_size = sizeof (u32),
1512   .format_trace = format_nat44_out2in_reass_trace,
1513   .type = VLIB_NODE_TYPE_INTERNAL,
1514
1515   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1516   .error_strings = snat_out2in_error_strings,
1517
1518   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1519
1520   /* edit / add dispositions here */
1521   .next_nodes = {
1522     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1523     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1524     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1525     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1526   },
1527 };
1528 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1529                               nat44_out2in_reass_node_fn);
1530
1531 /*******************************/
1532 /*** endpoint-dependent mode ***/
1533 /*******************************/
1534 typedef enum {
1535   NAT44_ED_OUT2IN_NEXT_DROP,
1536   NAT44_ED_OUT2IN_NEXT_LOOKUP,
1537   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1538   NAT44_ED_OUT2IN_NEXT_IN2OUT,
1539   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1540   NAT44_ED_OUT2IN_N_NEXT,
1541 } nat44_ed_out2in_next_t;
1542
1543 typedef struct {
1544   u32 sw_if_index;
1545   u32 next_index;
1546   u32 session_index;
1547   u32 is_slow_path;
1548 } nat44_ed_out2in_trace_t;
1549
1550 static u8 *
1551 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1552 {
1553   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1554   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1555   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1556   char * tag;
1557
1558   tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1559
1560   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1561               t->sw_if_index, t->next_index, t->session_index);
1562
1563   return s;
1564 }
1565
1566 static snat_session_t *
1567 create_session_for_static_mapping_ed (snat_main_t * sm,
1568                                       vlib_buffer_t *b,
1569                                       snat_session_key_t l_key,
1570                                       snat_session_key_t e_key,
1571                                       vlib_node_runtime_t * node,
1572                                       u32 thread_index,
1573                                       twice_nat_type_t twice_nat,
1574                                       u8 is_lb)
1575 {
1576   snat_session_t *s;
1577   snat_user_t *u;
1578   ip4_header_t *ip;
1579   udp_header_t *udp;
1580   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1581   clib_bihash_kv_16_8_t kv;
1582   snat_session_key_t eh_key;
1583   u32 address_index;
1584
1585   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1586     {
1587       b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1588       nat_log_notice ("maximum sessions exceeded");
1589       return 0;
1590     }
1591
1592   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1593   if (!u)
1594     {
1595       nat_log_warn ("create NAT user failed");
1596       return 0;
1597     }
1598
1599   s = nat_session_alloc_or_recycle (sm, u, thread_index);
1600   if (!s)
1601     {
1602       nat_log_warn ("create NAT session failed");
1603       return 0;
1604     }
1605
1606   ip = vlib_buffer_get_current (b);
1607   udp = ip4_next_header (ip);
1608
1609   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1610   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1611   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1612   if (is_lb)
1613     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1614   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1615   s->outside_address_index = ~0;
1616   s->out2in = e_key;
1617   s->in2out = l_key;
1618   s->in2out.protocol = s->out2in.protocol;
1619   user_session_increment (sm, u, 1);
1620
1621   /* Add to lookup tables */
1622   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1623               e_key.fib_index, e_key.port, s->ext_host_port);
1624   kv.value = s - tsm->sessions;
1625   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
1626     nat_log_notice ("out2in-ed key add failed");
1627
1628   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1629       ip->src_address.as_u32 == l_key.addr.as_u32))
1630     {
1631       eh_key.protocol = e_key.protocol;
1632       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1633                                                thread_index, &eh_key,
1634                                                &address_index,
1635                                                sm->port_per_thread,
1636                                                tsm->snat_thread_index))
1637         {
1638           b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1639           nat44_delete_session (sm, s, thread_index);
1640           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1641             nat_log_notice ("out2in-ed key del failed");
1642           return 0;
1643         }
1644       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1645       s->ext_host_nat_port = eh_key.port;
1646       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1647       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1648                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
1649     }
1650   else
1651     {
1652       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1653                   l_key.fib_index, l_key.port, s->ext_host_port);
1654     }
1655   kv.value = s - tsm->sessions;
1656   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1657     nat_log_notice ("in2out-ed key add failed");
1658
1659   return s;
1660 }
1661
1662 static_always_inline int
1663 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1664 {
1665   icmp46_header_t *icmp0;
1666   nat_ed_ses_key_t key0;
1667   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1668   ip4_header_t *inner_ip0;
1669   void *l4_header = 0;
1670   icmp46_header_t *inner_icmp0;
1671
1672   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1673   echo0 = (icmp_echo_header_t *)(icmp0+1);
1674
1675   if (!icmp_is_error_message (icmp0))
1676     {
1677       key0.proto = IP_PROTOCOL_ICMP;
1678       key0.l_addr = ip0->dst_address;
1679       key0.r_addr = ip0->src_address;
1680       key0.l_port = echo0->identifier;
1681       key0.r_port = 0;
1682     }
1683   else
1684     {
1685       inner_ip0 = (ip4_header_t *)(echo0+1);
1686       l4_header = ip4_next_header (inner_ip0);
1687       key0.proto = inner_ip0->protocol;
1688       key0.l_addr = inner_ip0->src_address;
1689       key0.r_addr = inner_ip0->dst_address;
1690       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1691         {
1692         case SNAT_PROTOCOL_ICMP:
1693           inner_icmp0 = (icmp46_header_t*)l4_header;
1694           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1695           key0.l_port = inner_echo0->identifier;
1696           key0.r_port = 0;
1697           break;
1698         case SNAT_PROTOCOL_UDP:
1699         case SNAT_PROTOCOL_TCP:
1700           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1701           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1702           break;
1703         default:
1704           return -1;
1705         }
1706     }
1707   *p_key0 = key0;
1708   return 0;
1709 }
1710
1711 static int
1712 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1713               u16 dst_port, u32 thread_index)
1714 {
1715   clib_bihash_kv_16_8_t kv, value;
1716   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1717
1718   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1719               sm->inside_fib_index, src_port, dst_port);
1720   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1721     return 1;
1722
1723   return 0;
1724 }
1725
1726 static void
1727 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1728                       u32 thread_index)
1729 {
1730   nat_ed_ses_key_t key;
1731   clib_bihash_kv_16_8_t kv, value;
1732   udp_header_t *udp;
1733   snat_user_t *u;
1734   snat_session_t *s = 0;
1735   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1736   f64 now = vlib_time_now (sm->vlib_main);
1737
1738   if (ip->protocol == IP_PROTOCOL_ICMP)
1739     {
1740       if (icmp_get_ed_key (ip, &key))
1741         return;
1742     }
1743   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1744     {
1745       udp = ip4_next_header(ip);
1746       key.r_addr = ip->src_address;
1747       key.l_addr = ip->dst_address;
1748       key.proto = ip->protocol;
1749       key.l_port = udp->dst_port;
1750       key.r_port = udp->src_port;
1751     }
1752   else
1753     {
1754       key.r_addr = ip->src_address;
1755       key.l_addr = ip->dst_address;
1756       key.proto = ip->protocol;
1757       key.l_port = key.r_port = 0;
1758     }
1759   key.fib_index = 0;
1760   kv.key[0] = key.as_u64[0];
1761   kv.key[1] = key.as_u64[1];
1762
1763   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1764     {
1765       s = pool_elt_at_index (tsm->sessions, value.value);
1766     }
1767   else
1768     {
1769       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1770         return;
1771
1772       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1773                                   thread_index);
1774       if (!u)
1775         {
1776           nat_log_warn ("create NAT user failed");
1777           return;
1778         }
1779
1780       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1781       if (!s)
1782         {
1783           nat_log_warn ("create NAT session failed");
1784           return;
1785         }
1786
1787       s->ext_host_addr = key.r_addr;
1788       s->ext_host_port = key.r_port;
1789       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1790       s->outside_address_index = ~0;
1791       s->out2in.addr = key.l_addr;
1792       s->out2in.port = key.l_port;
1793       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1794       s->out2in.fib_index = 0;
1795       s->in2out = s->out2in;
1796       user_session_increment (sm, u, 0);
1797
1798       kv.value = s - tsm->sessions;
1799       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1800         nat_log_notice ("in2out_ed key add failed");
1801     }
1802
1803   if (ip->protocol == IP_PROTOCOL_TCP)
1804     {
1805       tcp_header_t *tcp = ip4_next_header(ip);
1806       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1807         return;
1808     }
1809
1810   /* Per-user LRU list maintenance */
1811   nat44_session_update_lru (sm, s, thread_index);
1812   /* Accounting */
1813   nat44_session_update_counters (s, now, 0);
1814 }
1815
1816 u32
1817 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1818                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1819                       u8 * p_proto, snat_session_key_t * p_value,
1820                       u8 * p_dont_translate, void * d, void * e)
1821 {
1822   u32 next = ~0, sw_if_index, rx_fib_index;
1823   icmp46_header_t *icmp;
1824   nat_ed_ses_key_t key;
1825   clib_bihash_kv_16_8_t kv, value;
1826   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1827   snat_session_t *s = 0;
1828   u8 dont_translate = 0, is_addr_only;
1829   snat_session_key_t e_key, l_key;
1830
1831   icmp = (icmp46_header_t *) ip4_next_header (ip);
1832   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1833   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1834
1835   if (icmp_get_ed_key (ip, &key))
1836     {
1837       b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1838       next = SNAT_OUT2IN_NEXT_DROP;
1839       goto out;
1840     }
1841   key.fib_index = rx_fib_index;
1842   kv.key[0] = key.as_u64[0];
1843   kv.key[1] = key.as_u64[1];
1844
1845   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
1846     {
1847       /* Try to match static mapping */
1848       e_key.addr = ip->dst_address;
1849       e_key.port = key.l_port;
1850       e_key.protocol = ip_proto_to_snat_proto (key.proto);
1851       e_key.fib_index = rx_fib_index;
1852       if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
1853         {
1854           if (!sm->forwarding_enabled)
1855             {
1856               /* Don't NAT packet aimed at the intfc address */
1857               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
1858                                                   ip->dst_address.as_u32)))
1859                 {
1860                   dont_translate = 1;
1861                   goto out;
1862                 }
1863               b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1864               next = NAT44_ED_OUT2IN_NEXT_DROP;
1865               goto out;
1866             }
1867           else
1868             {
1869               dont_translate = 1;
1870               if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port, thread_index))
1871                 {
1872                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1873                   goto out;
1874                 }
1875               create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
1876               goto out;
1877             }
1878         }
1879
1880       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1881                         (icmp->type != ICMP4_echo_request || !is_addr_only)))
1882         {
1883           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1884           next = NAT44_ED_OUT2IN_NEXT_DROP;
1885           goto out;
1886         }
1887
1888       /* Create session initiated by host from external network */
1889       s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
1890                                                thread_index, 0, 0);
1891
1892       if (!s)
1893         {
1894           next = NAT44_ED_OUT2IN_NEXT_DROP;
1895           goto out;
1896         }
1897     }
1898   else
1899     {
1900       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1901                         icmp->type != ICMP4_echo_request &&
1902                         !icmp_is_error_message (icmp)))
1903         {
1904           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1905           next = SNAT_OUT2IN_NEXT_DROP;
1906           goto out;
1907         }
1908
1909       s = pool_elt_at_index (tsm->sessions, value.value);
1910     }
1911
1912   *p_proto = ip_proto_to_snat_proto (key.proto);
1913 out:
1914   if (s)
1915     *p_value = s->in2out;
1916   *p_dont_translate = dont_translate;
1917   if (d)
1918     *(snat_session_t**)d = s;
1919   return next;
1920 }
1921
1922 static snat_session_t *
1923 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
1924                                vlib_buffer_t * b,
1925                                ip4_header_t * ip,
1926                                u32 rx_fib_index,
1927                                u32 thread_index,
1928                                f64 now,
1929                                vlib_main_t * vm,
1930                                vlib_node_runtime_t * node)
1931 {
1932   clib_bihash_kv_8_8_t kv, value;
1933   clib_bihash_kv_16_8_t s_kv, s_value;
1934   snat_static_mapping_t *m;
1935   u32 old_addr, new_addr;
1936   ip_csum_t sum;
1937   snat_session_t * s;
1938   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1939   snat_user_t *u;
1940
1941   old_addr = ip->dst_address.as_u32;
1942
1943   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
1944               rx_fib_index, 0, 0);
1945
1946   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
1947     {
1948       s = pool_elt_at_index (tsm->sessions, s_value.value);
1949       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1950     }
1951   else
1952     {
1953       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1954         {
1955           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1956           nat_log_notice ("maximum sessions exceeded");
1957           return 0;
1958         }
1959
1960       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1961       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1962         {
1963           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1964           return 0;
1965         }
1966
1967       m = pool_elt_at_index (sm->static_mappings, value.value);
1968
1969       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1970
1971       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
1972                                   thread_index);
1973       if (!u)
1974         {
1975           nat_log_warn ("create NAT user failed");
1976           return 0;
1977         }
1978
1979       /* Create a new session */
1980       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1981       if (!s)
1982         {
1983           nat_log_warn ("create NAT session failed");
1984           return 0;
1985         }
1986
1987       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1988       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1989       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1990       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1991       s->outside_address_index = ~0;
1992       s->out2in.addr.as_u32 = old_addr;
1993       s->out2in.fib_index = rx_fib_index;
1994       s->in2out.addr.as_u32 = new_addr;
1995       s->in2out.fib_index = m->fib_index;
1996       s->in2out.port = s->out2in.port = ip->protocol;
1997       user_session_increment (sm, u, 1);
1998
1999       /* Add to lookup tables */
2000       s_kv.value = s - tsm->sessions;
2001       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2002         nat_log_notice ("out2in key add failed");
2003
2004       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2005                   m->fib_index, 0, 0);
2006       s_kv.value = s - tsm->sessions;
2007       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2008         nat_log_notice ("in2out key add failed");
2009    }
2010
2011   /* Update IP checksum */
2012   sum = ip->checksum;
2013   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2014   ip->checksum = ip_csum_fold (sum);
2015
2016   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2017
2018   /* Accounting */
2019   nat44_session_update_counters (s, now,
2020                                  vlib_buffer_length_in_chain (vm, b));
2021   /* Per-user LRU list maintenance */
2022   nat44_session_update_lru (sm, s, thread_index);
2023
2024   return s;
2025 }
2026
2027 static inline uword
2028 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2029                                 vlib_node_runtime_t * node,
2030                                 vlib_frame_t * frame, int is_slow_path)
2031 {
2032   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2033   nat44_ed_out2in_next_t next_index;
2034   snat_main_t *sm = &snat_main;
2035   f64 now = vlib_time_now (vm);
2036   u32 thread_index = vm->thread_index;
2037   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2038
2039   stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2040     nat44_ed_out2in_node.index;
2041
2042   from = vlib_frame_vector_args (frame);
2043   n_left_from = frame->n_vectors;
2044   next_index = node->cached_next_index;
2045
2046   while (n_left_from > 0)
2047     {
2048       u32 n_left_to_next;
2049
2050       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2051
2052       while (n_left_from >= 4 && n_left_to_next >= 2)
2053         {
2054           u32 bi0, bi1;
2055           vlib_buffer_t *b0, *b1;
2056           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2057           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2058           u16 old_port0, new_port0, old_port1, new_port1;
2059           ip4_header_t *ip0, *ip1;
2060           udp_header_t *udp0, *udp1;
2061           tcp_header_t *tcp0, *tcp1;
2062           icmp46_header_t *icmp0, *icmp1;
2063           snat_session_t *s0 = 0, *s1 = 0;
2064           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2065           ip_csum_t sum0, sum1;
2066           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2067           u8 is_lb0, is_lb1;
2068           twice_nat_type_t twice_nat0, twice_nat1;
2069
2070           /* Prefetch next iteration. */
2071           {
2072             vlib_buffer_t * p2, * p3;
2073
2074             p2 = vlib_get_buffer (vm, from[2]);
2075             p3 = vlib_get_buffer (vm, from[3]);
2076
2077             vlib_prefetch_buffer_header (p2, LOAD);
2078             vlib_prefetch_buffer_header (p3, LOAD);
2079
2080             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2081             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2082           }
2083
2084           /* speculatively enqueue b0 and b1 to the current next frame */
2085           to_next[0] = bi0 = from[0];
2086           to_next[1] = bi1 = from[1];
2087           from += 2;
2088           to_next += 2;
2089           n_left_from -= 2;
2090           n_left_to_next -= 2;
2091
2092           b0 = vlib_get_buffer (vm, bi0);
2093           b1 = vlib_get_buffer (vm, bi1);
2094
2095           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2096           vnet_buffer (b0)->snat.flags = 0;
2097           ip0 = vlib_buffer_get_current (b0);
2098
2099           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2100           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2101                                                                sw_if_index0);
2102
2103           if (PREDICT_FALSE(ip0->ttl == 1))
2104             {
2105               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2106               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2107                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2108                                            0);
2109               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2110               goto trace00;
2111             }
2112
2113           udp0 = ip4_next_header (ip0);
2114           tcp0 = (tcp_header_t *) udp0;
2115           icmp0 = (icmp46_header_t *) udp0;
2116           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2117
2118           if (is_slow_path)
2119             {
2120               if (PREDICT_FALSE (proto0 == ~0))
2121                 {
2122                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2123                                                      thread_index, now, vm, node);
2124                   if (!sm->forwarding_enabled)
2125                     {
2126                       if (!s0)
2127                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2128                       goto trace00;
2129                     }
2130                 }
2131
2132               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2133                 {
2134                   next0 = icmp_out2in_slow_path
2135                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2136                      next0, now, thread_index, &s0);
2137                   goto trace00;
2138                 }
2139             }
2140           else
2141             {
2142               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2143                 {
2144                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2145                   goto trace00;
2146                 }
2147
2148               if (ip4_is_fragment (ip0))
2149                 {
2150                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2151                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2152                   goto trace00;
2153                 }
2154             }
2155
2156           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2157                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2158
2159           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2160             {
2161               if (is_slow_path)
2162                 {
2163                   /* Try to match static mapping by external address and port,
2164                      destination address and port in packet */
2165                   e_key0.addr = ip0->dst_address;
2166                   e_key0.port = udp0->dst_port;
2167                   e_key0.protocol = proto0;
2168                   e_key0.fib_index = rx_fib_index0;
2169                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2170                       &twice_nat0, &is_lb0))
2171                     {
2172                       /*
2173                        * Send DHCP packets to the ipv4 stack, or we won't
2174                        * be able to use dhcp client on the outside interface
2175                        */
2176                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2177                           && (udp0->dst_port ==
2178                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2179                         {
2180                           vnet_feature_next (&next0, b0);
2181                           goto trace00;
2182                         }
2183
2184                       if (!sm->forwarding_enabled)
2185                         {
2186                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2187                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2188                         }
2189                       else
2190                         {
2191                           if (next_src_nat(sm, ip0, ip0->protocol,
2192                                            udp0->src_port, udp0->dst_port,
2193                                            thread_index))
2194                             {
2195                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2196                               goto trace00;
2197                             }
2198                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2199                                                 thread_index);
2200                         }
2201                       goto trace00;
2202                     }
2203
2204                   /* Create session initiated by host from external network */
2205                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2206                                                             e_key0, node,
2207                                                             thread_index,
2208                                                             twice_nat0, is_lb0);
2209
2210                   if (!s0)
2211                     {
2212                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2213                       goto trace00;
2214                     }
2215                 }
2216               else
2217                 {
2218                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2219                   goto trace00;
2220                 }
2221             }
2222           else
2223             {
2224               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2225             }
2226
2227           old_addr0 = ip0->dst_address.as_u32;
2228           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2229           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2230
2231           sum0 = ip0->checksum;
2232           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2233                                  dst_address);
2234           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2235             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2236                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2237                                    src_address);
2238           ip0->checksum = ip_csum_fold (sum0);
2239
2240           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2241             {
2242               old_port0 = tcp0->dst_port;
2243               new_port0 = tcp0->dst_port = s0->in2out.port;
2244
2245               sum0 = tcp0->checksum;
2246               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2247                                      dst_address);
2248               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2249                                      length);
2250               if (is_twice_nat_session (s0))
2251                 {
2252                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2253                                          s0->ext_host_nat_addr.as_u32,
2254                                          ip4_header_t, dst_address);
2255                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2256                                          s0->ext_host_nat_port, ip4_header_t,
2257                                          length);
2258                   tcp0->src_port = s0->ext_host_nat_port;
2259                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2260                 }
2261               tcp0->checksum = ip_csum_fold(sum0);
2262               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2263                 goto trace00;
2264             }
2265           else
2266             {
2267               udp0->dst_port = s0->in2out.port;
2268               if (is_twice_nat_session (s0))
2269                 {
2270                   udp0->src_port = s0->ext_host_nat_port;
2271                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2272                 }
2273               udp0->checksum = 0;
2274             }
2275
2276           /* Accounting */
2277           nat44_session_update_counters (s0, now,
2278                                          vlib_buffer_length_in_chain (vm, b0));
2279           /* Per-user LRU list maintenance */
2280           nat44_session_update_lru (sm, s0, thread_index);
2281
2282         trace00:
2283           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2284                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2285             {
2286               nat44_ed_out2in_trace_t *t =
2287                 vlib_add_trace (vm, node, b0, sizeof (*t));
2288               t->is_slow_path = is_slow_path;
2289               t->sw_if_index = sw_if_index0;
2290               t->next_index = next0;
2291               t->session_index = ~0;
2292               if (s0)
2293                 t->session_index = s0 - tsm->sessions;
2294             }
2295
2296           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2297
2298           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2299           vnet_buffer (b1)->snat.flags = 0;
2300           ip1 = vlib_buffer_get_current (b1);
2301
2302           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2303           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2304                                                                sw_if_index1);
2305
2306           if (PREDICT_FALSE(ip1->ttl == 1))
2307             {
2308               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2309               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2310                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2311                                            0);
2312               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2313               goto trace01;
2314             }
2315
2316           udp1 = ip4_next_header (ip1);
2317           tcp1 = (tcp_header_t *) udp1;
2318           icmp1 = (icmp46_header_t *) udp1;
2319           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2320
2321           if (is_slow_path)
2322             {
2323               if (PREDICT_FALSE (proto1 == ~0))
2324                 {
2325                   s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2326                                                      thread_index, now, vm, node);
2327                   if (!sm->forwarding_enabled)
2328                     {
2329                       if (!s1)
2330                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2331                       goto trace01;
2332                     }
2333                 }
2334
2335               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2336                 {
2337                   next1 = icmp_out2in_slow_path
2338                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2339                      next1, now, thread_index, &s1);
2340                   goto trace01;
2341                 }
2342             }
2343           else
2344             {
2345               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2346                 {
2347                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2348                   goto trace01;
2349                 }
2350
2351               if (ip4_is_fragment (ip1))
2352                 {
2353                   b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2354                   next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2355                   goto trace01;
2356                 }
2357             }
2358
2359           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2360                       rx_fib_index1, udp1->dst_port, udp1->src_port);
2361
2362           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2363             {
2364               if (is_slow_path)
2365                 {
2366                   /* Try to match static mapping by external address and port,
2367                      destination address and port in packet */
2368                   e_key1.addr = ip1->dst_address;
2369                   e_key1.port = udp1->dst_port;
2370                   e_key1.protocol = proto1;
2371                   e_key1.fib_index = rx_fib_index1;
2372                   if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2373                       &twice_nat1, &is_lb1))
2374                     {
2375                       /*
2376                        * Send DHCP packets to the ipv4 stack, or we won't
2377                        * be able to use dhcp client on the outside interface
2378                        */
2379                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2380                           && (udp1->dst_port ==
2381                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2382                         {
2383                           vnet_feature_next (&next1, b1);
2384                           goto trace01;
2385                         }
2386
2387                       if (!sm->forwarding_enabled)
2388                         {
2389                           b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2390                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2391                         }
2392                       else
2393                         {
2394                           if (next_src_nat(sm, ip1, ip1->protocol,
2395                                            udp1->src_port, udp1->dst_port,
2396                                            thread_index))
2397                             {
2398                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2399                               goto trace01;
2400                             }
2401                           create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2402                                                 thread_index);
2403                         }
2404                       goto trace01;
2405                     }
2406
2407                   /* Create session initiated by host from external network */
2408                   s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2409                                                             e_key1, node,
2410                                                             thread_index,
2411                                                             twice_nat1, is_lb1);
2412
2413                   if (!s1)
2414                     {
2415                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2416                       goto trace01;
2417                     }
2418                 }
2419               else
2420                 {
2421                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2422                   goto trace01;
2423                 }
2424             }
2425           else
2426             {
2427               s1 = pool_elt_at_index (tsm->sessions, value1.value);
2428             }
2429
2430           old_addr1 = ip1->dst_address.as_u32;
2431           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2432           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2433
2434           sum1 = ip1->checksum;
2435           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2436                                  dst_address);
2437           if (PREDICT_FALSE (is_twice_nat_session (s1)))
2438             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2439                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
2440                                    src_address);
2441           ip1->checksum = ip_csum_fold (sum1);
2442
2443           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2444             {
2445               old_port1 = tcp1->dst_port;
2446               new_port1 = tcp1->dst_port = s1->in2out.port;
2447
2448               sum1 = tcp1->checksum;
2449               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2450                                      dst_address);
2451               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2452                                      length);
2453               if (is_twice_nat_session (s1))
2454                 {
2455                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2456                                          s1->ext_host_nat_addr.as_u32,
2457                                          ip4_header_t, dst_address);
2458                   sum1 = ip_csum_update (sum1, tcp1->src_port,
2459                                          s1->ext_host_nat_port, ip4_header_t,
2460                                          length);
2461                   tcp1->src_port = s1->ext_host_nat_port;
2462                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2463                 }
2464               tcp1->checksum = ip_csum_fold(sum1);
2465               if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2466                 goto trace01;
2467             }
2468           else
2469             {
2470               udp1->dst_port = s1->in2out.port;
2471               if (is_twice_nat_session (s1))
2472                 {
2473                   udp1->src_port = s1->ext_host_nat_port;
2474                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2475                 }
2476               udp1->checksum = 0;
2477             }
2478
2479           /* Accounting */
2480           nat44_session_update_counters (s1, now,
2481                                          vlib_buffer_length_in_chain (vm, b1));
2482           /* Per-user LRU list maintenance */
2483           nat44_session_update_lru (sm, s1, thread_index);
2484
2485         trace01:
2486           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2487                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2488             {
2489               nat44_ed_out2in_trace_t *t =
2490                 vlib_add_trace (vm, node, b1, sizeof (*t));
2491               t->is_slow_path = is_slow_path;
2492               t->sw_if_index = sw_if_index1;
2493               t->next_index = next1;
2494               t->session_index = ~0;
2495               if (s1)
2496                 t->session_index = s1 - tsm->sessions;
2497             }
2498
2499           pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2500
2501           /* verify speculative enqueues, maybe switch current next frame */
2502           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2503                                            to_next, n_left_to_next,
2504                                            bi0, bi1, next0, next1);
2505         }
2506
2507       while (n_left_from > 0 && n_left_to_next > 0)
2508         {
2509           u32 bi0;
2510           vlib_buffer_t *b0;
2511           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2512           u16 old_port0, new_port0;
2513           ip4_header_t *ip0;
2514           udp_header_t *udp0;
2515           tcp_header_t *tcp0;
2516           icmp46_header_t * icmp0;
2517           snat_session_t *s0 = 0;
2518           clib_bihash_kv_16_8_t kv0, value0;
2519           ip_csum_t sum0;
2520           snat_session_key_t e_key0, l_key0;
2521           u8 is_lb0;
2522           twice_nat_type_t twice_nat0;
2523
2524           /* speculatively enqueue b0 to the current next frame */
2525           bi0 = from[0];
2526           to_next[0] = bi0;
2527           from += 1;
2528           to_next += 1;
2529           n_left_from -= 1;
2530           n_left_to_next -= 1;
2531
2532           b0 = vlib_get_buffer (vm, bi0);
2533           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2534           vnet_buffer (b0)->snat.flags = 0;
2535           ip0 = vlib_buffer_get_current (b0);
2536
2537           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2538           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2539                                                                sw_if_index0);
2540
2541           if (PREDICT_FALSE(ip0->ttl == 1))
2542             {
2543               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2544               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2545                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2546                                            0);
2547               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2548               goto trace0;
2549             }
2550
2551           udp0 = ip4_next_header (ip0);
2552           tcp0 = (tcp_header_t *) udp0;
2553           icmp0 = (icmp46_header_t *) udp0;
2554           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2555
2556           if (is_slow_path)
2557             {
2558               if (PREDICT_FALSE (proto0 == ~0))
2559                 {
2560                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2561                                                      thread_index, now, vm, node);
2562                   if (!sm->forwarding_enabled)
2563                     {
2564                       if (!s0)
2565                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2566                       goto trace0;
2567                     }
2568                 }
2569
2570               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2571                 {
2572                   next0 = icmp_out2in_slow_path
2573                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2574                      next0, now, thread_index, &s0);
2575                   goto trace0;
2576                 }
2577             }
2578           else
2579             {
2580               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2581                 {
2582                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2583                   goto trace0;
2584                 }
2585
2586               if (ip4_is_fragment (ip0))
2587                 {
2588                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2589                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2590                   goto trace0;
2591                 }
2592             }
2593
2594           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2595                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2596
2597           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2598             {
2599               if (is_slow_path)
2600                 {
2601                   /* Try to match static mapping by external address and port,
2602                      destination address and port in packet */
2603                   e_key0.addr = ip0->dst_address;
2604                   e_key0.port = udp0->dst_port;
2605                   e_key0.protocol = proto0;
2606                   e_key0.fib_index = rx_fib_index0;
2607                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2608                       &twice_nat0, &is_lb0))
2609                     {
2610                       /*
2611                        * Send DHCP packets to the ipv4 stack, or we won't
2612                        * be able to use dhcp client on the outside interface
2613                        */
2614                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2615                           && (udp0->dst_port ==
2616                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2617                         {
2618                           vnet_feature_next (&next0, b0);
2619                           goto trace0;
2620                         }
2621
2622                       if (!sm->forwarding_enabled)
2623                         {
2624                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2625                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2626                         }
2627                       else
2628                         {
2629                           if (next_src_nat(sm, ip0, ip0->protocol,
2630                                            udp0->src_port, udp0->dst_port,
2631                                            thread_index))
2632                             {
2633                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2634                               goto trace0;
2635                             }
2636                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2637                                                 thread_index);
2638                         }
2639                       goto trace0;
2640                     }
2641
2642                   /* Create session initiated by host from external network */
2643                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2644                                                             e_key0, node,
2645                                                             thread_index,
2646                                                             twice_nat0, is_lb0);
2647
2648                   if (!s0)
2649                     {
2650                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2651                       goto trace0;
2652                     }
2653                 }
2654               else
2655                 {
2656                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2657                   goto trace0;
2658                 }
2659             }
2660           else
2661             {
2662               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2663             }
2664
2665           old_addr0 = ip0->dst_address.as_u32;
2666           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2667           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2668
2669           sum0 = ip0->checksum;
2670           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2671                                  dst_address);
2672           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2673             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2674                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2675                                    src_address);
2676           ip0->checksum = ip_csum_fold (sum0);
2677
2678           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2679             {
2680               old_port0 = tcp0->dst_port;
2681               new_port0 = tcp0->dst_port = s0->in2out.port;
2682
2683               sum0 = tcp0->checksum;
2684               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2685                                      dst_address);
2686               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2687                                      length);
2688               if (is_twice_nat_session (s0))
2689                 {
2690                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2691                                          s0->ext_host_nat_addr.as_u32,
2692                                          ip4_header_t, dst_address);
2693                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2694                                          s0->ext_host_nat_port, ip4_header_t,
2695                                          length);
2696                   tcp0->src_port = s0->ext_host_nat_port;
2697                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2698                 }
2699               tcp0->checksum = ip_csum_fold(sum0);
2700               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2701                 goto trace0;
2702             }
2703           else
2704             {
2705               udp0->dst_port = s0->in2out.port;
2706               if (is_twice_nat_session (s0))
2707                 {
2708                   udp0->src_port = s0->ext_host_nat_port;
2709                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2710                 }
2711               udp0->checksum = 0;
2712             }
2713
2714           /* Accounting */
2715           nat44_session_update_counters (s0, now,
2716                                          vlib_buffer_length_in_chain (vm, b0));
2717           /* Per-user LRU list maintenance */
2718           nat44_session_update_lru (sm, s0, thread_index);
2719
2720         trace0:
2721           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2722                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2723             {
2724               nat44_ed_out2in_trace_t *t =
2725                 vlib_add_trace (vm, node, b0, sizeof (*t));
2726               t->is_slow_path = is_slow_path;
2727               t->sw_if_index = sw_if_index0;
2728               t->next_index = next0;
2729               t->session_index = ~0;
2730               if (s0)
2731                 t->session_index = s0 - tsm->sessions;
2732             }
2733
2734           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2735
2736           /* verify speculative enqueue, maybe switch current next frame */
2737           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2738                                            to_next, n_left_to_next,
2739                                            bi0, next0);
2740         }
2741
2742       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2743     }
2744
2745   vlib_node_increment_counter (vm, stats_node_index,
2746                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2747                                pkts_processed);
2748   return frame->n_vectors;
2749 }
2750
2751 static uword
2752 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2753                               vlib_node_runtime_t * node,
2754                               vlib_frame_t * frame)
2755 {
2756   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2757 }
2758
2759 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2760   .function = nat44_ed_out2in_fast_path_fn,
2761   .name = "nat44-ed-out2in",
2762   .vector_size = sizeof (u32),
2763   .format_trace = format_nat44_ed_out2in_trace,
2764   .type = VLIB_NODE_TYPE_INTERNAL,
2765
2766   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2767   .error_strings = snat_out2in_error_strings,
2768
2769   .runtime_data_bytes = sizeof (snat_runtime_t),
2770
2771   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2772
2773   /* edit / add dispositions here */
2774   .next_nodes = {
2775     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2776     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2777     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2778     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2779     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2780   },
2781 };
2782
2783 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2784
2785 static uword
2786 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2787                               vlib_node_runtime_t * node,
2788                               vlib_frame_t * frame)
2789 {
2790   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2791 }
2792
2793 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2794   .function = nat44_ed_out2in_slow_path_fn,
2795   .name = "nat44-ed-out2in-slowpath",
2796   .vector_size = sizeof (u32),
2797   .format_trace = format_nat44_ed_out2in_trace,
2798   .type = VLIB_NODE_TYPE_INTERNAL,
2799
2800   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2801   .error_strings = snat_out2in_error_strings,
2802
2803   .runtime_data_bytes = sizeof (snat_runtime_t),
2804
2805   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2806
2807   /* edit / add dispositions here */
2808   .next_nodes = {
2809     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2810     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2811     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2812     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2813     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2814   },
2815 };
2816
2817 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2818                               nat44_ed_out2in_slow_path_fn);
2819
2820 /**************************/
2821 /*** deterministic mode ***/
2822 /**************************/
2823 static uword
2824 snat_det_out2in_node_fn (vlib_main_t * vm,
2825                          vlib_node_runtime_t * node,
2826                          vlib_frame_t * frame)
2827 {
2828   u32 n_left_from, * from, * to_next;
2829   snat_out2in_next_t next_index;
2830   u32 pkts_processed = 0;
2831   snat_main_t * sm = &snat_main;
2832   u32 thread_index = vm->thread_index;
2833
2834   from = vlib_frame_vector_args (frame);
2835   n_left_from = frame->n_vectors;
2836   next_index = node->cached_next_index;
2837
2838   while (n_left_from > 0)
2839     {
2840       u32 n_left_to_next;
2841
2842       vlib_get_next_frame (vm, node, next_index,
2843                            to_next, n_left_to_next);
2844
2845       while (n_left_from >= 4 && n_left_to_next >= 2)
2846         {
2847           u32 bi0, bi1;
2848           vlib_buffer_t * b0, * b1;
2849           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2850           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2851           u32 sw_if_index0, sw_if_index1;
2852           ip4_header_t * ip0, * ip1;
2853           ip_csum_t sum0, sum1;
2854           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2855           u16 new_port0, old_port0, old_port1, new_port1;
2856           udp_header_t * udp0, * udp1;
2857           tcp_header_t * tcp0, * tcp1;
2858           u32 proto0, proto1;
2859           snat_det_out_key_t key0, key1;
2860           snat_det_map_t * dm0, * dm1;
2861           snat_det_session_t * ses0 = 0, * ses1 = 0;
2862           u32 rx_fib_index0, rx_fib_index1;
2863           icmp46_header_t * icmp0, * icmp1;
2864
2865           /* Prefetch next iteration. */
2866           {
2867             vlib_buffer_t * p2, * p3;
2868
2869             p2 = vlib_get_buffer (vm, from[2]);
2870             p3 = vlib_get_buffer (vm, from[3]);
2871
2872             vlib_prefetch_buffer_header (p2, LOAD);
2873             vlib_prefetch_buffer_header (p3, LOAD);
2874
2875             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2876             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2877           }
2878
2879           /* speculatively enqueue b0 and b1 to the current next frame */
2880           to_next[0] = bi0 = from[0];
2881           to_next[1] = bi1 = from[1];
2882           from += 2;
2883           to_next += 2;
2884           n_left_from -= 2;
2885           n_left_to_next -= 2;
2886
2887           b0 = vlib_get_buffer (vm, bi0);
2888           b1 = vlib_get_buffer (vm, bi1);
2889
2890           ip0 = vlib_buffer_get_current (b0);
2891           udp0 = ip4_next_header (ip0);
2892           tcp0 = (tcp_header_t *) udp0;
2893
2894           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2895
2896           if (PREDICT_FALSE(ip0->ttl == 1))
2897             {
2898               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2899               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2900                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2901                                            0);
2902               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2903               goto trace0;
2904             }
2905
2906           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2907
2908           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2909             {
2910               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2911               icmp0 = (icmp46_header_t *) udp0;
2912
2913               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2914                                   rx_fib_index0, node, next0, thread_index,
2915                                   &ses0, &dm0);
2916               goto trace0;
2917             }
2918
2919           key0.ext_host_addr = ip0->src_address;
2920           key0.ext_host_port = tcp0->src;
2921           key0.out_port = tcp0->dst;
2922
2923           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2924           if (PREDICT_FALSE(!dm0))
2925             {
2926               nat_log_info ("unknown dst address:  %U",
2927                             format_ip4_address, &ip0->dst_address);
2928               next0 = SNAT_OUT2IN_NEXT_DROP;
2929               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2930               goto trace0;
2931             }
2932
2933           snat_det_reverse(dm0, &ip0->dst_address,
2934                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2935
2936           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2937           if (PREDICT_FALSE(!ses0))
2938             {
2939               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2940                             format_ip4_address, &ip0->src_address,
2941                             clib_net_to_host_u16 (tcp0->src),
2942                             format_ip4_address, &ip0->dst_address,
2943                             clib_net_to_host_u16 (tcp0->dst),
2944                             format_ip4_address, &new_addr0);
2945               next0 = SNAT_OUT2IN_NEXT_DROP;
2946               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2947               goto trace0;
2948             }
2949           new_port0 = ses0->in_port;
2950
2951           old_addr0 = ip0->dst_address;
2952           ip0->dst_address = new_addr0;
2953           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2954
2955           sum0 = ip0->checksum;
2956           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2957                                  ip4_header_t,
2958                                  dst_address /* changed member */);
2959           ip0->checksum = ip_csum_fold (sum0);
2960
2961           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2962             {
2963               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2964                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2965               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2966                 snat_det_ses_close(dm0, ses0);
2967
2968               old_port0 = tcp0->dst;
2969               tcp0->dst = new_port0;
2970
2971               sum0 = tcp0->checksum;
2972               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2973                                      ip4_header_t,
2974                                      dst_address /* changed member */);
2975
2976               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2977                                      ip4_header_t /* cheat */,
2978                                      length /* changed member */);
2979               tcp0->checksum = ip_csum_fold(sum0);
2980             }
2981           else
2982             {
2983               old_port0 = udp0->dst_port;
2984               udp0->dst_port = new_port0;
2985               udp0->checksum = 0;
2986             }
2987
2988         trace0:
2989
2990           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2991                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2992             {
2993               snat_out2in_trace_t *t =
2994                  vlib_add_trace (vm, node, b0, sizeof (*t));
2995               t->sw_if_index = sw_if_index0;
2996               t->next_index = next0;
2997               t->session_index = ~0;
2998               if (ses0)
2999                 t->session_index = ses0 - dm0->sessions;
3000             }
3001
3002           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3003
3004           b1 = vlib_get_buffer (vm, bi1);
3005
3006           ip1 = vlib_buffer_get_current (b1);
3007           udp1 = ip4_next_header (ip1);
3008           tcp1 = (tcp_header_t *) udp1;
3009
3010           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3011
3012           if (PREDICT_FALSE(ip1->ttl == 1))
3013             {
3014               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3015               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3016                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3017                                            0);
3018               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3019               goto trace1;
3020             }
3021
3022           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3023
3024           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3025             {
3026               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3027               icmp1 = (icmp46_header_t *) udp1;
3028
3029               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3030                                   rx_fib_index1, node, next1, thread_index,
3031                                   &ses1, &dm1);
3032               goto trace1;
3033             }
3034
3035           key1.ext_host_addr = ip1->src_address;
3036           key1.ext_host_port = tcp1->src;
3037           key1.out_port = tcp1->dst;
3038
3039           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3040           if (PREDICT_FALSE(!dm1))
3041             {
3042               nat_log_info ("unknown dst address:  %U",
3043                             format_ip4_address, &ip1->dst_address);
3044               next1 = SNAT_OUT2IN_NEXT_DROP;
3045               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3046               goto trace1;
3047             }
3048
3049           snat_det_reverse(dm1, &ip1->dst_address,
3050                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
3051
3052           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3053           if (PREDICT_FALSE(!ses1))
3054             {
3055               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3056                             format_ip4_address, &ip1->src_address,
3057                             clib_net_to_host_u16 (tcp1->src),
3058                             format_ip4_address, &ip1->dst_address,
3059                             clib_net_to_host_u16 (tcp1->dst),
3060                             format_ip4_address, &new_addr1);
3061               next1 = SNAT_OUT2IN_NEXT_DROP;
3062               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3063               goto trace1;
3064             }
3065           new_port1 = ses1->in_port;
3066
3067           old_addr1 = ip1->dst_address;
3068           ip1->dst_address = new_addr1;
3069           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3070
3071           sum1 = ip1->checksum;
3072           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3073                                  ip4_header_t,
3074                                  dst_address /* changed member */);
3075           ip1->checksum = ip_csum_fold (sum1);
3076
3077           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3078             {
3079               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3080                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3081               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3082                 snat_det_ses_close(dm1, ses1);
3083
3084               old_port1 = tcp1->dst;
3085               tcp1->dst = new_port1;
3086
3087               sum1 = tcp1->checksum;
3088               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3089                                      ip4_header_t,
3090                                      dst_address /* changed member */);
3091
3092               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3093                                      ip4_header_t /* cheat */,
3094                                      length /* changed member */);
3095               tcp1->checksum = ip_csum_fold(sum1);
3096             }
3097           else
3098             {
3099               old_port1 = udp1->dst_port;
3100               udp1->dst_port = new_port1;
3101               udp1->checksum = 0;
3102             }
3103
3104         trace1:
3105
3106           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3107                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3108             {
3109               snat_out2in_trace_t *t =
3110                  vlib_add_trace (vm, node, b1, sizeof (*t));
3111               t->sw_if_index = sw_if_index1;
3112               t->next_index = next1;
3113               t->session_index = ~0;
3114               if (ses1)
3115                 t->session_index = ses1 - dm1->sessions;
3116             }
3117
3118           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3119
3120           /* verify speculative enqueues, maybe switch current next frame */
3121           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3122                                            to_next, n_left_to_next,
3123                                            bi0, bi1, next0, next1);
3124          }
3125
3126       while (n_left_from > 0 && n_left_to_next > 0)
3127         {
3128           u32 bi0;
3129           vlib_buffer_t * b0;
3130           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3131           u32 sw_if_index0;
3132           ip4_header_t * ip0;
3133           ip_csum_t sum0;
3134           ip4_address_t new_addr0, old_addr0;
3135           u16 new_port0, old_port0;
3136           udp_header_t * udp0;
3137           tcp_header_t * tcp0;
3138           u32 proto0;
3139           snat_det_out_key_t key0;
3140           snat_det_map_t * dm0;
3141           snat_det_session_t * ses0 = 0;
3142           u32 rx_fib_index0;
3143           icmp46_header_t * icmp0;
3144
3145           /* speculatively enqueue b0 to the current next frame */
3146           bi0 = from[0];
3147           to_next[0] = bi0;
3148           from += 1;
3149           to_next += 1;
3150           n_left_from -= 1;
3151           n_left_to_next -= 1;
3152
3153           b0 = vlib_get_buffer (vm, bi0);
3154
3155           ip0 = vlib_buffer_get_current (b0);
3156           udp0 = ip4_next_header (ip0);
3157           tcp0 = (tcp_header_t *) udp0;
3158
3159           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3160
3161           if (PREDICT_FALSE(ip0->ttl == 1))
3162             {
3163               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3164               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3165                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3166                                            0);
3167               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3168               goto trace00;
3169             }
3170
3171           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3172
3173           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3174             {
3175               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3176               icmp0 = (icmp46_header_t *) udp0;
3177
3178               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3179                                   rx_fib_index0, node, next0, thread_index,
3180                                   &ses0, &dm0);
3181               goto trace00;
3182             }
3183
3184           key0.ext_host_addr = ip0->src_address;
3185           key0.ext_host_port = tcp0->src;
3186           key0.out_port = tcp0->dst;
3187
3188           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3189           if (PREDICT_FALSE(!dm0))
3190             {
3191               nat_log_info ("unknown dst address:  %U",
3192                             format_ip4_address, &ip0->dst_address);
3193               next0 = SNAT_OUT2IN_NEXT_DROP;
3194               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3195               goto trace00;
3196             }
3197
3198           snat_det_reverse(dm0, &ip0->dst_address,
3199                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
3200
3201           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3202           if (PREDICT_FALSE(!ses0))
3203             {
3204               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3205                             format_ip4_address, &ip0->src_address,
3206                             clib_net_to_host_u16 (tcp0->src),
3207                             format_ip4_address, &ip0->dst_address,
3208                             clib_net_to_host_u16 (tcp0->dst),
3209                             format_ip4_address, &new_addr0);
3210               next0 = SNAT_OUT2IN_NEXT_DROP;
3211               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3212               goto trace00;
3213             }
3214           new_port0 = ses0->in_port;
3215
3216           old_addr0 = ip0->dst_address;
3217           ip0->dst_address = new_addr0;
3218           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3219
3220           sum0 = ip0->checksum;
3221           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3222                                  ip4_header_t,
3223                                  dst_address /* changed member */);
3224           ip0->checksum = ip_csum_fold (sum0);
3225
3226           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3227             {
3228               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3229                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3230               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3231                 snat_det_ses_close(dm0, ses0);
3232
3233               old_port0 = tcp0->dst;
3234               tcp0->dst = new_port0;
3235
3236               sum0 = tcp0->checksum;
3237               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3238                                      ip4_header_t,
3239                                      dst_address /* changed member */);
3240
3241               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3242                                      ip4_header_t /* cheat */,
3243                                      length /* changed member */);
3244               tcp0->checksum = ip_csum_fold(sum0);
3245             }
3246           else
3247             {
3248               old_port0 = udp0->dst_port;
3249               udp0->dst_port = new_port0;
3250               udp0->checksum = 0;
3251             }
3252
3253         trace00:
3254
3255           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3256                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3257             {
3258               snat_out2in_trace_t *t =
3259                  vlib_add_trace (vm, node, b0, sizeof (*t));
3260               t->sw_if_index = sw_if_index0;
3261               t->next_index = next0;
3262               t->session_index = ~0;
3263               if (ses0)
3264                 t->session_index = ses0 - dm0->sessions;
3265             }
3266
3267           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3268
3269           /* verify speculative enqueue, maybe switch current next frame */
3270           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3271                                            to_next, n_left_to_next,
3272                                            bi0, next0);
3273         }
3274
3275       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3276     }
3277
3278   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3279                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3280                                pkts_processed);
3281   return frame->n_vectors;
3282 }
3283
3284 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3285   .function = snat_det_out2in_node_fn,
3286   .name = "nat44-det-out2in",
3287   .vector_size = sizeof (u32),
3288   .format_trace = format_snat_out2in_trace,
3289   .type = VLIB_NODE_TYPE_INTERNAL,
3290
3291   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3292   .error_strings = snat_out2in_error_strings,
3293
3294   .runtime_data_bytes = sizeof (snat_runtime_t),
3295
3296   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3297
3298   /* edit / add dispositions here */
3299   .next_nodes = {
3300     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3301     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3302     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3303     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3304   },
3305 };
3306 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3307
3308 /**
3309  * Get address and port values to be used for ICMP packet translation
3310  * and create session if needed
3311  *
3312  * @param[in,out] sm             NAT main
3313  * @param[in,out] node           NAT node runtime
3314  * @param[in] thread_index       thread index
3315  * @param[in,out] b0             buffer containing packet to be translated
3316  * @param[out] p_proto           protocol used for matching
3317  * @param[out] p_value           address and port after NAT translation
3318  * @param[out] p_dont_translate  if packet should not be translated
3319  * @param d                      optional parameter
3320  * @param e                      optional parameter
3321  */
3322 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3323                           u32 thread_index, vlib_buffer_t *b0,
3324                           ip4_header_t *ip0, u8 *p_proto,
3325                           snat_session_key_t *p_value,
3326                           u8 *p_dont_translate, void *d, void *e)
3327 {
3328   icmp46_header_t *icmp0;
3329   u32 sw_if_index0;
3330   u8 protocol;
3331   snat_det_out_key_t key0;
3332   u8 dont_translate = 0;
3333   u32 next0 = ~0;
3334   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3335   ip4_header_t *inner_ip0;
3336   void *l4_header = 0;
3337   icmp46_header_t *inner_icmp0;
3338   snat_det_map_t * dm0 = 0;
3339   ip4_address_t new_addr0 = {{0}};
3340   snat_det_session_t * ses0 = 0;
3341   ip4_address_t out_addr;
3342
3343   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3344   echo0 = (icmp_echo_header_t *)(icmp0+1);
3345   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3346
3347   if (!icmp_is_error_message (icmp0))
3348     {
3349       protocol = SNAT_PROTOCOL_ICMP;
3350       key0.ext_host_addr = ip0->src_address;
3351       key0.ext_host_port = 0;
3352       key0.out_port = echo0->identifier;
3353       out_addr = ip0->dst_address;
3354     }
3355   else
3356     {
3357       inner_ip0 = (ip4_header_t *)(echo0+1);
3358       l4_header = ip4_next_header (inner_ip0);
3359       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3360       key0.ext_host_addr = inner_ip0->dst_address;
3361       out_addr = inner_ip0->src_address;
3362       switch (protocol)
3363         {
3364         case SNAT_PROTOCOL_ICMP:
3365           inner_icmp0 = (icmp46_header_t*)l4_header;
3366           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3367           key0.ext_host_port = 0;
3368           key0.out_port = inner_echo0->identifier;
3369           break;
3370         case SNAT_PROTOCOL_UDP:
3371         case SNAT_PROTOCOL_TCP:
3372           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3373           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3374           break;
3375         default:
3376           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3377           next0 = SNAT_OUT2IN_NEXT_DROP;
3378           goto out;
3379         }
3380     }
3381
3382   dm0 = snat_det_map_by_out(sm, &out_addr);
3383   if (PREDICT_FALSE(!dm0))
3384     {
3385       /* Don't NAT packet aimed at the intfc address */
3386       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3387                                           ip0->dst_address.as_u32)))
3388         {
3389           dont_translate = 1;
3390           goto out;
3391         }
3392       nat_log_info ("unknown dst address:  %U",
3393                     format_ip4_address, &ip0->dst_address);
3394       goto out;
3395     }
3396
3397   snat_det_reverse(dm0, &ip0->dst_address,
3398                    clib_net_to_host_u16(key0.out_port), &new_addr0);
3399
3400   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3401   if (PREDICT_FALSE(!ses0))
3402     {
3403       /* Don't NAT packet aimed at the intfc address */
3404       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3405                                           ip0->dst_address.as_u32)))
3406         {
3407           dont_translate = 1;
3408           goto out;
3409         }
3410       nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3411                     format_ip4_address, &key0.ext_host_addr,
3412                     clib_net_to_host_u16 (key0.ext_host_port),
3413                     format_ip4_address, &out_addr,
3414                     clib_net_to_host_u16 (key0.out_port),
3415                     format_ip4_address, &new_addr0);
3416       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3417       next0 = SNAT_OUT2IN_NEXT_DROP;
3418       goto out;
3419     }
3420
3421   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3422                     !icmp_is_error_message (icmp0)))
3423     {
3424       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3425       next0 = SNAT_OUT2IN_NEXT_DROP;
3426       goto out;
3427     }
3428
3429   goto out;
3430
3431 out:
3432   *p_proto = protocol;
3433   if (ses0)
3434     {
3435       p_value->addr = new_addr0;
3436       p_value->fib_index = sm->inside_fib_index;
3437       p_value->port = ses0->in_port;
3438     }
3439   *p_dont_translate = dont_translate;
3440   if (d)
3441     *(snat_det_session_t**)d = ses0;
3442   if (e)
3443     *(snat_det_map_t**)e = dm0;
3444   return next0;
3445 }
3446
3447 /**********************/
3448 /*** worker handoff ***/
3449 /**********************/
3450 static uword
3451 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3452                                vlib_node_runtime_t * node,
3453                                vlib_frame_t * frame)
3454 {
3455   snat_main_t *sm = &snat_main;
3456   vlib_thread_main_t *tm = vlib_get_thread_main ();
3457   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3458   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3459   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3460     = 0;
3461   vlib_frame_queue_elt_t *hf = 0;
3462   vlib_frame_queue_t *fq;
3463   vlib_frame_t *f = 0;
3464   int i;
3465   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3466   u32 next_worker_index = 0;
3467   u32 current_worker_index = ~0;
3468   u32 thread_index = vm->thread_index;
3469   vlib_frame_t *d = 0;
3470
3471   ASSERT (vec_len (sm->workers));
3472
3473   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3474     {
3475       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3476
3477       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3478                                tm->n_vlib_mains - 1,
3479                                (vlib_frame_queue_t *) (~0));
3480     }
3481
3482   from = vlib_frame_vector_args (frame);
3483   n_left_from = frame->n_vectors;
3484
3485   while (n_left_from > 0)
3486     {
3487       u32 bi0;
3488       vlib_buffer_t *b0;
3489       u32 sw_if_index0;
3490       u32 rx_fib_index0;
3491       ip4_header_t * ip0;
3492       u8 do_handoff;
3493
3494       bi0 = from[0];
3495       from += 1;
3496       n_left_from -= 1;
3497
3498       b0 = vlib_get_buffer (vm, bi0);
3499
3500       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3501       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3502
3503       ip0 = vlib_buffer_get_current (b0);
3504
3505       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3506
3507       if (PREDICT_FALSE (next_worker_index != thread_index))
3508         {
3509           do_handoff = 1;
3510
3511           if (next_worker_index != current_worker_index)
3512             {
3513               fq = is_vlib_frame_queue_congested (
3514                 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3515                 congested_handoff_queue_by_worker_index);
3516
3517               if (fq)
3518                 {
3519                   /* if this is 1st frame */
3520                   if (!d)
3521                     {
3522                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
3523                       to_next_drop = vlib_frame_vector_args (d);
3524                     }
3525
3526                   to_next_drop[0] = bi0;
3527                   to_next_drop += 1;
3528                   d->n_vectors++;
3529                   b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3530                   goto trace0;
3531                 }
3532
3533               if (hf)
3534                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3535
3536               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3537                                                       next_worker_index,
3538                                                       handoff_queue_elt_by_worker_index);
3539
3540               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3541               to_next_worker = &hf->buffer_index[hf->n_vectors];
3542               current_worker_index = next_worker_index;
3543             }
3544
3545           /* enqueue to correct worker thread */
3546           to_next_worker[0] = bi0;
3547           to_next_worker++;
3548           n_left_to_next_worker--;
3549
3550           if (n_left_to_next_worker == 0)
3551             {
3552               hf->n_vectors = VLIB_FRAME_SIZE;
3553               vlib_put_frame_queue_elt (hf);
3554               current_worker_index = ~0;
3555               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3556               hf = 0;
3557             }
3558         }
3559       else
3560         {
3561           do_handoff = 0;
3562           /* if this is 1st frame */
3563           if (!f)
3564             {
3565               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3566               to_next = vlib_frame_vector_args (f);
3567             }
3568
3569           to_next[0] = bi0;
3570           to_next += 1;
3571           f->n_vectors++;
3572         }
3573
3574 trace0:
3575       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3576                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3577         {
3578           snat_out2in_worker_handoff_trace_t *t =
3579             vlib_add_trace (vm, node, b0, sizeof (*t));
3580           t->next_worker_index = next_worker_index;
3581           t->do_handoff = do_handoff;
3582         }
3583     }
3584
3585   if (f)
3586     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3587
3588   if (d)
3589     vlib_put_frame_to_node (vm, sm->error_node_index, d);
3590
3591   if (hf)
3592     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3593
3594   /* Ship frames to the worker nodes */
3595   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3596     {
3597       if (handoff_queue_elt_by_worker_index[i])
3598         {
3599           hf = handoff_queue_elt_by_worker_index[i];
3600           /*
3601            * It works better to let the handoff node
3602            * rate-adapt, always ship the handoff queue element.
3603            */
3604           if (1 || hf->n_vectors == hf->last_n_vectors)
3605             {
3606               vlib_put_frame_queue_elt (hf);
3607               handoff_queue_elt_by_worker_index[i] = 0;
3608             }
3609           else
3610             hf->last_n_vectors = hf->n_vectors;
3611         }
3612       congested_handoff_queue_by_worker_index[i] =
3613         (vlib_frame_queue_t *) (~0);
3614     }
3615   hf = 0;
3616   current_worker_index = ~0;
3617   return frame->n_vectors;
3618 }
3619
3620 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3621   .function = snat_out2in_worker_handoff_fn,
3622   .name = "nat44-out2in-worker-handoff",
3623   .vector_size = sizeof (u32),
3624   .format_trace = format_snat_out2in_worker_handoff_trace,
3625   .type = VLIB_NODE_TYPE_INTERNAL,
3626
3627   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3628   .error_strings = snat_out2in_error_strings,
3629
3630   .n_next_nodes = 1,
3631
3632   .next_nodes = {
3633     [0] = "error-drop",
3634   },
3635 };
3636
3637 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3638
3639 static uword
3640 snat_out2in_fast_node_fn (vlib_main_t * vm,
3641                           vlib_node_runtime_t * node,
3642                           vlib_frame_t * frame)
3643 {
3644   u32 n_left_from, * from, * to_next;
3645   snat_out2in_next_t next_index;
3646   u32 pkts_processed = 0;
3647   snat_main_t * sm = &snat_main;
3648
3649   from = vlib_frame_vector_args (frame);
3650   n_left_from = frame->n_vectors;
3651   next_index = node->cached_next_index;
3652
3653   while (n_left_from > 0)
3654     {
3655       u32 n_left_to_next;
3656
3657       vlib_get_next_frame (vm, node, next_index,
3658                            to_next, n_left_to_next);
3659
3660       while (n_left_from > 0 && n_left_to_next > 0)
3661         {
3662           u32 bi0;
3663           vlib_buffer_t * b0;
3664           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3665           u32 sw_if_index0;
3666           ip4_header_t * ip0;
3667           ip_csum_t sum0;
3668           u32 new_addr0, old_addr0;
3669           u16 new_port0, old_port0;
3670           udp_header_t * udp0;
3671           tcp_header_t * tcp0;
3672           icmp46_header_t * icmp0;
3673           snat_session_key_t key0, sm0;
3674           u32 proto0;
3675           u32 rx_fib_index0;
3676
3677           /* speculatively enqueue b0 to the current next frame */
3678           bi0 = from[0];
3679           to_next[0] = bi0;
3680           from += 1;
3681           to_next += 1;
3682           n_left_from -= 1;
3683           n_left_to_next -= 1;
3684
3685           b0 = vlib_get_buffer (vm, bi0);
3686
3687           ip0 = vlib_buffer_get_current (b0);
3688           udp0 = ip4_next_header (ip0);
3689           tcp0 = (tcp_header_t *) udp0;
3690           icmp0 = (icmp46_header_t *) udp0;
3691
3692           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3693           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3694
3695           vnet_feature_next (&next0, b0);
3696
3697           if (PREDICT_FALSE(ip0->ttl == 1))
3698             {
3699               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3700               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3701                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3702                                            0);
3703               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3704               goto trace00;
3705             }
3706
3707           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3708
3709           if (PREDICT_FALSE (proto0 == ~0))
3710               goto trace00;
3711
3712           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3713             {
3714               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3715                                   rx_fib_index0, node, next0, ~0, 0, 0);
3716               goto trace00;
3717             }
3718
3719           key0.addr = ip0->dst_address;
3720           key0.port = udp0->dst_port;
3721           key0.fib_index = rx_fib_index0;
3722
3723           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3724             {
3725               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3726               goto trace00;
3727             }
3728
3729           new_addr0 = sm0.addr.as_u32;
3730           new_port0 = sm0.port;
3731           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3732           old_addr0 = ip0->dst_address.as_u32;
3733           ip0->dst_address.as_u32 = new_addr0;
3734
3735           sum0 = ip0->checksum;
3736           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3737                                  ip4_header_t,
3738                                  dst_address /* changed member */);
3739           ip0->checksum = ip_csum_fold (sum0);
3740
3741           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3742             {
3743                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3744                 {
3745                   old_port0 = tcp0->dst_port;
3746                   tcp0->dst_port = new_port0;
3747
3748                   sum0 = tcp0->checksum;
3749                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3750                                          ip4_header_t,
3751                                          dst_address /* changed member */);
3752
3753                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3754                                          ip4_header_t /* cheat */,
3755                                          length /* changed member */);
3756                   tcp0->checksum = ip_csum_fold(sum0);
3757                 }
3758               else
3759                 {
3760                   old_port0 = udp0->dst_port;
3761                   udp0->dst_port = new_port0;
3762                   udp0->checksum = 0;
3763                 }
3764             }
3765           else
3766             {
3767               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3768                 {
3769                   sum0 = tcp0->checksum;
3770                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3771                                          ip4_header_t,
3772                                          dst_address /* changed member */);
3773
3774                   tcp0->checksum = ip_csum_fold(sum0);
3775                 }
3776             }
3777
3778         trace00:
3779
3780           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3781                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3782             {
3783               snat_out2in_trace_t *t =
3784                  vlib_add_trace (vm, node, b0, sizeof (*t));
3785               t->sw_if_index = sw_if_index0;
3786               t->next_index = next0;
3787             }
3788
3789           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3790
3791           /* verify speculative enqueue, maybe switch current next frame */
3792           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3793                                            to_next, n_left_to_next,
3794                                            bi0, next0);
3795         }
3796
3797       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3798     }
3799
3800   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3801                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3802                                pkts_processed);
3803   return frame->n_vectors;
3804 }
3805
3806 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3807   .function = snat_out2in_fast_node_fn,
3808   .name = "nat44-out2in-fast",
3809   .vector_size = sizeof (u32),
3810   .format_trace = format_snat_out2in_fast_trace,
3811   .type = VLIB_NODE_TYPE_INTERNAL,
3812
3813   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3814   .error_strings = snat_out2in_error_strings,
3815
3816   .runtime_data_bytes = sizeof (snat_runtime_t),
3817
3818   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3819
3820   /* edit / add dispositions here */
3821   .next_nodes = {
3822     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3823     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3824     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3825     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3826   },
3827 };
3828 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);