src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29 #include <nat/nat_inlines.h>
  30
  31 #include <vppinfra/hash.h>
  32 #include <vppinfra/error.h>
  33 #include <vppinfra/elog.h>
  34
  35 typedef struct {
  36   u32 sw_if_index;
  37   u32 next_index;
  38   u32 session_index;
  39 } snat_out2in_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_out2in_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  52
  53   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  54               t->sw_if_index, t->next_index, t->session_index);
  55   return s;
  56 }
  57
  58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  63
  64   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  65               t->sw_if_index, t->next_index);
  66   return s;
  67 }
  68
  69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  70 {
  71   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  72   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  73   snat_out2in_worker_handoff_trace_t * t =
  74     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  75   char * m;
  76
  77   m = t->do_handoff ? "next worker" : "same worker";
  78   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  79
  80   return s;
  81 }
  82
  83 typedef struct {
  84   u32 sw_if_index;
  85   u32 next_index;
  86   u8 cached;
  87 } nat44_out2in_reass_trace_t;
  88
  89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  90 {
  91   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  92   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  93   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  94
  95   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  96               t->sw_if_index, t->next_index,
  97               t->cached ? "cached" : "translated");
  98
  99   return s;
 100 }
 101
 102 vlib_node_registration_t snat_out2in_node;
 103 vlib_node_registration_t snat_out2in_fast_node;
 104 vlib_node_registration_t snat_out2in_worker_handoff_node;
 105 vlib_node_registration_t snat_det_out2in_node;
 106 vlib_node_registration_t nat44_out2in_reass_node;
 107 vlib_node_registration_t nat44_ed_out2in_node;
 108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
 109
 110 #define foreach_snat_out2in_error                       \
 111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 112 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 113 _(OUT_OF_PORTS, "Out of ports")                         \
 114 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 115 _(NO_TRANSLATION, "No translation")                     \
 116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 117 _(DROP_FRAGMENT, "Drop fragment")                       \
 118 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 120 _(FQ_CONGESTED, "Handoff frame queue congested")
 121
 122 typedef enum {
 123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 124   foreach_snat_out2in_error
 125 #undef _
 126   SNAT_OUT2IN_N_ERROR,
 127 } snat_out2in_error_t;
 128
 129 static char * snat_out2in_error_strings[] = {
 130 #define _(sym,string) string,
 131   foreach_snat_out2in_error
 132 #undef _
 133 };
 134
 135 typedef enum {
 136   SNAT_OUT2IN_NEXT_DROP,
 137   SNAT_OUT2IN_NEXT_LOOKUP,
 138   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 139   SNAT_OUT2IN_NEXT_REASS,
 140   SNAT_OUT2IN_N_NEXT,
 141 } snat_out2in_next_t;
 142
 143 /**
 144  * @brief Create session for static mapping.
 145  *
 146  * Create NAT session initiated by host from external network with static
 147  * mapping.
 148  *
 149  * @param sm     NAT main.
 150  * @param b0     Vlib buffer.
 151  * @param in2out In2out NAT44 session key.
 152  * @param out2in Out2in NAT44 session key.
 153  * @param node   Vlib node.
 154  *
 155  * @returns SNAT session if successfully created otherwise 0.
 156  */
 157 static inline snat_session_t *
 158 create_session_for_static_mapping (snat_main_t *sm,
 159                                    vlib_buffer_t *b0,
 160                                    snat_session_key_t in2out,
 161                                    snat_session_key_t out2in,
 162                                    vlib_node_runtime_t * node,
 163                                    u32 thread_index)
 164 {
 165   snat_user_t *u;
 166   snat_session_t *s;
 167   clib_bihash_kv_8_8_t kv0;
 168   ip4_header_t *ip0;
 169   udp_header_t *udp0;
 170
 171   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 172     {
 173       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 174       nat_log_notice ("maximum sessions exceeded");
 175       return 0;
 176     }
 177
 178   ip0 = vlib_buffer_get_current (b0);
 179   udp0 = ip4_next_header (ip0);
 180
 181   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 182   if (!u)
 183     {
 184       nat_log_warn ("create NAT user failed");
 185       return 0;
 186     }
 187
 188   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 189   if (!s)
 190     {
 191       nat_log_warn ("create NAT session failed");
 192       return 0;
 193     }
 194
 195   s->outside_address_index = ~0;
 196   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 197   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 198   s->ext_host_port = udp0->src_port;
 199   user_session_increment (sm, u, 1 /* static */);
 200   s->in2out = in2out;
 201   s->out2in = out2in;
 202   s->in2out.protocol = out2in.protocol;
 203
 204   /* Add to translation hashes */
 205   kv0.key = s->in2out.as_u64;
 206   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 207   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 208                                1 /* is_add */))
 209       nat_log_notice ("in2out key add failed");
 210
 211   kv0.key = s->out2in.as_u64;
 212
 213   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 214                                1 /* is_add */))
 215       nat_log_notice ("out2in key add failed");
 216
 217   /* log NAT event */
 218   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 219                                       s->out2in.addr.as_u32,
 220                                       s->in2out.protocol,
 221                                       s->in2out.port,
 222                                       s->out2in.port,
 223                                       s->in2out.fib_index);
 224    return s;
 225 }
 226
 227 static_always_inline
 228 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 229                                  snat_session_key_t *p_key0)
 230 {
 231   icmp46_header_t *icmp0;
 232   snat_session_key_t key0;
 233   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 234   ip4_header_t *inner_ip0;
 235   void *l4_header = 0;
 236   icmp46_header_t *inner_icmp0;
 237
 238   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 239   echo0 = (icmp_echo_header_t *)(icmp0+1);
 240
 241   if (!icmp_is_error_message (icmp0))
 242     {
 243       key0.protocol = SNAT_PROTOCOL_ICMP;
 244       key0.addr = ip0->dst_address;
 245       key0.port = echo0->identifier;
 246     }
 247   else
 248     {
 249       inner_ip0 = (ip4_header_t *)(echo0+1);
 250       l4_header = ip4_next_header (inner_ip0);
 251       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 252       key0.addr = inner_ip0->src_address;
 253       switch (key0.protocol)
 254         {
 255         case SNAT_PROTOCOL_ICMP:
 256           inner_icmp0 = (icmp46_header_t*)l4_header;
 257           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 258           key0.port = inner_echo0->identifier;
 259           break;
 260         case SNAT_PROTOCOL_UDP:
 261         case SNAT_PROTOCOL_TCP:
 262           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 263           break;
 264         default:
 265           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 266         }
 267     }
 268   *p_key0 = key0;
 269   return -1; /* success */
 270 }
 271
 272 /**
 273  * Get address and port values to be used for ICMP packet translation
 274  * and create session if needed
 275  *
 276  * @param[in,out] sm             NAT main
 277  * @param[in,out] node           NAT node runtime
 278  * @param[in] thread_index       thread index
 279  * @param[in,out] b0             buffer containing packet to be translated
 280  * @param[out] p_proto           protocol used for matching
 281  * @param[out] p_value           address and port after NAT translation
 282  * @param[out] p_dont_translate  if packet should not be translated
 283  * @param d                      optional parameter
 284  * @param e                      optional parameter
 285  */
 286 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 287                            u32 thread_index, vlib_buffer_t *b0,
 288                            ip4_header_t *ip0, u8 *p_proto,
 289                            snat_session_key_t *p_value,
 290                            u8 *p_dont_translate, void *d, void *e)
 291 {
 292   icmp46_header_t *icmp0;
 293   u32 sw_if_index0;
 294   u32 rx_fib_index0;
 295   snat_session_key_t key0;
 296   snat_session_key_t sm0;
 297   snat_session_t *s0 = 0;
 298   u8 dont_translate = 0;
 299   clib_bihash_kv_8_8_t kv0, value0;
 300   u8 is_addr_only;
 301   u32 next0 = ~0;
 302   int err;
 303
 304   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 305   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 306   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 307
 308   key0.protocol = 0;
 309
 310   err = icmp_get_key (ip0, &key0);
 311   if (err != -1)
 312     {
 313       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 314       next0 = SNAT_OUT2IN_NEXT_DROP;
 315       goto out;
 316     }
 317   key0.fib_index = rx_fib_index0;
 318
 319   kv0.key = key0.as_u64;
 320
 321   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 322                               &value0))
 323     {
 324       /* Try to match static mapping by external address and port,
 325          destination address and port in packet */
 326       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 327         {
 328           if (!sm->forwarding_enabled)
 329             {
 330               /* Don't NAT packet aimed at the intfc address */
 331               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 332                                                   ip0->dst_address.as_u32)))
 333                 {
 334                   dont_translate = 1;
 335                   goto out;
 336                 }
 337               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 338               next0 = SNAT_OUT2IN_NEXT_DROP;
 339               goto out;
 340             }
 341           else
 342             {
 343               dont_translate = 1;
 344               goto out;
 345             }
 346         }
 347
 348       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 349                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 350         {
 351           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 352           next0 = SNAT_OUT2IN_NEXT_DROP;
 353           goto out;
 354         }
 355
 356       /* Create session initiated by host from external network */
 357       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 358                                              node, thread_index);
 359
 360       if (!s0)
 361         {
 362           next0 = SNAT_OUT2IN_NEXT_DROP;
 363           goto out;
 364         }
 365     }
 366   else
 367     {
 368       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 369                         icmp0->type != ICMP4_echo_request &&
 370                         !icmp_is_error_message (icmp0)))
 371         {
 372           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 373           next0 = SNAT_OUT2IN_NEXT_DROP;
 374           goto out;
 375         }
 376
 377       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 378                               value0.value);
 379     }
 380
 381 out:
 382   *p_proto = key0.protocol;
 383   if (s0)
 384     *p_value = s0->in2out;
 385   *p_dont_translate = dont_translate;
 386   if (d)
 387     *(snat_session_t**)d = s0;
 388   return next0;
 389 }
 390
 391 /**
 392  * Get address and port values to be used for ICMP packet translation
 393  *
 394  * @param[in] sm                 NAT main
 395  * @param[in,out] node           NAT node runtime
 396  * @param[in] thread_index       thread index
 397  * @param[in,out] b0             buffer containing packet to be translated
 398  * @param[out] p_proto           protocol used for matching
 399  * @param[out] p_value           address and port after NAT translation
 400  * @param[out] p_dont_translate  if packet should not be translated
 401  * @param d                      optional parameter
 402  * @param e                      optional parameter
 403  */
 404 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 405                            u32 thread_index, vlib_buffer_t *b0,
 406                            ip4_header_t *ip0, u8 *p_proto,
 407                            snat_session_key_t *p_value,
 408                            u8 *p_dont_translate, void *d, void *e)
 409 {
 410   icmp46_header_t *icmp0;
 411   u32 sw_if_index0;
 412   u32 rx_fib_index0;
 413   snat_session_key_t key0;
 414   snat_session_key_t sm0;
 415   u8 dont_translate = 0;
 416   u8 is_addr_only;
 417   u32 next0 = ~0;
 418   int err;
 419
 420   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 421   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 422   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 423
 424   err = icmp_get_key (ip0, &key0);
 425   if (err != -1)
 426     {
 427       b0->error = node->errors[err];
 428       next0 = SNAT_OUT2IN_NEXT_DROP;
 429       goto out2;
 430     }
 431   key0.fib_index = rx_fib_index0;
 432
 433   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
 434     {
 435       /* Don't NAT packet aimed at the intfc address */
 436       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 437         {
 438           dont_translate = 1;
 439           goto out;
 440         }
 441       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 442       next0 = SNAT_OUT2IN_NEXT_DROP;
 443       goto out;
 444     }
 445
 446   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 447                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 448                     !icmp_is_error_message (icmp0)))
 449     {
 450       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 451       next0 = SNAT_OUT2IN_NEXT_DROP;
 452       goto out;
 453     }
 454
 455 out:
 456   *p_value = sm0;
 457 out2:
 458   *p_proto = key0.protocol;
 459   *p_dont_translate = dont_translate;
 460   return next0;
 461 }
 462
 463 static inline u32 icmp_out2in (snat_main_t *sm,
 464                                vlib_buffer_t * b0,
 465                                ip4_header_t * ip0,
 466                                icmp46_header_t * icmp0,
 467                                u32 sw_if_index0,
 468                                u32 rx_fib_index0,
 469                                vlib_node_runtime_t * node,
 470                                u32 next0,
 471                                u32 thread_index,
 472                                void *d,
 473                                void *e)
 474 {
 475   snat_session_key_t sm0;
 476   u8 protocol;
 477   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 478   ip4_header_t *inner_ip0 = 0;
 479   void *l4_header = 0;
 480   icmp46_header_t *inner_icmp0;
 481   u8 dont_translate;
 482   u32 new_addr0, old_addr0;
 483   u16 old_id0, new_id0;
 484   ip_csum_t sum0;
 485   u16 checksum0;
 486   u32 next0_tmp;
 487
 488   echo0 = (icmp_echo_header_t *)(icmp0+1);
 489
 490   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 491                                        &protocol, &sm0, &dont_translate, d, e);
 492   if (next0_tmp != ~0)
 493     next0 = next0_tmp;
 494   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 495     goto out;
 496
 497   sum0 = ip_incremental_checksum (0, icmp0,
 498                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 499   checksum0 = ~ip_csum_fold (sum0);
 500   if (checksum0 != 0 && checksum0 != 0xffff)
 501     {
 502       next0 = SNAT_OUT2IN_NEXT_DROP;
 503       goto out;
 504     }
 505
 506   old_addr0 = ip0->dst_address.as_u32;
 507   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 508   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 509
 510   sum0 = ip0->checksum;
 511   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 512                          dst_address /* changed member */);
 513   ip0->checksum = ip_csum_fold (sum0);
 514
 515   if (icmp0->checksum == 0)
 516     icmp0->checksum = 0xffff;
 517
 518   if (!icmp_is_error_message (icmp0))
 519     {
 520       new_id0 = sm0.port;
 521       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 522         {
 523           old_id0 = echo0->identifier;
 524           new_id0 = sm0.port;
 525           echo0->identifier = new_id0;
 526
 527           sum0 = icmp0->checksum;
 528           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 529                                  identifier /* changed member */);
 530           icmp0->checksum = ip_csum_fold (sum0);
 531         }
 532     }
 533   else
 534     {
 535       inner_ip0 = (ip4_header_t *)(echo0+1);
 536       l4_header = ip4_next_header (inner_ip0);
 537
 538       if (!ip4_header_checksum_is_valid (inner_ip0))
 539         {
 540           next0 = SNAT_OUT2IN_NEXT_DROP;
 541           goto out;
 542         }
 543
 544       old_addr0 = inner_ip0->src_address.as_u32;
 545       inner_ip0->src_address = sm0.addr;
 546       new_addr0 = inner_ip0->src_address.as_u32;
 547
 548       sum0 = icmp0->checksum;
 549       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 550                              src_address /* changed member */);
 551       icmp0->checksum = ip_csum_fold (sum0);
 552
 553       switch (protocol)
 554         {
 555         case SNAT_PROTOCOL_ICMP:
 556           inner_icmp0 = (icmp46_header_t*)l4_header;
 557           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 558
 559           old_id0 = inner_echo0->identifier;
 560           new_id0 = sm0.port;
 561           inner_echo0->identifier = new_id0;
 562
 563           sum0 = icmp0->checksum;
 564           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 565                                  identifier);
 566           icmp0->checksum = ip_csum_fold (sum0);
 567           break;
 568         case SNAT_PROTOCOL_UDP:
 569         case SNAT_PROTOCOL_TCP:
 570           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 571           new_id0 = sm0.port;
 572           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 573
 574           sum0 = icmp0->checksum;
 575           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 576                                  src_port);
 577           icmp0->checksum = ip_csum_fold (sum0);
 578           break;
 579         default:
 580           ASSERT(0);
 581         }
 582     }
 583
 584 out:
 585   return next0;
 586 }
 587
 588
 589 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 590                                          vlib_buffer_t * b0,
 591                                          ip4_header_t * ip0,
 592                                          icmp46_header_t * icmp0,
 593                                          u32 sw_if_index0,
 594                                          u32 rx_fib_index0,
 595                                          vlib_node_runtime_t * node,
 596                                          u32 next0, f64 now,
 597                                          u32 thread_index,
 598                                          snat_session_t ** p_s0)
 599 {
 600   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 601                       next0, thread_index, p_s0, 0);
 602   snat_session_t * s0 = *p_s0;
 603   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 604     {
 605       /* Accounting */
 606       nat44_session_update_counters (s0, now,
 607                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
 608       /* Per-user LRU list maintenance */
 609       nat44_session_update_lru (sm, s0, thread_index);
 610     }
 611   return next0;
 612 }
 613
 614 static int
 615 nat_out2in_sm_unknown_proto (snat_main_t *sm,
 616                              vlib_buffer_t * b,
 617                              ip4_header_t * ip,
 618                              u32 rx_fib_index)
 619 {
 620   clib_bihash_kv_8_8_t kv, value;
 621   snat_static_mapping_t *m;
 622   snat_session_key_t m_key;
 623   u32 old_addr, new_addr;
 624   ip_csum_t sum;
 625
 626   m_key.addr = ip->dst_address;
 627   m_key.port = 0;
 628   m_key.protocol = 0;
 629   m_key.fib_index = rx_fib_index;
 630   kv.key = m_key.as_u64;
 631   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 632     return 1;
 633
 634   m = pool_elt_at_index (sm->static_mappings, value.value);
 635
 636   old_addr = ip->dst_address.as_u32;
 637   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 638   sum = ip->checksum;
 639   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 640   ip->checksum = ip_csum_fold (sum);
 641
 642   vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 643   return 0;
 644 }
 645
 646 static uword
 647 snat_out2in_node_fn (vlib_main_t * vm,
 648                   vlib_node_runtime_t * node,
 649                   vlib_frame_t * frame)
 650 {
 651   u32 n_left_from, * from, * to_next;
 652   snat_out2in_next_t next_index;
 653   u32 pkts_processed = 0;
 654   snat_main_t * sm = &snat_main;
 655   f64 now = vlib_time_now (vm);
 656   u32 thread_index = vlib_get_thread_index ();
 657
 658   from = vlib_frame_vector_args (frame);
 659   n_left_from = frame->n_vectors;
 660   next_index = node->cached_next_index;
 661
 662   while (n_left_from > 0)
 663     {
 664       u32 n_left_to_next;
 665
 666       vlib_get_next_frame (vm, node, next_index,
 667                            to_next, n_left_to_next);
 668
 669       while (n_left_from >= 4 && n_left_to_next >= 2)
 670         {
 671           u32 bi0, bi1;
 672           vlib_buffer_t * b0, * b1;
 673           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 674           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 675           u32 sw_if_index0, sw_if_index1;
 676           ip4_header_t * ip0, *ip1;
 677           ip_csum_t sum0, sum1;
 678           u32 new_addr0, old_addr0;
 679           u16 new_port0, old_port0;
 680           u32 new_addr1, old_addr1;
 681           u16 new_port1, old_port1;
 682           udp_header_t * udp0, * udp1;
 683           tcp_header_t * tcp0, * tcp1;
 684           icmp46_header_t * icmp0, * icmp1;
 685           snat_session_key_t key0, key1, sm0, sm1;
 686           u32 rx_fib_index0, rx_fib_index1;
 687           u32 proto0, proto1;
 688           snat_session_t * s0 = 0, * s1 = 0;
 689           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 690
 691           /* Prefetch next iteration. */
 692           {
 693             vlib_buffer_t * p2, * p3;
 694
 695             p2 = vlib_get_buffer (vm, from[2]);
 696             p3 = vlib_get_buffer (vm, from[3]);
 697
 698             vlib_prefetch_buffer_header (p2, LOAD);
 699             vlib_prefetch_buffer_header (p3, LOAD);
 700
 701             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 702             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 703           }
 704
 705           /* speculatively enqueue b0 and b1 to the current next frame */
 706           to_next[0] = bi0 = from[0];
 707           to_next[1] = bi1 = from[1];
 708           from += 2;
 709           to_next += 2;
 710           n_left_from -= 2;
 711           n_left_to_next -= 2;
 712
 713           b0 = vlib_get_buffer (vm, bi0);
 714           b1 = vlib_get_buffer (vm, bi1);
 715
 716           vnet_buffer (b0)->snat.flags = 0;
 717           vnet_buffer (b1)->snat.flags = 0;
 718
 719           ip0 = vlib_buffer_get_current (b0);
 720           udp0 = ip4_next_header (ip0);
 721           tcp0 = (tcp_header_t *) udp0;
 722           icmp0 = (icmp46_header_t *) udp0;
 723
 724           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 725           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 726                                    sw_if_index0);
 727
 728           if (PREDICT_FALSE(ip0->ttl == 1))
 729             {
 730               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 731               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 732                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 733                                            0);
 734               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 735               goto trace0;
 736             }
 737
 738           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 739
 740           if (PREDICT_FALSE (proto0 == ~0))
 741             {
 742               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
 743                 {
 744                   if (!sm->forwarding_enabled)
 745                     {
 746                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 747                       next0 = SNAT_OUT2IN_NEXT_DROP;
 748                     }
 749                 }
 750               goto trace0;
 751             }
 752
 753           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 754             {
 755               next0 = icmp_out2in_slow_path
 756                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 757                  next0, now, thread_index, &s0);
 758               goto trace0;
 759             }
 760
 761           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
 762             {
 763               next0 = SNAT_OUT2IN_NEXT_REASS;
 764               goto trace0;
 765             }
 766
 767           key0.addr = ip0->dst_address;
 768           key0.port = udp0->dst_port;
 769           key0.protocol = proto0;
 770           key0.fib_index = rx_fib_index0;
 771
 772           kv0.key = key0.as_u64;
 773
 774           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 775                                       &kv0, &value0))
 776             {
 777               /* Try to match static mapping by external address and port,
 778                  destination address and port in packet */
 779               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 780                 {
 781                   /*
 782                    * Send DHCP packets to the ipv4 stack, or we won't
 783                    * be able to use dhcp client on the outside interface
 784                    */
 785                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 786                       && (udp0->dst_port ==
 787                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 788                     {
 789                       vnet_feature_next
 790                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
 791                       goto trace0;
 792                     }
 793
 794                   if (!sm->forwarding_enabled)
 795                     {
 796                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 797                       next0 = SNAT_OUT2IN_NEXT_DROP;
 798                       goto trace0;
 799                     }
 800                 }
 801
 802               /* Create session initiated by host from external network */
 803               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 804                                                      thread_index);
 805               if (!s0)
 806                 {
 807                   next0 = SNAT_OUT2IN_NEXT_DROP;
 808                   goto trace0;
 809                 }
 810             }
 811           else
 812             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 813                                     value0.value);
 814
 815           old_addr0 = ip0->dst_address.as_u32;
 816           ip0->dst_address = s0->in2out.addr;
 817           new_addr0 = ip0->dst_address.as_u32;
 818           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 819
 820           sum0 = ip0->checksum;
 821           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 822                                  ip4_header_t,
 823                                  dst_address /* changed member */);
 824           ip0->checksum = ip_csum_fold (sum0);
 825
 826           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 827             {
 828               old_port0 = tcp0->dst_port;
 829               tcp0->dst_port = s0->in2out.port;
 830               new_port0 = tcp0->dst_port;
 831
 832               sum0 = tcp0->checksum;
 833               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 834                                      ip4_header_t,
 835                                      dst_address /* changed member */);
 836
 837               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 838                                      ip4_header_t /* cheat */,
 839                                      length /* changed member */);
 840               tcp0->checksum = ip_csum_fold(sum0);
 841             }
 842           else
 843             {
 844               old_port0 = udp0->dst_port;
 845               udp0->dst_port = s0->in2out.port;
 846               udp0->checksum = 0;
 847             }
 848
 849           /* Accounting */
 850           nat44_session_update_counters (s0, now,
 851                                          vlib_buffer_length_in_chain (vm, b0));
 852           /* Per-user LRU list maintenance */
 853           nat44_session_update_lru (sm, s0, thread_index);
 854         trace0:
 855
 856           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 857                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 858             {
 859               snat_out2in_trace_t *t =
 860                  vlib_add_trace (vm, node, b0, sizeof (*t));
 861               t->sw_if_index = sw_if_index0;
 862               t->next_index = next0;
 863               t->session_index = ~0;
 864               if (s0)
 865                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 866             }
 867
 868           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 869
 870
 871           ip1 = vlib_buffer_get_current (b1);
 872           udp1 = ip4_next_header (ip1);
 873           tcp1 = (tcp_header_t *) udp1;
 874           icmp1 = (icmp46_header_t *) udp1;
 875
 876           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 877           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 878                                    sw_if_index1);
 879
 880           if (PREDICT_FALSE(ip1->ttl == 1))
 881             {
 882               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 883               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 884                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 885                                            0);
 886               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 887               goto trace1;
 888             }
 889
 890           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 891
 892           if (PREDICT_FALSE (proto1 == ~0))
 893             {
 894               if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
 895                 {
 896                   if (!sm->forwarding_enabled)
 897                     {
 898                       b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 899                       next1 = SNAT_OUT2IN_NEXT_DROP;
 900                     }
 901                 }
 902               goto trace1;
 903             }
 904
 905           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 906             {
 907               next1 = icmp_out2in_slow_path
 908                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 909                  next1, now, thread_index, &s1);
 910               goto trace1;
 911             }
 912
 913           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
 914             {
 915               next1 = SNAT_OUT2IN_NEXT_REASS;
 916               goto trace1;
 917             }
 918
 919           key1.addr = ip1->dst_address;
 920           key1.port = udp1->dst_port;
 921           key1.protocol = proto1;
 922           key1.fib_index = rx_fib_index1;
 923
 924           kv1.key = key1.as_u64;
 925
 926           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
 927                                       &kv1, &value1))
 928             {
 929               /* Try to match static mapping by external address and port,
 930                  destination address and port in packet */
 931               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
 932                 {
 933                   /*
 934                    * Send DHCP packets to the ipv4 stack, or we won't
 935                    * be able to use dhcp client on the outside interface
 936                    */
 937                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
 938                       && (udp1->dst_port ==
 939                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
 940                     {
 941                       vnet_feature_next
 942                         (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
 943                       goto trace1;
 944                     }
 945
 946                   if (!sm->forwarding_enabled)
 947                     {
 948                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 949                       next1 = SNAT_OUT2IN_NEXT_DROP;
 950                       goto trace1;
 951                     }
 952                 }
 953
 954               /* Create session initiated by host from external network */
 955               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 956                                                      thread_index);
 957               if (!s1)
 958                 {
 959                   next1 = SNAT_OUT2IN_NEXT_DROP;
 960                   goto trace1;
 961                 }
 962             }
 963           else
 964             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 965                                     value1.value);
 966
 967           old_addr1 = ip1->dst_address.as_u32;
 968           ip1->dst_address = s1->in2out.addr;
 969           new_addr1 = ip1->dst_address.as_u32;
 970           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 971
 972           sum1 = ip1->checksum;
 973           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 974                                  ip4_header_t,
 975                                  dst_address /* changed member */);
 976           ip1->checksum = ip_csum_fold (sum1);
 977
 978           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 979             {
 980               old_port1 = tcp1->dst_port;
 981               tcp1->dst_port = s1->in2out.port;
 982               new_port1 = tcp1->dst_port;
 983
 984               sum1 = tcp1->checksum;
 985               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 986                                      ip4_header_t,
 987                                      dst_address /* changed member */);
 988
 989               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 990                                      ip4_header_t /* cheat */,
 991                                      length /* changed member */);
 992               tcp1->checksum = ip_csum_fold(sum1);
 993             }
 994           else
 995             {
 996               old_port1 = udp1->dst_port;
 997               udp1->dst_port = s1->in2out.port;
 998               udp1->checksum = 0;
 999             }
1000
1001           /* Accounting */
1002           nat44_session_update_counters (s1, now,
1003                                          vlib_buffer_length_in_chain (vm, b1));
1004           /* Per-user LRU list maintenance */
1005           nat44_session_update_lru (sm, s1, thread_index);
1006         trace1:
1007
1008           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1009                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1010             {
1011               snat_out2in_trace_t *t =
1012                  vlib_add_trace (vm, node, b1, sizeof (*t));
1013               t->sw_if_index = sw_if_index1;
1014               t->next_index = next1;
1015               t->session_index = ~0;
1016               if (s1)
1017                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1018             }
1019
1020           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1021
1022           /* verify speculative enqueues, maybe switch current next frame */
1023           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1024                                            to_next, n_left_to_next,
1025                                            bi0, bi1, next0, next1);
1026         }
1027
1028       while (n_left_from > 0 && n_left_to_next > 0)
1029         {
1030           u32 bi0;
1031           vlib_buffer_t * b0;
1032           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1033           u32 sw_if_index0;
1034           ip4_header_t * ip0;
1035           ip_csum_t sum0;
1036           u32 new_addr0, old_addr0;
1037           u16 new_port0, old_port0;
1038           udp_header_t * udp0;
1039           tcp_header_t * tcp0;
1040           icmp46_header_t * icmp0;
1041           snat_session_key_t key0, sm0;
1042           u32 rx_fib_index0;
1043           u32 proto0;
1044           snat_session_t * s0 = 0;
1045           clib_bihash_kv_8_8_t kv0, value0;
1046
1047           /* speculatively enqueue b0 to the current next frame */
1048           bi0 = from[0];
1049           to_next[0] = bi0;
1050           from += 1;
1051           to_next += 1;
1052           n_left_from -= 1;
1053           n_left_to_next -= 1;
1054
1055           b0 = vlib_get_buffer (vm, bi0);
1056
1057           vnet_buffer (b0)->snat.flags = 0;
1058
1059           ip0 = vlib_buffer_get_current (b0);
1060           udp0 = ip4_next_header (ip0);
1061           tcp0 = (tcp_header_t *) udp0;
1062           icmp0 = (icmp46_header_t *) udp0;
1063
1064           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1065           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1066                                    sw_if_index0);
1067
1068           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1069
1070           if (PREDICT_FALSE (proto0 == ~0))
1071             {
1072               if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1073                 {
1074                   if (!sm->forwarding_enabled)
1075                     {
1076                       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1077                       next0 = SNAT_OUT2IN_NEXT_DROP;
1078                     }
1079                 }
1080               goto trace00;
1081             }
1082
1083           if (PREDICT_FALSE(ip0->ttl == 1))
1084             {
1085               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1086               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1087                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1088                                            0);
1089               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1090               goto trace00;
1091             }
1092
1093           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1094             {
1095               next0 = icmp_out2in_slow_path
1096                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1097                  next0, now, thread_index, &s0);
1098               goto trace00;
1099             }
1100
1101           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1102             {
1103               next0 = SNAT_OUT2IN_NEXT_REASS;
1104               goto trace00;
1105             }
1106
1107           key0.addr = ip0->dst_address;
1108           key0.port = udp0->dst_port;
1109           key0.protocol = proto0;
1110           key0.fib_index = rx_fib_index0;
1111
1112           kv0.key = key0.as_u64;
1113
1114           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1115                                       &kv0, &value0))
1116             {
1117               /* Try to match static mapping by external address and port,
1118                  destination address and port in packet */
1119               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1120                 {
1121                   /*
1122                    * Send DHCP packets to the ipv4 stack, or we won't
1123                    * be able to use dhcp client on the outside interface
1124                    */
1125                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1126                       && (udp0->dst_port ==
1127                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1128                     {
1129                       vnet_feature_next
1130                         (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1131                       goto trace00;
1132                     }
1133
1134                   if (!sm->forwarding_enabled)
1135                     {
1136                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1137                       next0 = SNAT_OUT2IN_NEXT_DROP;
1138                       goto trace00;
1139                     }
1140                 }
1141
1142               /* Create session initiated by host from external network */
1143               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1144                                                      thread_index);
1145               if (!s0)
1146                 {
1147                   next0 = SNAT_OUT2IN_NEXT_DROP;
1148                   goto trace00;
1149                 }
1150             }
1151           else
1152             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1153                                     value0.value);
1154
1155           old_addr0 = ip0->dst_address.as_u32;
1156           ip0->dst_address = s0->in2out.addr;
1157           new_addr0 = ip0->dst_address.as_u32;
1158           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1159
1160           sum0 = ip0->checksum;
1161           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1162                                  ip4_header_t,
1163                                  dst_address /* changed member */);
1164           ip0->checksum = ip_csum_fold (sum0);
1165
1166           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1167             {
1168               old_port0 = tcp0->dst_port;
1169               tcp0->dst_port = s0->in2out.port;
1170               new_port0 = tcp0->dst_port;
1171
1172               sum0 = tcp0->checksum;
1173               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1174                                      ip4_header_t,
1175                                      dst_address /* changed member */);
1176
1177               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1178                                      ip4_header_t /* cheat */,
1179                                      length /* changed member */);
1180               tcp0->checksum = ip_csum_fold(sum0);
1181             }
1182           else
1183             {
1184               old_port0 = udp0->dst_port;
1185               udp0->dst_port = s0->in2out.port;
1186               udp0->checksum = 0;
1187             }
1188
1189           /* Accounting */
1190           nat44_session_update_counters (s0, now,
1191                                          vlib_buffer_length_in_chain (vm, b0));
1192           /* Per-user LRU list maintenance */
1193           nat44_session_update_lru (sm, s0, thread_index);
1194         trace00:
1195
1196           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1197                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1198             {
1199               snat_out2in_trace_t *t =
1200                  vlib_add_trace (vm, node, b0, sizeof (*t));
1201               t->sw_if_index = sw_if_index0;
1202               t->next_index = next0;
1203               t->session_index = ~0;
1204               if (s0)
1205                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1206             }
1207
1208           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1209
1210           /* verify speculative enqueue, maybe switch current next frame */
1211           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1212                                            to_next, n_left_to_next,
1213                                            bi0, next0);
1214         }
1215
1216       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1217     }
1218
1219   vlib_node_increment_counter (vm, snat_out2in_node.index,
1220                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1221                                pkts_processed);
1222   return frame->n_vectors;
1223 }
1224
1225 VLIB_REGISTER_NODE (snat_out2in_node) = {
1226   .function = snat_out2in_node_fn,
1227   .name = "nat44-out2in",
1228   .vector_size = sizeof (u32),
1229   .format_trace = format_snat_out2in_trace,
1230   .type = VLIB_NODE_TYPE_INTERNAL,
1231
1232   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1233   .error_strings = snat_out2in_error_strings,
1234
1235   .runtime_data_bytes = sizeof (snat_runtime_t),
1236
1237   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1238
1239   /* edit / add dispositions here */
1240   .next_nodes = {
1241     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1242     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1243     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1244     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1245   },
1246 };
1247 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1248
1249 static uword
1250 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1251                             vlib_node_runtime_t * node,
1252                             vlib_frame_t * frame)
1253 {
1254   u32 n_left_from, *from, *to_next;
1255   snat_out2in_next_t next_index;
1256   u32 pkts_processed = 0;
1257   snat_main_t *sm = &snat_main;
1258   f64 now = vlib_time_now (vm);
1259   u32 thread_index = vlib_get_thread_index ();
1260   snat_main_per_thread_data_t *per_thread_data =
1261     &sm->per_thread_data[thread_index];
1262   u32 *fragments_to_drop = 0;
1263   u32 *fragments_to_loopback = 0;
1264
1265   from = vlib_frame_vector_args (frame);
1266   n_left_from = frame->n_vectors;
1267   next_index = node->cached_next_index;
1268
1269   while (n_left_from > 0)
1270     {
1271       u32 n_left_to_next;
1272
1273       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1274
1275       while (n_left_from > 0 && n_left_to_next > 0)
1276        {
1277           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1278           vlib_buffer_t *b0;
1279           u32 next0;
1280           u8 cached0 = 0;
1281           ip4_header_t *ip0;
1282           nat_reass_ip4_t *reass0;
1283           udp_header_t * udp0;
1284           tcp_header_t * tcp0;
1285           snat_session_key_t key0, sm0;
1286           clib_bihash_kv_8_8_t kv0, value0;
1287           snat_session_t * s0 = 0;
1288           u16 old_port0, new_port0;
1289           ip_csum_t sum0;
1290
1291           /* speculatively enqueue b0 to the current next frame */
1292           bi0 = from[0];
1293           to_next[0] = bi0;
1294           from += 1;
1295           to_next += 1;
1296           n_left_from -= 1;
1297           n_left_to_next -= 1;
1298
1299           b0 = vlib_get_buffer (vm, bi0);
1300           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1301
1302           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1303           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1304                                                                sw_if_index0);
1305
1306           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1307             {
1308               next0 = SNAT_OUT2IN_NEXT_DROP;
1309               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1310               goto trace0;
1311             }
1312
1313           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1314           udp0 = ip4_next_header (ip0);
1315           tcp0 = (tcp_header_t *) udp0;
1316           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1317
1318           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1319                                                  ip0->dst_address,
1320                                                  ip0->fragment_id,
1321                                                  ip0->protocol,
1322                                                  1,
1323                                                  &fragments_to_drop);
1324
1325           if (PREDICT_FALSE (!reass0))
1326             {
1327               next0 = SNAT_OUT2IN_NEXT_DROP;
1328               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1329               nat_log_notice ("maximum reassemblies exceeded");
1330               goto trace0;
1331             }
1332
1333           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1334             {
1335               key0.addr = ip0->dst_address;
1336               key0.port = udp0->dst_port;
1337               key0.protocol = proto0;
1338               key0.fib_index = rx_fib_index0;
1339               kv0.key = key0.as_u64;
1340
1341               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1342                 {
1343                   /* Try to match static mapping by external address and port,
1344                      destination address and port in packet */
1345                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1346                     {
1347                       /*
1348                        * Send DHCP packets to the ipv4 stack, or we won't
1349                        * be able to use dhcp client on the outside interface
1350                        */
1351                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1352                           && (udp0->dst_port
1353                               == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1354                         {
1355                           vnet_feature_next
1356                             (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1357                              &next0, b0);
1358                           goto trace0;
1359                         }
1360
1361                       if (!sm->forwarding_enabled)
1362                         {
1363                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1364                           next0 = SNAT_OUT2IN_NEXT_DROP;
1365                         }
1366                       goto trace0;
1367                     }
1368
1369                   /* Create session initiated by host from external network */
1370                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1371                                                          thread_index);
1372                   if (!s0)
1373                     {
1374                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1375                       next0 = SNAT_OUT2IN_NEXT_DROP;
1376                       goto trace0;
1377                     }
1378                   reass0->sess_index = s0 - per_thread_data->sessions;
1379                   reass0->thread_index = thread_index;
1380                 }
1381               else
1382                 {
1383                   s0 = pool_elt_at_index (per_thread_data->sessions,
1384                                           value0.value);
1385                   reass0->sess_index = value0.value;
1386                 }
1387               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1388             }
1389           else
1390             {
1391               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1392                 {
1393                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1394                     {
1395                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1396                       nat_log_notice ("maximum fragments per reassembly exceeded");
1397                       next0 = SNAT_OUT2IN_NEXT_DROP;
1398                       goto trace0;
1399                     }
1400                   cached0 = 1;
1401                   goto trace0;
1402                 }
1403               s0 = pool_elt_at_index (per_thread_data->sessions,
1404                                       reass0->sess_index);
1405             }
1406
1407           old_addr0 = ip0->dst_address.as_u32;
1408           ip0->dst_address = s0->in2out.addr;
1409           new_addr0 = ip0->dst_address.as_u32;
1410           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1411
1412           sum0 = ip0->checksum;
1413           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1414                                  ip4_header_t,
1415                                  dst_address /* changed member */);
1416           ip0->checksum = ip_csum_fold (sum0);
1417
1418           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1419             {
1420               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1421                 {
1422                   old_port0 = tcp0->dst_port;
1423                   tcp0->dst_port = s0->in2out.port;
1424                   new_port0 = tcp0->dst_port;
1425
1426                   sum0 = tcp0->checksum;
1427                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1428                                          ip4_header_t,
1429                                          dst_address /* changed member */);
1430
1431                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1432                                          ip4_header_t /* cheat */,
1433                                          length /* changed member */);
1434                   tcp0->checksum = ip_csum_fold(sum0);
1435                 }
1436               else
1437                 {
1438                   old_port0 = udp0->dst_port;
1439                   udp0->dst_port = s0->in2out.port;
1440                   udp0->checksum = 0;
1441                 }
1442             }
1443
1444           /* Accounting */
1445           nat44_session_update_counters (s0, now,
1446                                          vlib_buffer_length_in_chain (vm, b0));
1447           /* Per-user LRU list maintenance */
1448           nat44_session_update_lru (sm, s0, thread_index);
1449
1450         trace0:
1451           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1452                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1453             {
1454               nat44_out2in_reass_trace_t *t =
1455                  vlib_add_trace (vm, node, b0, sizeof (*t));
1456               t->cached = cached0;
1457               t->sw_if_index = sw_if_index0;
1458               t->next_index = next0;
1459             }
1460
1461           if (cached0)
1462             {
1463               n_left_to_next++;
1464               to_next--;
1465             }
1466           else
1467             {
1468               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1469
1470               /* verify speculative enqueue, maybe switch current next frame */
1471               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1472                                                to_next, n_left_to_next,
1473                                                bi0, next0);
1474             }
1475
1476           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1477             {
1478               from = vlib_frame_vector_args (frame);
1479               u32 len = vec_len (fragments_to_loopback);
1480               if (len <= VLIB_FRAME_SIZE)
1481                 {
1482                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1483                   n_left_from = len;
1484                   vec_reset_length (fragments_to_loopback);
1485                 }
1486               else
1487                 {
1488                   clib_memcpy (from,
1489                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1490                                sizeof (u32) * VLIB_FRAME_SIZE);
1491                   n_left_from = VLIB_FRAME_SIZE;
1492                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1493                 }
1494             }
1495        }
1496
1497       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1498     }
1499
1500   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1501                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1502                                pkts_processed);
1503
1504   nat_send_all_to_node (vm, fragments_to_drop, node,
1505                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1506                         SNAT_OUT2IN_NEXT_DROP);
1507
1508   vec_free (fragments_to_drop);
1509   vec_free (fragments_to_loopback);
1510   return frame->n_vectors;
1511 }
1512
1513 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1514   .function = nat44_out2in_reass_node_fn,
1515   .name = "nat44-out2in-reass",
1516   .vector_size = sizeof (u32),
1517   .format_trace = format_nat44_out2in_reass_trace,
1518   .type = VLIB_NODE_TYPE_INTERNAL,
1519
1520   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1521   .error_strings = snat_out2in_error_strings,
1522
1523   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1524
1525   /* edit / add dispositions here */
1526   .next_nodes = {
1527     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1528     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1529     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1530     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1531   },
1532 };
1533 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1534                               nat44_out2in_reass_node_fn);
1535
1536 /*******************************/
1537 /*** endpoint-dependent mode ***/
1538 /*******************************/
1539 typedef enum {
1540   NAT44_ED_OUT2IN_NEXT_DROP,
1541   NAT44_ED_OUT2IN_NEXT_LOOKUP,
1542   NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1543   NAT44_ED_OUT2IN_NEXT_IN2OUT,
1544   NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1545   NAT44_ED_OUT2IN_N_NEXT,
1546 } nat44_ed_out2in_next_t;
1547
1548 typedef struct {
1549   u32 sw_if_index;
1550   u32 next_index;
1551   u32 session_index;
1552   u32 is_slow_path;
1553 } nat44_ed_out2in_trace_t;
1554
1555 static u8 *
1556 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1557 {
1558   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1559   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1560   nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1561   char * tag;
1562
1563   tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1564
1565   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1566               t->sw_if_index, t->next_index, t->session_index);
1567
1568   return s;
1569 }
1570
1571 static snat_session_t *
1572 create_session_for_static_mapping_ed (snat_main_t * sm,
1573                                       vlib_buffer_t *b,
1574                                       snat_session_key_t l_key,
1575                                       snat_session_key_t e_key,
1576                                       vlib_node_runtime_t * node,
1577                                       u32 thread_index,
1578                                       twice_nat_type_t twice_nat,
1579                                       u8 is_lb)
1580 {
1581   snat_session_t *s;
1582   snat_user_t *u;
1583   ip4_header_t *ip;
1584   udp_header_t *udp;
1585   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1586   clib_bihash_kv_16_8_t kv;
1587   snat_session_key_t eh_key;
1588   u32 address_index;
1589
1590   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1591     {
1592       b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1593       nat_log_notice ("maximum sessions exceeded");
1594       return 0;
1595     }
1596
1597   u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1598   if (!u)
1599     {
1600       nat_log_warn ("create NAT user failed");
1601       return 0;
1602     }
1603
1604   s = nat_session_alloc_or_recycle (sm, u, thread_index);
1605   if (!s)
1606     {
1607       nat_log_warn ("create NAT session failed");
1608       return 0;
1609     }
1610
1611   ip = vlib_buffer_get_current (b);
1612   udp = ip4_next_header (ip);
1613
1614   s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1615   s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1616   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1617   if (is_lb)
1618     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1619   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1620   s->outside_address_index = ~0;
1621   s->out2in = e_key;
1622   s->in2out = l_key;
1623   s->in2out.protocol = s->out2in.protocol;
1624   user_session_increment (sm, u, 1);
1625
1626   /* Add to lookup tables */
1627   make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1628               e_key.fib_index, e_key.port, s->ext_host_port);
1629   kv.value = s - tsm->sessions;
1630   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
1631     nat_log_notice ("out2in-ed key add failed");
1632
1633   if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1634       ip->src_address.as_u32 == l_key.addr.as_u32))
1635     {
1636       eh_key.protocol = e_key.protocol;
1637       if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1638                                                thread_index, &eh_key,
1639                                                &address_index,
1640                                                sm->port_per_thread,
1641                                                tsm->snat_thread_index))
1642         {
1643           b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1644           nat44_delete_session (sm, s, thread_index);
1645           if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1646             nat_log_notice ("out2in-ed key del failed");
1647           return 0;
1648         }
1649       s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1650       s->ext_host_nat_port = eh_key.port;
1651       s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1652       make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1653                   l_key.fib_index, l_key.port, s->ext_host_nat_port);
1654     }
1655   else
1656     {
1657       make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1658                   l_key.fib_index, l_key.port, s->ext_host_port);
1659     }
1660   kv.value = s - tsm->sessions;
1661   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1662     nat_log_notice ("in2out-ed key add failed");
1663
1664   return s;
1665 }
1666
1667 static_always_inline int
1668 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1669 {
1670   icmp46_header_t *icmp0;
1671   nat_ed_ses_key_t key0;
1672   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1673   ip4_header_t *inner_ip0;
1674   void *l4_header = 0;
1675   icmp46_header_t *inner_icmp0;
1676
1677   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1678   echo0 = (icmp_echo_header_t *)(icmp0+1);
1679
1680   if (!icmp_is_error_message (icmp0))
1681     {
1682       key0.proto = IP_PROTOCOL_ICMP;
1683       key0.l_addr = ip0->dst_address;
1684       key0.r_addr = ip0->src_address;
1685       key0.l_port = echo0->identifier;
1686       key0.r_port = 0;
1687     }
1688   else
1689     {
1690       inner_ip0 = (ip4_header_t *)(echo0+1);
1691       l4_header = ip4_next_header (inner_ip0);
1692       key0.proto = inner_ip0->protocol;
1693       key0.l_addr = inner_ip0->src_address;
1694       key0.r_addr = inner_ip0->dst_address;
1695       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1696         {
1697         case SNAT_PROTOCOL_ICMP:
1698           inner_icmp0 = (icmp46_header_t*)l4_header;
1699           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1700           key0.l_port = inner_echo0->identifier;
1701           key0.r_port = 0;
1702           break;
1703         case SNAT_PROTOCOL_UDP:
1704         case SNAT_PROTOCOL_TCP:
1705           key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1706           key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1707           break;
1708         default:
1709           return -1;
1710         }
1711     }
1712   *p_key0 = key0;
1713   return 0;
1714 }
1715
1716 static int
1717 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1718               u16 dst_port, u32 thread_index)
1719 {
1720   clib_bihash_kv_16_8_t kv, value;
1721   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1722
1723   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1724               sm->inside_fib_index, src_port, dst_port);
1725   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1726     return 1;
1727
1728   return 0;
1729 }
1730
1731 static void
1732 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1733                       u32 thread_index)
1734 {
1735   nat_ed_ses_key_t key;
1736   clib_bihash_kv_16_8_t kv, value;
1737   udp_header_t *udp;
1738   snat_user_t *u;
1739   snat_session_t *s = 0;
1740   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1741   f64 now = vlib_time_now (sm->vlib_main);
1742
1743   if (ip->protocol == IP_PROTOCOL_ICMP)
1744     {
1745       if (icmp_get_ed_key (ip, &key))
1746         return;
1747     }
1748   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1749     {
1750       udp = ip4_next_header(ip);
1751       key.r_addr = ip->src_address;
1752       key.l_addr = ip->dst_address;
1753       key.proto = ip->protocol;
1754       key.l_port = udp->dst_port;
1755       key.r_port = udp->src_port;
1756     }
1757   else
1758     {
1759       key.r_addr = ip->src_address;
1760       key.l_addr = ip->dst_address;
1761       key.proto = ip->protocol;
1762       key.l_port = key.r_port = 0;
1763     }
1764   key.fib_index = 0;
1765   kv.key[0] = key.as_u64[0];
1766   kv.key[1] = key.as_u64[1];
1767
1768   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1769     {
1770       s = pool_elt_at_index (tsm->sessions, value.value);
1771     }
1772   else
1773     {
1774       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1775         return;
1776
1777       u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1778                                   thread_index);
1779       if (!u)
1780         {
1781           nat_log_warn ("create NAT user failed");
1782           return;
1783         }
1784
1785       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1786       if (!s)
1787         {
1788           nat_log_warn ("create NAT session failed");
1789           return;
1790         }
1791
1792       s->ext_host_addr = key.r_addr;
1793       s->ext_host_port = key.r_port;
1794       s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1795       s->outside_address_index = ~0;
1796       s->out2in.addr = key.l_addr;
1797       s->out2in.port = key.l_port;
1798       s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1799       s->out2in.fib_index = 0;
1800       s->in2out = s->out2in;
1801       user_session_increment (sm, u, 0);
1802
1803       kv.value = s - tsm->sessions;
1804       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1805         nat_log_notice ("in2out_ed key add failed");
1806     }
1807
1808   if (ip->protocol == IP_PROTOCOL_TCP)
1809     {
1810       tcp_header_t *tcp = ip4_next_header(ip);
1811       if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1812         return;
1813     }
1814
1815   /* Per-user LRU list maintenance */
1816   nat44_session_update_lru (sm, s, thread_index);
1817   /* Accounting */
1818   nat44_session_update_counters (s, now, 0);
1819 }
1820
1821 u32
1822 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1823                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1824                       u8 * p_proto, snat_session_key_t * p_value,
1825                       u8 * p_dont_translate, void * d, void * e)
1826 {
1827   u32 next = ~0, sw_if_index, rx_fib_index;
1828   icmp46_header_t *icmp;
1829   nat_ed_ses_key_t key;
1830   clib_bihash_kv_16_8_t kv, value;
1831   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1832   snat_session_t *s = 0;
1833   u8 dont_translate = 0, is_addr_only;
1834   snat_session_key_t e_key, l_key;
1835
1836   icmp = (icmp46_header_t *) ip4_next_header (ip);
1837   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1838   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1839
1840   if (icmp_get_ed_key (ip, &key))
1841     {
1842       b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1843       next = SNAT_OUT2IN_NEXT_DROP;
1844       goto out;
1845     }
1846   key.fib_index = rx_fib_index;
1847   kv.key[0] = key.as_u64[0];
1848   kv.key[1] = key.as_u64[1];
1849
1850   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
1851     {
1852       /* Try to match static mapping */
1853       e_key.addr = ip->dst_address;
1854       e_key.port = key.l_port;
1855       e_key.protocol = ip_proto_to_snat_proto (key.proto);
1856       e_key.fib_index = rx_fib_index;
1857       if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
1858         {
1859           if (!sm->forwarding_enabled)
1860             {
1861               /* Don't NAT packet aimed at the intfc address */
1862               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
1863                                                   ip->dst_address.as_u32)))
1864                 {
1865                   dont_translate = 1;
1866                   goto out;
1867                 }
1868               b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1869               next = NAT44_ED_OUT2IN_NEXT_DROP;
1870               goto out;
1871             }
1872           else
1873             {
1874               dont_translate = 1;
1875               if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port, thread_index))
1876                 {
1877                   next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1878                   goto out;
1879                 }
1880               create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
1881               goto out;
1882             }
1883         }
1884
1885       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1886                         (icmp->type != ICMP4_echo_request || !is_addr_only)))
1887         {
1888           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1889           next = NAT44_ED_OUT2IN_NEXT_DROP;
1890           goto out;
1891         }
1892
1893       /* Create session initiated by host from external network */
1894       s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
1895                                                thread_index, 0, 0);
1896
1897       if (!s)
1898         {
1899           next = NAT44_ED_OUT2IN_NEXT_DROP;
1900           goto out;
1901         }
1902     }
1903   else
1904     {
1905       if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1906                         icmp->type != ICMP4_echo_request &&
1907                         !icmp_is_error_message (icmp)))
1908         {
1909           b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1910           next = SNAT_OUT2IN_NEXT_DROP;
1911           goto out;
1912         }
1913
1914       s = pool_elt_at_index (tsm->sessions, value.value);
1915     }
1916
1917   *p_proto = ip_proto_to_snat_proto (key.proto);
1918 out:
1919   if (s)
1920     *p_value = s->in2out;
1921   *p_dont_translate = dont_translate;
1922   if (d)
1923     *(snat_session_t**)d = s;
1924   return next;
1925 }
1926
1927 static snat_session_t *
1928 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
1929                                vlib_buffer_t * b,
1930                                ip4_header_t * ip,
1931                                u32 rx_fib_index,
1932                                u32 thread_index,
1933                                f64 now,
1934                                vlib_main_t * vm,
1935                                vlib_node_runtime_t * node)
1936 {
1937   clib_bihash_kv_8_8_t kv, value;
1938   clib_bihash_kv_16_8_t s_kv, s_value;
1939   snat_static_mapping_t *m;
1940   u32 old_addr, new_addr;
1941   ip_csum_t sum;
1942   snat_session_t * s;
1943   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1944   snat_user_t *u;
1945
1946   old_addr = ip->dst_address.as_u32;
1947
1948   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
1949               rx_fib_index, 0, 0);
1950
1951   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
1952     {
1953       s = pool_elt_at_index (tsm->sessions, s_value.value);
1954       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1955     }
1956   else
1957     {
1958       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1959         {
1960           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1961           nat_log_notice ("maximum sessions exceeded");
1962           return 0;
1963         }
1964
1965       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
1966       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1967         {
1968           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1969           return 0;
1970         }
1971
1972       m = pool_elt_at_index (sm->static_mappings, value.value);
1973
1974       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1975
1976       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
1977                                   thread_index);
1978       if (!u)
1979         {
1980           nat_log_warn ("create NAT user failed");
1981           return 0;
1982         }
1983
1984       /* Create a new session */
1985       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1986       if (!s)
1987         {
1988           nat_log_warn ("create NAT session failed");
1989           return 0;
1990         }
1991
1992       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1993       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1994       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1995       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1996       s->outside_address_index = ~0;
1997       s->out2in.addr.as_u32 = old_addr;
1998       s->out2in.fib_index = rx_fib_index;
1999       s->in2out.addr.as_u32 = new_addr;
2000       s->in2out.fib_index = m->fib_index;
2001       s->in2out.port = s->out2in.port = ip->protocol;
2002       user_session_increment (sm, u, 1);
2003
2004       /* Add to lookup tables */
2005       s_kv.value = s - tsm->sessions;
2006       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2007         nat_log_notice ("out2in key add failed");
2008
2009       make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2010                   m->fib_index, 0, 0);
2011       s_kv.value = s - tsm->sessions;
2012       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2013         nat_log_notice ("in2out key add failed");
2014    }
2015
2016   /* Update IP checksum */
2017   sum = ip->checksum;
2018   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2019   ip->checksum = ip_csum_fold (sum);
2020
2021   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2022
2023   /* Accounting */
2024   nat44_session_update_counters (s, now,
2025                                  vlib_buffer_length_in_chain (vm, b));
2026   /* Per-user LRU list maintenance */
2027   nat44_session_update_lru (sm, s, thread_index);
2028
2029   return s;
2030 }
2031
2032 static inline uword
2033 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2034                                 vlib_node_runtime_t * node,
2035                                 vlib_frame_t * frame, int is_slow_path)
2036 {
2037   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2038   nat44_ed_out2in_next_t next_index;
2039   snat_main_t *sm = &snat_main;
2040   f64 now = vlib_time_now (vm);
2041   u32 thread_index = vlib_get_thread_index ();
2042   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2043
2044   stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2045     nat44_ed_out2in_node.index;
2046
2047   from = vlib_frame_vector_args (frame);
2048   n_left_from = frame->n_vectors;
2049   next_index = node->cached_next_index;
2050
2051   while (n_left_from > 0)
2052     {
2053       u32 n_left_to_next;
2054
2055       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2056
2057       while (n_left_from >= 4 && n_left_to_next >= 2)
2058         {
2059           u32 bi0, bi1;
2060           vlib_buffer_t *b0, *b1;
2061           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2062           u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2063           u16 old_port0, new_port0, old_port1, new_port1;
2064           ip4_header_t *ip0, *ip1;
2065           udp_header_t *udp0, *udp1;
2066           tcp_header_t *tcp0, *tcp1;
2067           icmp46_header_t *icmp0, *icmp1;
2068           snat_session_t *s0 = 0, *s1 = 0;
2069           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2070           ip_csum_t sum0, sum1;
2071           snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2072           u8 is_lb0, is_lb1;
2073           twice_nat_type_t twice_nat0, twice_nat1;
2074
2075           /* Prefetch next iteration. */
2076           {
2077             vlib_buffer_t * p2, * p3;
2078
2079             p2 = vlib_get_buffer (vm, from[2]);
2080             p3 = vlib_get_buffer (vm, from[3]);
2081
2082             vlib_prefetch_buffer_header (p2, LOAD);
2083             vlib_prefetch_buffer_header (p3, LOAD);
2084
2085             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2086             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2087           }
2088
2089           /* speculatively enqueue b0 and b1 to the current next frame */
2090           to_next[0] = bi0 = from[0];
2091           to_next[1] = bi1 = from[1];
2092           from += 2;
2093           to_next += 2;
2094           n_left_from -= 2;
2095           n_left_to_next -= 2;
2096
2097           b0 = vlib_get_buffer (vm, bi0);
2098           b1 = vlib_get_buffer (vm, bi1);
2099
2100           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2101           vnet_buffer (b0)->snat.flags = 0;
2102           ip0 = vlib_buffer_get_current (b0);
2103
2104           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2105           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2106                                                                sw_if_index0);
2107
2108           if (PREDICT_FALSE(ip0->ttl == 1))
2109             {
2110               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2111               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2112                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2113                                            0);
2114               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2115               goto trace00;
2116             }
2117
2118           udp0 = ip4_next_header (ip0);
2119           tcp0 = (tcp_header_t *) udp0;
2120           icmp0 = (icmp46_header_t *) udp0;
2121           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2122
2123           if (is_slow_path)
2124             {
2125               if (PREDICT_FALSE (proto0 == ~0))
2126                 {
2127                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2128                                                      thread_index, now, vm, node);
2129                   if (!sm->forwarding_enabled)
2130                     {
2131                       if (!s0)
2132                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2133                       goto trace00;
2134                     }
2135                 }
2136
2137               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2138                 {
2139                   next0 = icmp_out2in_slow_path
2140                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2141                      next0, now, thread_index, &s0);
2142                   goto trace00;
2143                 }
2144             }
2145           else
2146             {
2147               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2148                 {
2149                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2150                   goto trace00;
2151                 }
2152
2153               if (ip4_is_fragment (ip0))
2154                 {
2155                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2156                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2157                   goto trace00;
2158                 }
2159             }
2160
2161           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2162                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2163
2164           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2165             {
2166               if (is_slow_path)
2167                 {
2168                   /* Try to match static mapping by external address and port,
2169                      destination address and port in packet */
2170                   e_key0.addr = ip0->dst_address;
2171                   e_key0.port = udp0->dst_port;
2172                   e_key0.protocol = proto0;
2173                   e_key0.fib_index = rx_fib_index0;
2174                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2175                       &twice_nat0, &is_lb0))
2176                     {
2177                       /*
2178                        * Send DHCP packets to the ipv4 stack, or we won't
2179                        * be able to use dhcp client on the outside interface
2180                        */
2181                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2182                           && (udp0->dst_port ==
2183                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2184                         {
2185                           vnet_feature_next
2186                             (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
2187                           goto trace00;
2188                         }
2189
2190                       if (!sm->forwarding_enabled)
2191                         {
2192                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2193                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2194                         }
2195                       else
2196                         {
2197                           if (next_src_nat(sm, ip0, ip0->protocol,
2198                                            udp0->src_port, udp0->dst_port,
2199                                            thread_index))
2200                             {
2201                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2202                               goto trace00;
2203                             }
2204                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2205                                                 thread_index);
2206                         }
2207                       goto trace00;
2208                     }
2209
2210                   /* Create session initiated by host from external network */
2211                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2212                                                             e_key0, node,
2213                                                             thread_index,
2214                                                             twice_nat0, is_lb0);
2215
2216                   if (!s0)
2217                     {
2218                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2219                       goto trace00;
2220                     }
2221                 }
2222               else
2223                 {
2224                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2225                   goto trace00;
2226                 }
2227             }
2228           else
2229             {
2230               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2231             }
2232
2233           old_addr0 = ip0->dst_address.as_u32;
2234           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2235           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2236
2237           sum0 = ip0->checksum;
2238           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2239                                  dst_address);
2240           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2241             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2242                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2243                                    src_address);
2244           ip0->checksum = ip_csum_fold (sum0);
2245
2246           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2247             {
2248               old_port0 = tcp0->dst_port;
2249               new_port0 = tcp0->dst_port = s0->in2out.port;
2250
2251               sum0 = tcp0->checksum;
2252               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2253                                      dst_address);
2254               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2255                                      length);
2256               if (is_twice_nat_session (s0))
2257                 {
2258                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2259                                          s0->ext_host_nat_addr.as_u32,
2260                                          ip4_header_t, dst_address);
2261                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2262                                          s0->ext_host_nat_port, ip4_header_t,
2263                                          length);
2264                   tcp0->src_port = s0->ext_host_nat_port;
2265                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2266                 }
2267               tcp0->checksum = ip_csum_fold(sum0);
2268               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2269                 goto trace00;
2270             }
2271           else
2272             {
2273               udp0->dst_port = s0->in2out.port;
2274               if (is_twice_nat_session (s0))
2275                 {
2276                   udp0->src_port = s0->ext_host_nat_port;
2277                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2278                 }
2279               udp0->checksum = 0;
2280             }
2281
2282           /* Accounting */
2283           nat44_session_update_counters (s0, now,
2284                                          vlib_buffer_length_in_chain (vm, b0));
2285           /* Per-user LRU list maintenance */
2286           nat44_session_update_lru (sm, s0, thread_index);
2287
2288         trace00:
2289           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2290                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2291             {
2292               nat44_ed_out2in_trace_t *t =
2293                 vlib_add_trace (vm, node, b0, sizeof (*t));
2294               t->is_slow_path = is_slow_path;
2295               t->sw_if_index = sw_if_index0;
2296               t->next_index = next0;
2297               t->session_index = ~0;
2298               if (s0)
2299                 t->session_index = s0 - tsm->sessions;
2300             }
2301
2302           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2303
2304           next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2305           vnet_buffer (b1)->snat.flags = 0;
2306           ip1 = vlib_buffer_get_current (b1);
2307
2308           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2309           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2310                                                                sw_if_index1);
2311
2312           if (PREDICT_FALSE(ip1->ttl == 1))
2313             {
2314               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2315               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2316                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2317                                            0);
2318               next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2319               goto trace01;
2320             }
2321
2322           udp1 = ip4_next_header (ip1);
2323           tcp1 = (tcp_header_t *) udp1;
2324           icmp1 = (icmp46_header_t *) udp1;
2325           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2326
2327           if (is_slow_path)
2328             {
2329               if (PREDICT_FALSE (proto1 == ~0))
2330                 {
2331                   s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2332                                                      thread_index, now, vm, node);
2333                   if (!sm->forwarding_enabled)
2334                     {
2335                       if (!s1)
2336                         next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2337                       goto trace01;
2338                     }
2339                 }
2340
2341               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2342                 {
2343                   next1 = icmp_out2in_slow_path
2344                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2345                      next1, now, thread_index, &s1);
2346                   goto trace01;
2347                 }
2348             }
2349           else
2350             {
2351               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2352                 {
2353                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2354                   goto trace01;
2355                 }
2356
2357               if (ip4_is_fragment (ip1))
2358                 {
2359                   b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2360                   next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2361                   goto trace01;
2362                 }
2363             }
2364
2365           make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2366                       rx_fib_index1, udp1->dst_port, udp1->src_port);
2367
2368           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2369             {
2370               if (is_slow_path)
2371                 {
2372                   /* Try to match static mapping by external address and port,
2373                      destination address and port in packet */
2374                   e_key1.addr = ip1->dst_address;
2375                   e_key1.port = udp1->dst_port;
2376                   e_key1.protocol = proto1;
2377                   e_key1.fib_index = rx_fib_index1;
2378                   if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2379                       &twice_nat1, &is_lb1))
2380                     {
2381                       /*
2382                        * Send DHCP packets to the ipv4 stack, or we won't
2383                        * be able to use dhcp client on the outside interface
2384                        */
2385                       if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2386                           && (udp1->dst_port ==
2387                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2388                         {
2389                           vnet_feature_next
2390                             (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
2391                           goto trace01;
2392                         }
2393
2394                       if (!sm->forwarding_enabled)
2395                         {
2396                           b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2397                           next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2398                         }
2399                       else
2400                         {
2401                           if (next_src_nat(sm, ip1, ip1->protocol,
2402                                            udp1->src_port, udp1->dst_port,
2403                                            thread_index))
2404                             {
2405                               next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2406                               goto trace01;
2407                             }
2408                           create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2409                                                 thread_index);
2410                         }
2411                       goto trace01;
2412                     }
2413
2414                   /* Create session initiated by host from external network */
2415                   s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2416                                                             e_key1, node,
2417                                                             thread_index,
2418                                                             twice_nat1, is_lb1);
2419
2420                   if (!s1)
2421                     {
2422                       next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2423                       goto trace01;
2424                     }
2425                 }
2426               else
2427                 {
2428                   next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2429                   goto trace01;
2430                 }
2431             }
2432           else
2433             {
2434               s1 = pool_elt_at_index (tsm->sessions, value1.value);
2435             }
2436
2437           old_addr1 = ip1->dst_address.as_u32;
2438           new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2439           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2440
2441           sum1 = ip1->checksum;
2442           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2443                                  dst_address);
2444           if (PREDICT_FALSE (is_twice_nat_session (s1)))
2445             sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2446                                    s1->ext_host_nat_addr.as_u32, ip4_header_t,
2447                                    src_address);
2448           ip1->checksum = ip_csum_fold (sum1);
2449
2450           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2451             {
2452               old_port1 = tcp1->dst_port;
2453               new_port1 = tcp1->dst_port = s1->in2out.port;
2454
2455               sum1 = tcp1->checksum;
2456               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2457                                      dst_address);
2458               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2459                                      length);
2460               if (is_twice_nat_session (s1))
2461                 {
2462                   sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2463                                          s1->ext_host_nat_addr.as_u32,
2464                                          ip4_header_t, dst_address);
2465                   sum1 = ip_csum_update (sum1, tcp1->src_port,
2466                                          s1->ext_host_nat_port, ip4_header_t,
2467                                          length);
2468                   tcp1->src_port = s1->ext_host_nat_port;
2469                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2470                 }
2471               tcp1->checksum = ip_csum_fold(sum1);
2472               if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2473                 goto trace01;
2474             }
2475           else
2476             {
2477               udp1->dst_port = s1->in2out.port;
2478               if (is_twice_nat_session (s1))
2479                 {
2480                   udp1->src_port = s1->ext_host_nat_port;
2481                   ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2482                 }
2483               udp1->checksum = 0;
2484             }
2485
2486           /* Accounting */
2487           nat44_session_update_counters (s1, now,
2488                                          vlib_buffer_length_in_chain (vm, b1));
2489           /* Per-user LRU list maintenance */
2490           nat44_session_update_lru (sm, s1, thread_index);
2491
2492         trace01:
2493           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2494                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2495             {
2496               nat44_ed_out2in_trace_t *t =
2497                 vlib_add_trace (vm, node, b1, sizeof (*t));
2498               t->is_slow_path = is_slow_path;
2499               t->sw_if_index = sw_if_index1;
2500               t->next_index = next1;
2501               t->session_index = ~0;
2502               if (s1)
2503                 t->session_index = s1 - tsm->sessions;
2504             }
2505
2506           pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2507
2508           /* verify speculative enqueues, maybe switch current next frame */
2509           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2510                                            to_next, n_left_to_next,
2511                                            bi0, bi1, next0, next1);
2512         }
2513
2514       while (n_left_from > 0 && n_left_to_next > 0)
2515         {
2516           u32 bi0;
2517           vlib_buffer_t *b0;
2518           u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2519           u16 old_port0, new_port0;
2520           ip4_header_t *ip0;
2521           udp_header_t *udp0;
2522           tcp_header_t *tcp0;
2523           icmp46_header_t * icmp0;
2524           snat_session_t *s0 = 0;
2525           clib_bihash_kv_16_8_t kv0, value0;
2526           ip_csum_t sum0;
2527           snat_session_key_t e_key0, l_key0;
2528           u8 is_lb0;
2529           twice_nat_type_t twice_nat0;
2530
2531           /* speculatively enqueue b0 to the current next frame */
2532           bi0 = from[0];
2533           to_next[0] = bi0;
2534           from += 1;
2535           to_next += 1;
2536           n_left_from -= 1;
2537           n_left_to_next -= 1;
2538
2539           b0 = vlib_get_buffer (vm, bi0);
2540           next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2541           vnet_buffer (b0)->snat.flags = 0;
2542           ip0 = vlib_buffer_get_current (b0);
2543
2544           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2545           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2546                                                                sw_if_index0);
2547
2548           if (PREDICT_FALSE(ip0->ttl == 1))
2549             {
2550               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2551               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2552                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2553                                            0);
2554               next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2555               goto trace0;
2556             }
2557
2558           udp0 = ip4_next_header (ip0);
2559           tcp0 = (tcp_header_t *) udp0;
2560           icmp0 = (icmp46_header_t *) udp0;
2561           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2562
2563           if (is_slow_path)
2564             {
2565               if (PREDICT_FALSE (proto0 == ~0))
2566                 {
2567                   s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2568                                                      thread_index, now, vm, node);
2569                   if (!sm->forwarding_enabled)
2570                     {
2571                       if (!s0)
2572                         next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2573                       goto trace0;
2574                     }
2575                 }
2576
2577               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2578                 {
2579                   next0 = icmp_out2in_slow_path
2580                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2581                      next0, now, thread_index, &s0);
2582                   goto trace0;
2583                 }
2584             }
2585           else
2586             {
2587               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2588                 {
2589                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2590                   goto trace0;
2591                 }
2592
2593               if (ip4_is_fragment (ip0))
2594                 {
2595                   b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2596                   next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2597                   goto trace0;
2598                 }
2599             }
2600
2601           make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2602                       rx_fib_index0, udp0->dst_port, udp0->src_port);
2603
2604           if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2605             {
2606               if (is_slow_path)
2607                 {
2608                   /* Try to match static mapping by external address and port,
2609                      destination address and port in packet */
2610                   e_key0.addr = ip0->dst_address;
2611                   e_key0.port = udp0->dst_port;
2612                   e_key0.protocol = proto0;
2613                   e_key0.fib_index = rx_fib_index0;
2614                   if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2615                       &twice_nat0, &is_lb0))
2616                     {
2617                       /*
2618                        * Send DHCP packets to the ipv4 stack, or we won't
2619                        * be able to use dhcp client on the outside interface
2620                        */
2621                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2622                           && (udp0->dst_port ==
2623                           clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2624                         {
2625                           vnet_feature_next
2626                             (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
2627                           goto trace0;
2628                         }
2629
2630                       if (!sm->forwarding_enabled)
2631                         {
2632                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2633                           next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2634                         }
2635                       else
2636                         {
2637                           if (next_src_nat(sm, ip0, ip0->protocol,
2638                                            udp0->src_port, udp0->dst_port,
2639                                            thread_index))
2640                             {
2641                               next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2642                               goto trace0;
2643                             }
2644                           create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2645                                                 thread_index);
2646                         }
2647                       goto trace0;
2648                     }
2649
2650                   /* Create session initiated by host from external network */
2651                   s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2652                                                             e_key0, node,
2653                                                             thread_index,
2654                                                             twice_nat0, is_lb0);
2655
2656                   if (!s0)
2657                     {
2658                       next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2659                       goto trace0;
2660                     }
2661                 }
2662               else
2663                 {
2664                   next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2665                   goto trace0;
2666                 }
2667             }
2668           else
2669             {
2670               s0 = pool_elt_at_index (tsm->sessions, value0.value);
2671             }
2672
2673           old_addr0 = ip0->dst_address.as_u32;
2674           new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2675           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2676
2677           sum0 = ip0->checksum;
2678           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2679                                  dst_address);
2680           if (PREDICT_FALSE (is_twice_nat_session (s0)))
2681             sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2682                                    s0->ext_host_nat_addr.as_u32, ip4_header_t,
2683                                    src_address);
2684           ip0->checksum = ip_csum_fold (sum0);
2685
2686           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2687             {
2688               old_port0 = tcp0->dst_port;
2689               new_port0 = tcp0->dst_port = s0->in2out.port;
2690
2691               sum0 = tcp0->checksum;
2692               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2693                                      dst_address);
2694               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2695                                      length);
2696               if (is_twice_nat_session (s0))
2697                 {
2698                   sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2699                                          s0->ext_host_nat_addr.as_u32,
2700                                          ip4_header_t, dst_address);
2701                   sum0 = ip_csum_update (sum0, tcp0->src_port,
2702                                          s0->ext_host_nat_port, ip4_header_t,
2703                                          length);
2704                   tcp0->src_port = s0->ext_host_nat_port;
2705                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2706                 }
2707               tcp0->checksum = ip_csum_fold(sum0);
2708               if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2709                 goto trace0;
2710             }
2711           else
2712             {
2713               udp0->dst_port = s0->in2out.port;
2714               if (is_twice_nat_session (s0))
2715                 {
2716                   udp0->src_port = s0->ext_host_nat_port;
2717                   ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2718                 }
2719               udp0->checksum = 0;
2720             }
2721
2722           /* Accounting */
2723           nat44_session_update_counters (s0, now,
2724                                          vlib_buffer_length_in_chain (vm, b0));
2725           /* Per-user LRU list maintenance */
2726           nat44_session_update_lru (sm, s0, thread_index);
2727
2728         trace0:
2729           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2730                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2731             {
2732               nat44_ed_out2in_trace_t *t =
2733                 vlib_add_trace (vm, node, b0, sizeof (*t));
2734               t->is_slow_path = is_slow_path;
2735               t->sw_if_index = sw_if_index0;
2736               t->next_index = next0;
2737               t->session_index = ~0;
2738               if (s0)
2739                 t->session_index = s0 - tsm->sessions;
2740             }
2741
2742           pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2743
2744           /* verify speculative enqueue, maybe switch current next frame */
2745           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2746                                            to_next, n_left_to_next,
2747                                            bi0, next0);
2748         }
2749
2750       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2751     }
2752
2753   vlib_node_increment_counter (vm, stats_node_index,
2754                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2755                                pkts_processed);
2756   return frame->n_vectors;
2757 }
2758
2759 static uword
2760 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2761                               vlib_node_runtime_t * node,
2762                               vlib_frame_t * frame)
2763 {
2764   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2765 }
2766
2767 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2768   .function = nat44_ed_out2in_fast_path_fn,
2769   .name = "nat44-ed-out2in",
2770   .vector_size = sizeof (u32),
2771   .format_trace = format_nat44_ed_out2in_trace,
2772   .type = VLIB_NODE_TYPE_INTERNAL,
2773
2774   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2775   .error_strings = snat_out2in_error_strings,
2776
2777   .runtime_data_bytes = sizeof (snat_runtime_t),
2778
2779   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2780
2781   /* edit / add dispositions here */
2782   .next_nodes = {
2783     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2784     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2785     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2786     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2787     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2788   },
2789 };
2790
2791 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2792
2793 static uword
2794 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2795                               vlib_node_runtime_t * node,
2796                               vlib_frame_t * frame)
2797 {
2798   return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2799 }
2800
2801 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2802   .function = nat44_ed_out2in_slow_path_fn,
2803   .name = "nat44-ed-out2in-slowpath",
2804   .vector_size = sizeof (u32),
2805   .format_trace = format_nat44_ed_out2in_trace,
2806   .type = VLIB_NODE_TYPE_INTERNAL,
2807
2808   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2809   .error_strings = snat_out2in_error_strings,
2810
2811   .runtime_data_bytes = sizeof (snat_runtime_t),
2812
2813   .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2814
2815   /* edit / add dispositions here */
2816   .next_nodes = {
2817     [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2818     [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2819     [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2820     [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2821     [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2822   },
2823 };
2824
2825 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2826                               nat44_ed_out2in_slow_path_fn);
2827
2828 /**************************/
2829 /*** deterministic mode ***/
2830 /**************************/
2831 static uword
2832 snat_det_out2in_node_fn (vlib_main_t * vm,
2833                          vlib_node_runtime_t * node,
2834                          vlib_frame_t * frame)
2835 {
2836   u32 n_left_from, * from, * to_next;
2837   snat_out2in_next_t next_index;
2838   u32 pkts_processed = 0;
2839   snat_main_t * sm = &snat_main;
2840   u32 thread_index = vlib_get_thread_index ();
2841
2842   from = vlib_frame_vector_args (frame);
2843   n_left_from = frame->n_vectors;
2844   next_index = node->cached_next_index;
2845
2846   while (n_left_from > 0)
2847     {
2848       u32 n_left_to_next;
2849
2850       vlib_get_next_frame (vm, node, next_index,
2851                            to_next, n_left_to_next);
2852
2853       while (n_left_from >= 4 && n_left_to_next >= 2)
2854         {
2855           u32 bi0, bi1;
2856           vlib_buffer_t * b0, * b1;
2857           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2858           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2859           u32 sw_if_index0, sw_if_index1;
2860           ip4_header_t * ip0, * ip1;
2861           ip_csum_t sum0, sum1;
2862           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2863           u16 new_port0, old_port0, old_port1, new_port1;
2864           udp_header_t * udp0, * udp1;
2865           tcp_header_t * tcp0, * tcp1;
2866           u32 proto0, proto1;
2867           snat_det_out_key_t key0, key1;
2868           snat_det_map_t * dm0, * dm1;
2869           snat_det_session_t * ses0 = 0, * ses1 = 0;
2870           u32 rx_fib_index0, rx_fib_index1;
2871           icmp46_header_t * icmp0, * icmp1;
2872
2873           /* Prefetch next iteration. */
2874           {
2875             vlib_buffer_t * p2, * p3;
2876
2877             p2 = vlib_get_buffer (vm, from[2]);
2878             p3 = vlib_get_buffer (vm, from[3]);
2879
2880             vlib_prefetch_buffer_header (p2, LOAD);
2881             vlib_prefetch_buffer_header (p3, LOAD);
2882
2883             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2884             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2885           }
2886
2887           /* speculatively enqueue b0 and b1 to the current next frame */
2888           to_next[0] = bi0 = from[0];
2889           to_next[1] = bi1 = from[1];
2890           from += 2;
2891           to_next += 2;
2892           n_left_from -= 2;
2893           n_left_to_next -= 2;
2894
2895           b0 = vlib_get_buffer (vm, bi0);
2896           b1 = vlib_get_buffer (vm, bi1);
2897
2898           ip0 = vlib_buffer_get_current (b0);
2899           udp0 = ip4_next_header (ip0);
2900           tcp0 = (tcp_header_t *) udp0;
2901
2902           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2903
2904           if (PREDICT_FALSE(ip0->ttl == 1))
2905             {
2906               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2907               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2908                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2909                                            0);
2910               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2911               goto trace0;
2912             }
2913
2914           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2915
2916           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2917             {
2918               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2919               icmp0 = (icmp46_header_t *) udp0;
2920
2921               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2922                                   rx_fib_index0, node, next0, thread_index,
2923                                   &ses0, &dm0);
2924               goto trace0;
2925             }
2926
2927           key0.ext_host_addr = ip0->src_address;
2928           key0.ext_host_port = tcp0->src;
2929           key0.out_port = tcp0->dst;
2930
2931           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2932           if (PREDICT_FALSE(!dm0))
2933             {
2934               nat_log_info ("unknown dst address:  %U",
2935                             format_ip4_address, &ip0->dst_address);
2936               next0 = SNAT_OUT2IN_NEXT_DROP;
2937               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2938               goto trace0;
2939             }
2940
2941           snat_det_reverse(dm0, &ip0->dst_address,
2942                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2943
2944           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2945           if (PREDICT_FALSE(!ses0))
2946             {
2947               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2948                             format_ip4_address, &ip0->src_address,
2949                             clib_net_to_host_u16 (tcp0->src),
2950                             format_ip4_address, &ip0->dst_address,
2951                             clib_net_to_host_u16 (tcp0->dst),
2952                             format_ip4_address, &new_addr0);
2953               next0 = SNAT_OUT2IN_NEXT_DROP;
2954               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2955               goto trace0;
2956             }
2957           new_port0 = ses0->in_port;
2958
2959           old_addr0 = ip0->dst_address;
2960           ip0->dst_address = new_addr0;
2961           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2962
2963           sum0 = ip0->checksum;
2964           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2965                                  ip4_header_t,
2966                                  dst_address /* changed member */);
2967           ip0->checksum = ip_csum_fold (sum0);
2968
2969           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2970             {
2971               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2972                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2973               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2974                 snat_det_ses_close(dm0, ses0);
2975
2976               old_port0 = tcp0->dst;
2977               tcp0->dst = new_port0;
2978
2979               sum0 = tcp0->checksum;
2980               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2981                                      ip4_header_t,
2982                                      dst_address /* changed member */);
2983
2984               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2985                                      ip4_header_t /* cheat */,
2986                                      length /* changed member */);
2987               tcp0->checksum = ip_csum_fold(sum0);
2988             }
2989           else
2990             {
2991               old_port0 = udp0->dst_port;
2992               udp0->dst_port = new_port0;
2993               udp0->checksum = 0;
2994             }
2995
2996         trace0:
2997
2998           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2999                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3000             {
3001               snat_out2in_trace_t *t =
3002                  vlib_add_trace (vm, node, b0, sizeof (*t));
3003               t->sw_if_index = sw_if_index0;
3004               t->next_index = next0;
3005               t->session_index = ~0;
3006               if (ses0)
3007                 t->session_index = ses0 - dm0->sessions;
3008             }
3009
3010           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3011
3012           b1 = vlib_get_buffer (vm, bi1);
3013
3014           ip1 = vlib_buffer_get_current (b1);
3015           udp1 = ip4_next_header (ip1);
3016           tcp1 = (tcp_header_t *) udp1;
3017
3018           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3019
3020           if (PREDICT_FALSE(ip1->ttl == 1))
3021             {
3022               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3023               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3024                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3025                                            0);
3026               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3027               goto trace1;
3028             }
3029
3030           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3031
3032           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3033             {
3034               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3035               icmp1 = (icmp46_header_t *) udp1;
3036
3037               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3038                                   rx_fib_index1, node, next1, thread_index,
3039                                   &ses1, &dm1);
3040               goto trace1;
3041             }
3042
3043           key1.ext_host_addr = ip1->src_address;
3044           key1.ext_host_port = tcp1->src;
3045           key1.out_port = tcp1->dst;
3046
3047           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3048           if (PREDICT_FALSE(!dm1))
3049             {
3050               nat_log_info ("unknown dst address:  %U",
3051                             format_ip4_address, &ip1->dst_address);
3052               next1 = SNAT_OUT2IN_NEXT_DROP;
3053               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3054               goto trace1;
3055             }
3056
3057           snat_det_reverse(dm1, &ip1->dst_address,
3058                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
3059
3060           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3061           if (PREDICT_FALSE(!ses1))
3062             {
3063               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3064                             format_ip4_address, &ip1->src_address,
3065                             clib_net_to_host_u16 (tcp1->src),
3066                             format_ip4_address, &ip1->dst_address,
3067                             clib_net_to_host_u16 (tcp1->dst),
3068                             format_ip4_address, &new_addr1);
3069               next1 = SNAT_OUT2IN_NEXT_DROP;
3070               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3071               goto trace1;
3072             }
3073           new_port1 = ses1->in_port;
3074
3075           old_addr1 = ip1->dst_address;
3076           ip1->dst_address = new_addr1;
3077           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3078
3079           sum1 = ip1->checksum;
3080           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3081                                  ip4_header_t,
3082                                  dst_address /* changed member */);
3083           ip1->checksum = ip_csum_fold (sum1);
3084
3085           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3086             {
3087               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3088                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3089               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3090                 snat_det_ses_close(dm1, ses1);
3091
3092               old_port1 = tcp1->dst;
3093               tcp1->dst = new_port1;
3094
3095               sum1 = tcp1->checksum;
3096               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3097                                      ip4_header_t,
3098                                      dst_address /* changed member */);
3099
3100               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3101                                      ip4_header_t /* cheat */,
3102                                      length /* changed member */);
3103               tcp1->checksum = ip_csum_fold(sum1);
3104             }
3105           else
3106             {
3107               old_port1 = udp1->dst_port;
3108               udp1->dst_port = new_port1;
3109               udp1->checksum = 0;
3110             }
3111
3112         trace1:
3113
3114           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3115                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3116             {
3117               snat_out2in_trace_t *t =
3118                  vlib_add_trace (vm, node, b1, sizeof (*t));
3119               t->sw_if_index = sw_if_index1;
3120               t->next_index = next1;
3121               t->session_index = ~0;
3122               if (ses1)
3123                 t->session_index = ses1 - dm1->sessions;
3124             }
3125
3126           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3127
3128           /* verify speculative enqueues, maybe switch current next frame */
3129           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3130                                            to_next, n_left_to_next,
3131                                            bi0, bi1, next0, next1);
3132          }
3133
3134       while (n_left_from > 0 && n_left_to_next > 0)
3135         {
3136           u32 bi0;
3137           vlib_buffer_t * b0;
3138           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3139           u32 sw_if_index0;
3140           ip4_header_t * ip0;
3141           ip_csum_t sum0;
3142           ip4_address_t new_addr0, old_addr0;
3143           u16 new_port0, old_port0;
3144           udp_header_t * udp0;
3145           tcp_header_t * tcp0;
3146           u32 proto0;
3147           snat_det_out_key_t key0;
3148           snat_det_map_t * dm0;
3149           snat_det_session_t * ses0 = 0;
3150           u32 rx_fib_index0;
3151           icmp46_header_t * icmp0;
3152
3153           /* speculatively enqueue b0 to the current next frame */
3154           bi0 = from[0];
3155           to_next[0] = bi0;
3156           from += 1;
3157           to_next += 1;
3158           n_left_from -= 1;
3159           n_left_to_next -= 1;
3160
3161           b0 = vlib_get_buffer (vm, bi0);
3162
3163           ip0 = vlib_buffer_get_current (b0);
3164           udp0 = ip4_next_header (ip0);
3165           tcp0 = (tcp_header_t *) udp0;
3166
3167           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3168
3169           if (PREDICT_FALSE(ip0->ttl == 1))
3170             {
3171               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3172               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3173                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3174                                            0);
3175               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3176               goto trace00;
3177             }
3178
3179           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3180
3181           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3182             {
3183               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3184               icmp0 = (icmp46_header_t *) udp0;
3185
3186               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3187                                   rx_fib_index0, node, next0, thread_index,
3188                                   &ses0, &dm0);
3189               goto trace00;
3190             }
3191
3192           key0.ext_host_addr = ip0->src_address;
3193           key0.ext_host_port = tcp0->src;
3194           key0.out_port = tcp0->dst;
3195
3196           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3197           if (PREDICT_FALSE(!dm0))
3198             {
3199               nat_log_info ("unknown dst address:  %U",
3200                             format_ip4_address, &ip0->dst_address);
3201               next0 = SNAT_OUT2IN_NEXT_DROP;
3202               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3203               goto trace00;
3204             }
3205
3206           snat_det_reverse(dm0, &ip0->dst_address,
3207                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
3208
3209           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3210           if (PREDICT_FALSE(!ses0))
3211             {
3212               nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3213                             format_ip4_address, &ip0->src_address,
3214                             clib_net_to_host_u16 (tcp0->src),
3215                             format_ip4_address, &ip0->dst_address,
3216                             clib_net_to_host_u16 (tcp0->dst),
3217                             format_ip4_address, &new_addr0);
3218               next0 = SNAT_OUT2IN_NEXT_DROP;
3219               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3220               goto trace00;
3221             }
3222           new_port0 = ses0->in_port;
3223
3224           old_addr0 = ip0->dst_address;
3225           ip0->dst_address = new_addr0;
3226           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3227
3228           sum0 = ip0->checksum;
3229           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3230                                  ip4_header_t,
3231                                  dst_address /* changed member */);
3232           ip0->checksum = ip_csum_fold (sum0);
3233
3234           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3235             {
3236               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3237                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3238               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3239                 snat_det_ses_close(dm0, ses0);
3240
3241               old_port0 = tcp0->dst;
3242               tcp0->dst = new_port0;
3243
3244               sum0 = tcp0->checksum;
3245               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3246                                      ip4_header_t,
3247                                      dst_address /* changed member */);
3248
3249               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3250                                      ip4_header_t /* cheat */,
3251                                      length /* changed member */);
3252               tcp0->checksum = ip_csum_fold(sum0);
3253             }
3254           else
3255             {
3256               old_port0 = udp0->dst_port;
3257               udp0->dst_port = new_port0;
3258               udp0->checksum = 0;
3259             }
3260
3261         trace00:
3262
3263           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3264                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3265             {
3266               snat_out2in_trace_t *t =
3267                  vlib_add_trace (vm, node, b0, sizeof (*t));
3268               t->sw_if_index = sw_if_index0;
3269               t->next_index = next0;
3270               t->session_index = ~0;
3271               if (ses0)
3272                 t->session_index = ses0 - dm0->sessions;
3273             }
3274
3275           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3276
3277           /* verify speculative enqueue, maybe switch current next frame */
3278           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3279                                            to_next, n_left_to_next,
3280                                            bi0, next0);
3281         }
3282
3283       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3284     }
3285
3286   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3287                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3288                                pkts_processed);
3289   return frame->n_vectors;
3290 }
3291
3292 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3293   .function = snat_det_out2in_node_fn,
3294   .name = "nat44-det-out2in",
3295   .vector_size = sizeof (u32),
3296   .format_trace = format_snat_out2in_trace,
3297   .type = VLIB_NODE_TYPE_INTERNAL,
3298
3299   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3300   .error_strings = snat_out2in_error_strings,
3301
3302   .runtime_data_bytes = sizeof (snat_runtime_t),
3303
3304   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3305
3306   /* edit / add dispositions here */
3307   .next_nodes = {
3308     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3309     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3310     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3311     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3312   },
3313 };
3314 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3315
3316 /**
3317  * Get address and port values to be used for ICMP packet translation
3318  * and create session if needed
3319  *
3320  * @param[in,out] sm             NAT main
3321  * @param[in,out] node           NAT node runtime
3322  * @param[in] thread_index       thread index
3323  * @param[in,out] b0             buffer containing packet to be translated
3324  * @param[out] p_proto           protocol used for matching
3325  * @param[out] p_value           address and port after NAT translation
3326  * @param[out] p_dont_translate  if packet should not be translated
3327  * @param d                      optional parameter
3328  * @param e                      optional parameter
3329  */
3330 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3331                           u32 thread_index, vlib_buffer_t *b0,
3332                           ip4_header_t *ip0, u8 *p_proto,
3333                           snat_session_key_t *p_value,
3334                           u8 *p_dont_translate, void *d, void *e)
3335 {
3336   icmp46_header_t *icmp0;
3337   u32 sw_if_index0;
3338   u8 protocol;
3339   snat_det_out_key_t key0;
3340   u8 dont_translate = 0;
3341   u32 next0 = ~0;
3342   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3343   ip4_header_t *inner_ip0;
3344   void *l4_header = 0;
3345   icmp46_header_t *inner_icmp0;
3346   snat_det_map_t * dm0 = 0;
3347   ip4_address_t new_addr0 = {{0}};
3348   snat_det_session_t * ses0 = 0;
3349   ip4_address_t out_addr;
3350
3351   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3352   echo0 = (icmp_echo_header_t *)(icmp0+1);
3353   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3354
3355   if (!icmp_is_error_message (icmp0))
3356     {
3357       protocol = SNAT_PROTOCOL_ICMP;
3358       key0.ext_host_addr = ip0->src_address;
3359       key0.ext_host_port = 0;
3360       key0.out_port = echo0->identifier;
3361       out_addr = ip0->dst_address;
3362     }
3363   else
3364     {
3365       inner_ip0 = (ip4_header_t *)(echo0+1);
3366       l4_header = ip4_next_header (inner_ip0);
3367       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3368       key0.ext_host_addr = inner_ip0->dst_address;
3369       out_addr = inner_ip0->src_address;
3370       switch (protocol)
3371         {
3372         case SNAT_PROTOCOL_ICMP:
3373           inner_icmp0 = (icmp46_header_t*)l4_header;
3374           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3375           key0.ext_host_port = 0;
3376           key0.out_port = inner_echo0->identifier;
3377           break;
3378         case SNAT_PROTOCOL_UDP:
3379         case SNAT_PROTOCOL_TCP:
3380           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3381           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3382           break;
3383         default:
3384           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3385           next0 = SNAT_OUT2IN_NEXT_DROP;
3386           goto out;
3387         }
3388     }
3389
3390   dm0 = snat_det_map_by_out(sm, &out_addr);
3391   if (PREDICT_FALSE(!dm0))
3392     {
3393       /* Don't NAT packet aimed at the intfc address */
3394       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3395                                           ip0->dst_address.as_u32)))
3396         {
3397           dont_translate = 1;
3398           goto out;
3399         }
3400       nat_log_info ("unknown dst address:  %U",
3401                     format_ip4_address, &ip0->dst_address);
3402       goto out;
3403     }
3404
3405   snat_det_reverse(dm0, &ip0->dst_address,
3406                    clib_net_to_host_u16(key0.out_port), &new_addr0);
3407
3408   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3409   if (PREDICT_FALSE(!ses0))
3410     {
3411       /* Don't NAT packet aimed at the intfc address */
3412       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3413                                           ip0->dst_address.as_u32)))
3414         {
3415           dont_translate = 1;
3416           goto out;
3417         }
3418       nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3419                     format_ip4_address, &key0.ext_host_addr,
3420                     clib_net_to_host_u16 (key0.ext_host_port),
3421                     format_ip4_address, &out_addr,
3422                     clib_net_to_host_u16 (key0.out_port),
3423                     format_ip4_address, &new_addr0);
3424       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3425       next0 = SNAT_OUT2IN_NEXT_DROP;
3426       goto out;
3427     }
3428
3429   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3430                     !icmp_is_error_message (icmp0)))
3431     {
3432       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3433       next0 = SNAT_OUT2IN_NEXT_DROP;
3434       goto out;
3435     }
3436
3437   goto out;
3438
3439 out:
3440   *p_proto = protocol;
3441   if (ses0)
3442     {
3443       p_value->addr = new_addr0;
3444       p_value->fib_index = sm->inside_fib_index;
3445       p_value->port = ses0->in_port;
3446     }
3447   *p_dont_translate = dont_translate;
3448   if (d)
3449     *(snat_det_session_t**)d = ses0;
3450   if (e)
3451     *(snat_det_map_t**)e = dm0;
3452   return next0;
3453 }
3454
3455 /**********************/
3456 /*** worker handoff ***/
3457 /**********************/
3458 static uword
3459 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3460                                vlib_node_runtime_t * node,
3461                                vlib_frame_t * frame)
3462 {
3463   snat_main_t *sm = &snat_main;
3464   vlib_thread_main_t *tm = vlib_get_thread_main ();
3465   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3466   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3467   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3468     = 0;
3469   vlib_frame_queue_elt_t *hf = 0;
3470   vlib_frame_queue_t *fq;
3471   vlib_frame_t *f = 0;
3472   int i;
3473   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3474   u32 next_worker_index = 0;
3475   u32 current_worker_index = ~0;
3476   u32 thread_index = vlib_get_thread_index ();
3477   vlib_frame_t *d = 0;
3478
3479   ASSERT (vec_len (sm->workers));
3480
3481   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3482     {
3483       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3484
3485       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3486                                tm->n_vlib_mains - 1,
3487                                (vlib_frame_queue_t *) (~0));
3488     }
3489
3490   from = vlib_frame_vector_args (frame);
3491   n_left_from = frame->n_vectors;
3492
3493   while (n_left_from > 0)
3494     {
3495       u32 bi0;
3496       vlib_buffer_t *b0;
3497       u32 sw_if_index0;
3498       u32 rx_fib_index0;
3499       ip4_header_t * ip0;
3500       u8 do_handoff;
3501
3502       bi0 = from[0];
3503       from += 1;
3504       n_left_from -= 1;
3505
3506       b0 = vlib_get_buffer (vm, bi0);
3507
3508       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3509       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3510
3511       ip0 = vlib_buffer_get_current (b0);
3512
3513       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3514
3515       if (PREDICT_FALSE (next_worker_index != thread_index))
3516         {
3517           do_handoff = 1;
3518
3519           if (next_worker_index != current_worker_index)
3520             {
3521               fq = is_vlib_frame_queue_congested (
3522                 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3523                 congested_handoff_queue_by_worker_index);
3524
3525               if (fq)
3526                 {
3527                   /* if this is 1st frame */
3528                   if (!d)
3529                     {
3530                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
3531                       to_next_drop = vlib_frame_vector_args (d);
3532                     }
3533
3534                   to_next_drop[0] = bi0;
3535                   to_next_drop += 1;
3536                   d->n_vectors++;
3537                   b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3538                   goto trace0;
3539                 }
3540
3541               if (hf)
3542                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3543
3544               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3545                                                       next_worker_index,
3546                                                       handoff_queue_elt_by_worker_index);
3547
3548               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3549               to_next_worker = &hf->buffer_index[hf->n_vectors];
3550               current_worker_index = next_worker_index;
3551             }
3552
3553           /* enqueue to correct worker thread */
3554           to_next_worker[0] = bi0;
3555           to_next_worker++;
3556           n_left_to_next_worker--;
3557
3558           if (n_left_to_next_worker == 0)
3559             {
3560               hf->n_vectors = VLIB_FRAME_SIZE;
3561               vlib_put_frame_queue_elt (hf);
3562               current_worker_index = ~0;
3563               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3564               hf = 0;
3565             }
3566         }
3567       else
3568         {
3569           do_handoff = 0;
3570           /* if this is 1st frame */
3571           if (!f)
3572             {
3573               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3574               to_next = vlib_frame_vector_args (f);
3575             }
3576
3577           to_next[0] = bi0;
3578           to_next += 1;
3579           f->n_vectors++;
3580         }
3581
3582 trace0:
3583       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3584                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3585         {
3586           snat_out2in_worker_handoff_trace_t *t =
3587             vlib_add_trace (vm, node, b0, sizeof (*t));
3588           t->next_worker_index = next_worker_index;
3589           t->do_handoff = do_handoff;
3590         }
3591     }
3592
3593   if (f)
3594     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3595
3596   if (d)
3597     vlib_put_frame_to_node (vm, sm->error_node_index, d);
3598
3599   if (hf)
3600     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3601
3602   /* Ship frames to the worker nodes */
3603   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3604     {
3605       if (handoff_queue_elt_by_worker_index[i])
3606         {
3607           hf = handoff_queue_elt_by_worker_index[i];
3608           /*
3609            * It works better to let the handoff node
3610            * rate-adapt, always ship the handoff queue element.
3611            */
3612           if (1 || hf->n_vectors == hf->last_n_vectors)
3613             {
3614               vlib_put_frame_queue_elt (hf);
3615               handoff_queue_elt_by_worker_index[i] = 0;
3616             }
3617           else
3618             hf->last_n_vectors = hf->n_vectors;
3619         }
3620       congested_handoff_queue_by_worker_index[i] =
3621         (vlib_frame_queue_t *) (~0);
3622     }
3623   hf = 0;
3624   current_worker_index = ~0;
3625   return frame->n_vectors;
3626 }
3627
3628 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3629   .function = snat_out2in_worker_handoff_fn,
3630   .name = "nat44-out2in-worker-handoff",
3631   .vector_size = sizeof (u32),
3632   .format_trace = format_snat_out2in_worker_handoff_trace,
3633   .type = VLIB_NODE_TYPE_INTERNAL,
3634
3635   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3636   .error_strings = snat_out2in_error_strings,
3637
3638   .n_next_nodes = 1,
3639
3640   .next_nodes = {
3641     [0] = "error-drop",
3642   },
3643 };
3644
3645 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3646
3647 static uword
3648 snat_out2in_fast_node_fn (vlib_main_t * vm,
3649                           vlib_node_runtime_t * node,
3650                           vlib_frame_t * frame)
3651 {
3652   u32 n_left_from, * from, * to_next;
3653   snat_out2in_next_t next_index;
3654   u32 pkts_processed = 0;
3655   snat_main_t * sm = &snat_main;
3656
3657   from = vlib_frame_vector_args (frame);
3658   n_left_from = frame->n_vectors;
3659   next_index = node->cached_next_index;
3660
3661   while (n_left_from > 0)
3662     {
3663       u32 n_left_to_next;
3664
3665       vlib_get_next_frame (vm, node, next_index,
3666                            to_next, n_left_to_next);
3667
3668       while (n_left_from > 0 && n_left_to_next > 0)
3669         {
3670           u32 bi0;
3671           vlib_buffer_t * b0;
3672           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3673           u32 sw_if_index0;
3674           ip4_header_t * ip0;
3675           ip_csum_t sum0;
3676           u32 new_addr0, old_addr0;
3677           u16 new_port0, old_port0;
3678           udp_header_t * udp0;
3679           tcp_header_t * tcp0;
3680           icmp46_header_t * icmp0;
3681           snat_session_key_t key0, sm0;
3682           u32 proto0;
3683           u32 rx_fib_index0;
3684
3685           /* speculatively enqueue b0 to the current next frame */
3686           bi0 = from[0];
3687           to_next[0] = bi0;
3688           from += 1;
3689           to_next += 1;
3690           n_left_from -= 1;
3691           n_left_to_next -= 1;
3692
3693           b0 = vlib_get_buffer (vm, bi0);
3694
3695           ip0 = vlib_buffer_get_current (b0);
3696           udp0 = ip4_next_header (ip0);
3697           tcp0 = (tcp_header_t *) udp0;
3698           icmp0 = (icmp46_header_t *) udp0;
3699
3700           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3701           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3702
3703           vnet_feature_next (sw_if_index0, &next0, b0);
3704
3705           if (PREDICT_FALSE(ip0->ttl == 1))
3706             {
3707               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3708               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3709                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3710                                            0);
3711               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3712               goto trace00;
3713             }
3714
3715           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3716
3717           if (PREDICT_FALSE (proto0 == ~0))
3718               goto trace00;
3719
3720           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3721             {
3722               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3723                                   rx_fib_index0, node, next0, ~0, 0, 0);
3724               goto trace00;
3725             }
3726
3727           key0.addr = ip0->dst_address;
3728           key0.port = udp0->dst_port;
3729           key0.fib_index = rx_fib_index0;
3730
3731           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3732             {
3733               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3734               goto trace00;
3735             }
3736
3737           new_addr0 = sm0.addr.as_u32;
3738           new_port0 = sm0.port;
3739           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3740           old_addr0 = ip0->dst_address.as_u32;
3741           ip0->dst_address.as_u32 = new_addr0;
3742
3743           sum0 = ip0->checksum;
3744           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3745                                  ip4_header_t,
3746                                  dst_address /* changed member */);
3747           ip0->checksum = ip_csum_fold (sum0);
3748
3749           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3750             {
3751                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3752                 {
3753                   old_port0 = tcp0->dst_port;
3754                   tcp0->dst_port = new_port0;
3755
3756                   sum0 = tcp0->checksum;
3757                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3758                                          ip4_header_t,
3759                                          dst_address /* changed member */);
3760
3761                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
3762                                          ip4_header_t /* cheat */,
3763                                          length /* changed member */);
3764                   tcp0->checksum = ip_csum_fold(sum0);
3765                 }
3766               else
3767                 {
3768                   old_port0 = udp0->dst_port;
3769                   udp0->dst_port = new_port0;
3770                   udp0->checksum = 0;
3771                 }
3772             }
3773           else
3774             {
3775               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3776                 {
3777                   sum0 = tcp0->checksum;
3778                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3779                                          ip4_header_t,
3780                                          dst_address /* changed member */);
3781
3782                   tcp0->checksum = ip_csum_fold(sum0);
3783                 }
3784             }
3785
3786         trace00:
3787
3788           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3789                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3790             {
3791               snat_out2in_trace_t *t =
3792                  vlib_add_trace (vm, node, b0, sizeof (*t));
3793               t->sw_if_index = sw_if_index0;
3794               t->next_index = next0;
3795             }
3796
3797           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3798
3799           /* verify speculative enqueue, maybe switch current next frame */
3800           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3801                                            to_next, n_left_to_next,
3802                                            bi0, next0);
3803         }
3804
3805       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3806     }
3807
3808   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3809                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3810                                pkts_processed);
3811   return frame->n_vectors;
3812 }
3813
3814 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3815   .function = snat_out2in_fast_node_fn,
3816   .name = "nat44-out2in-fast",
3817   .vector_size = sizeof (u32),
3818   .format_trace = format_snat_out2in_fast_trace,
3819   .type = VLIB_NODE_TYPE_INTERNAL,
3820
3821   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3822   .error_strings = snat_out2in_error_strings,
3823
3824   .runtime_data_bytes = sizeof (snat_runtime_t),
3825
3826   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3827
3828   /* edit / add dispositions here */
3829   .next_nodes = {
3830     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3831     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3832     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3833     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3834   },
3835 };
3836 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);