src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28 #include <nat/nat_reass.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38 } snat_out2in_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_out2in_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  53               t->sw_if_index, t->next_index, t->session_index);
  54   return s;
  55 }
  56
  57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  58 {
  59   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  60   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  61   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  62
  63   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  64               t->sw_if_index, t->next_index);
  65   return s;
  66 }
  67
  68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   snat_out2in_worker_handoff_trace_t * t =
  73     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  74   char * m;
  75
  76   m = t->do_handoff ? "next worker" : "same worker";
  77   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  78
  79   return s;
  80 }
  81
  82 typedef struct {
  83   u32 sw_if_index;
  84   u32 next_index;
  85   u8 cached;
  86 } nat44_out2in_reass_trace_t;
  87
  88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
  89 {
  90   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  91   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  92   nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
  93
  94   s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
  95               t->sw_if_index, t->next_index,
  96               t->cached ? "cached" : "translated");
  97
  98   return s;
  99 }
 100
 101 vlib_node_registration_t snat_out2in_node;
 102 vlib_node_registration_t snat_out2in_fast_node;
 103 vlib_node_registration_t snat_out2in_worker_handoff_node;
 104 vlib_node_registration_t snat_det_out2in_node;
 105 vlib_node_registration_t nat44_out2in_reass_node;
 106
 107 #define foreach_snat_out2in_error                       \
 108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 109 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
 110 _(OUT_OF_PORTS, "Out of ports")                         \
 111 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 112 _(NO_TRANSLATION, "No translation")                     \
 113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 114 _(DROP_FRAGMENT, "Drop fragment")                       \
 115 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 117
 118 typedef enum {
 119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
 120   foreach_snat_out2in_error
 121 #undef _
 122   SNAT_OUT2IN_N_ERROR,
 123 } snat_out2in_error_t;
 124
 125 static char * snat_out2in_error_strings[] = {
 126 #define _(sym,string) string,
 127   foreach_snat_out2in_error
 128 #undef _
 129 };
 130
 131 typedef enum {
 132   SNAT_OUT2IN_NEXT_DROP,
 133   SNAT_OUT2IN_NEXT_LOOKUP,
 134   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 135   SNAT_OUT2IN_NEXT_REASS,
 136   SNAT_OUT2IN_N_NEXT,
 137 } snat_out2in_next_t;
 138
 139 /**
 140  * @brief Create session for static mapping.
 141  *
 142  * Create NAT session initiated by host from external network with static
 143  * mapping.
 144  *
 145  * @param sm     NAT main.
 146  * @param b0     Vlib buffer.
 147  * @param in2out In2out NAT44 session key.
 148  * @param out2in Out2in NAT44 session key.
 149  * @param node   Vlib node.
 150  *
 151  * @returns SNAT session if successfully created otherwise 0.
 152  */
 153 static inline snat_session_t *
 154 create_session_for_static_mapping (snat_main_t *sm,
 155                                    vlib_buffer_t *b0,
 156                                    snat_session_key_t in2out,
 157                                    snat_session_key_t out2in,
 158                                    vlib_node_runtime_t * node,
 159                                    u32 thread_index)
 160 {
 161   snat_user_t *u;
 162   snat_session_t *s;
 163   clib_bihash_kv_8_8_t kv0;
 164   ip4_header_t *ip0;
 165   udp_header_t *udp0;
 166
 167   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 168     {
 169       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 170       return 0;
 171     }
 172
 173   ip0 = vlib_buffer_get_current (b0);
 174   udp0 = ip4_next_header (ip0);
 175
 176   u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 177   if (!u)
 178     {
 179       clib_warning ("create NAT user failed");
 180       return 0;
 181     }
 182
 183   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 184   if (!s)
 185     {
 186       clib_warning ("create NAT session failed");
 187       return 0;
 188     }
 189
 190   s->outside_address_index = ~0;
 191   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 192   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 193   s->ext_host_port = udp0->src_port;
 194   u->nstaticsessions++;
 195   s->in2out = in2out;
 196   s->out2in = out2in;
 197   s->in2out.protocol = out2in.protocol;
 198
 199   /* Add to translation hashes */
 200   kv0.key = s->in2out.as_u64;
 201   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 202   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 203                                1 /* is_add */))
 204       clib_warning ("in2out key add failed");
 205
 206   kv0.key = s->out2in.as_u64;
 207
 208   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 209                                1 /* is_add */))
 210       clib_warning ("out2in key add failed");
 211
 212   /* log NAT event */
 213   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 214                                       s->out2in.addr.as_u32,
 215                                       s->in2out.protocol,
 216                                       s->in2out.port,
 217                                       s->out2in.port,
 218                                       s->in2out.fib_index);
 219    return s;
 220 }
 221
 222 static_always_inline
 223 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 224                                  snat_session_key_t *p_key0)
 225 {
 226   icmp46_header_t *icmp0;
 227   snat_session_key_t key0;
 228   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 229   ip4_header_t *inner_ip0;
 230   void *l4_header = 0;
 231   icmp46_header_t *inner_icmp0;
 232
 233   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 234   echo0 = (icmp_echo_header_t *)(icmp0+1);
 235
 236   if (!icmp_is_error_message (icmp0))
 237     {
 238       key0.protocol = SNAT_PROTOCOL_ICMP;
 239       key0.addr = ip0->dst_address;
 240       key0.port = echo0->identifier;
 241     }
 242   else
 243     {
 244       inner_ip0 = (ip4_header_t *)(echo0+1);
 245       l4_header = ip4_next_header (inner_ip0);
 246       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 247       key0.addr = inner_ip0->src_address;
 248       switch (key0.protocol)
 249         {
 250         case SNAT_PROTOCOL_ICMP:
 251           inner_icmp0 = (icmp46_header_t*)l4_header;
 252           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 253           key0.port = inner_echo0->identifier;
 254           break;
 255         case SNAT_PROTOCOL_UDP:
 256         case SNAT_PROTOCOL_TCP:
 257           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 258           break;
 259         default:
 260           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 261         }
 262     }
 263   *p_key0 = key0;
 264   return -1; /* success */
 265 }
 266
 267 /**
 268  * Get address and port values to be used for ICMP packet translation
 269  * and create session if needed
 270  *
 271  * @param[in,out] sm             NAT main
 272  * @param[in,out] node           NAT node runtime
 273  * @param[in] thread_index       thread index
 274  * @param[in,out] b0             buffer containing packet to be translated
 275  * @param[out] p_proto           protocol used for matching
 276  * @param[out] p_value           address and port after NAT translation
 277  * @param[out] p_dont_translate  if packet should not be translated
 278  * @param d                      optional parameter
 279  * @param e                      optional parameter
 280  */
 281 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 282                            u32 thread_index, vlib_buffer_t *b0,
 283                            ip4_header_t *ip0, u8 *p_proto,
 284                            snat_session_key_t *p_value,
 285                            u8 *p_dont_translate, void *d, void *e)
 286 {
 287   icmp46_header_t *icmp0;
 288   u32 sw_if_index0;
 289   u32 rx_fib_index0;
 290   snat_session_key_t key0;
 291   snat_session_key_t sm0;
 292   snat_session_t *s0 = 0;
 293   u8 dont_translate = 0;
 294   clib_bihash_kv_8_8_t kv0, value0;
 295   u8 is_addr_only;
 296   u32 next0 = ~0;
 297   int err;
 298
 299   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 300   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 301   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 302
 303   key0.protocol = 0;
 304
 305   err = icmp_get_key (ip0, &key0);
 306   if (err != -1)
 307     {
 308       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 309       next0 = SNAT_OUT2IN_NEXT_DROP;
 310       goto out;
 311     }
 312   key0.fib_index = rx_fib_index0;
 313
 314   kv0.key = key0.as_u64;
 315
 316   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 317                               &value0))
 318     {
 319       /* Try to match static mapping by external address and port,
 320          destination address and port in packet */
 321       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0))
 322         {
 323           if (!sm->forwarding_enabled)
 324             {
 325               /* Don't NAT packet aimed at the intfc address */
 326               if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 327                                                   ip0->dst_address.as_u32)))
 328                 {
 329                   dont_translate = 1;
 330                   goto out;
 331                 }
 332               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 333               next0 = SNAT_OUT2IN_NEXT_DROP;
 334               goto out;
 335             }
 336           else
 337             {
 338               dont_translate = 1;
 339               goto out;
 340             }
 341         }
 342
 343       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 344                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 345         {
 346           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 347           next0 = SNAT_OUT2IN_NEXT_DROP;
 348           goto out;
 349         }
 350
 351       /* Create session initiated by host from external network */
 352       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 353                                              node, thread_index);
 354
 355       if (!s0)
 356         {
 357           next0 = SNAT_OUT2IN_NEXT_DROP;
 358           goto out;
 359         }
 360     }
 361   else
 362     {
 363       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 364                         icmp0->type != ICMP4_echo_request &&
 365                         !icmp_is_error_message (icmp0)))
 366         {
 367           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 368           next0 = SNAT_OUT2IN_NEXT_DROP;
 369           goto out;
 370         }
 371
 372       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 373                               value0.value);
 374     }
 375
 376 out:
 377   *p_proto = key0.protocol;
 378   if (s0)
 379     *p_value = s0->in2out;
 380   *p_dont_translate = dont_translate;
 381   if (d)
 382     *(snat_session_t**)d = s0;
 383   return next0;
 384 }
 385
 386 /**
 387  * Get address and port values to be used for ICMP packet translation
 388  *
 389  * @param[in] sm                 NAT main
 390  * @param[in,out] node           NAT node runtime
 391  * @param[in] thread_index       thread index
 392  * @param[in,out] b0             buffer containing packet to be translated
 393  * @param[out] p_proto           protocol used for matching
 394  * @param[out] p_value           address and port after NAT translation
 395  * @param[out] p_dont_translate  if packet should not be translated
 396  * @param d                      optional parameter
 397  * @param e                      optional parameter
 398  */
 399 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 400                            u32 thread_index, vlib_buffer_t *b0,
 401                            ip4_header_t *ip0, u8 *p_proto,
 402                            snat_session_key_t *p_value,
 403                            u8 *p_dont_translate, void *d, void *e)
 404 {
 405   icmp46_header_t *icmp0;
 406   u32 sw_if_index0;
 407   u32 rx_fib_index0;
 408   snat_session_key_t key0;
 409   snat_session_key_t sm0;
 410   u8 dont_translate = 0;
 411   u8 is_addr_only;
 412   u32 next0 = ~0;
 413   int err;
 414
 415   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 416   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 417   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 418
 419   err = icmp_get_key (ip0, &key0);
 420   if (err != -1)
 421     {
 422       b0->error = node->errors[err];
 423       next0 = SNAT_OUT2IN_NEXT_DROP;
 424       goto out2;
 425     }
 426   key0.fib_index = rx_fib_index0;
 427
 428   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0))
 429     {
 430       /* Don't NAT packet aimed at the intfc address */
 431       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 432         {
 433           dont_translate = 1;
 434           goto out;
 435         }
 436       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 437       next0 = SNAT_OUT2IN_NEXT_DROP;
 438       goto out;
 439     }
 440
 441   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 442                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 443                     !icmp_is_error_message (icmp0)))
 444     {
 445       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 446       next0 = SNAT_OUT2IN_NEXT_DROP;
 447       goto out;
 448     }
 449
 450 out:
 451   *p_value = sm0;
 452 out2:
 453   *p_proto = key0.protocol;
 454   *p_dont_translate = dont_translate;
 455   return next0;
 456 }
 457
 458 static inline u32 icmp_out2in (snat_main_t *sm,
 459                                vlib_buffer_t * b0,
 460                                ip4_header_t * ip0,
 461                                icmp46_header_t * icmp0,
 462                                u32 sw_if_index0,
 463                                u32 rx_fib_index0,
 464                                vlib_node_runtime_t * node,
 465                                u32 next0,
 466                                u32 thread_index,
 467                                void *d,
 468                                void *e)
 469 {
 470   snat_session_key_t sm0;
 471   u8 protocol;
 472   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 473   ip4_header_t *inner_ip0 = 0;
 474   void *l4_header = 0;
 475   icmp46_header_t *inner_icmp0;
 476   u8 dont_translate;
 477   u32 new_addr0, old_addr0;
 478   u16 old_id0, new_id0;
 479   ip_csum_t sum0;
 480   u16 checksum0;
 481   u32 next0_tmp;
 482
 483   echo0 = (icmp_echo_header_t *)(icmp0+1);
 484
 485   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 486                                        &protocol, &sm0, &dont_translate, d, e);
 487   if (next0_tmp != ~0)
 488     next0 = next0_tmp;
 489   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 490     goto out;
 491
 492   sum0 = ip_incremental_checksum (0, icmp0,
 493                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 494   checksum0 = ~ip_csum_fold (sum0);
 495   if (checksum0 != 0 && checksum0 != 0xffff)
 496     {
 497       next0 = SNAT_OUT2IN_NEXT_DROP;
 498       goto out;
 499     }
 500
 501   old_addr0 = ip0->dst_address.as_u32;
 502   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 503   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 504
 505   sum0 = ip0->checksum;
 506   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 507                          dst_address /* changed member */);
 508   ip0->checksum = ip_csum_fold (sum0);
 509
 510   if (!icmp_is_error_message (icmp0))
 511     {
 512       new_id0 = sm0.port;
 513       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 514         {
 515           old_id0 = echo0->identifier;
 516           new_id0 = sm0.port;
 517           echo0->identifier = new_id0;
 518
 519           sum0 = icmp0->checksum;
 520           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 521                                  identifier /* changed member */);
 522           icmp0->checksum = ip_csum_fold (sum0);
 523         }
 524     }
 525   else
 526     {
 527       inner_ip0 = (ip4_header_t *)(echo0+1);
 528       l4_header = ip4_next_header (inner_ip0);
 529
 530       if (!ip4_header_checksum_is_valid (inner_ip0))
 531         {
 532           next0 = SNAT_OUT2IN_NEXT_DROP;
 533           goto out;
 534         }
 535
 536       old_addr0 = inner_ip0->src_address.as_u32;
 537       inner_ip0->src_address = sm0.addr;
 538       new_addr0 = inner_ip0->src_address.as_u32;
 539
 540       sum0 = icmp0->checksum;
 541       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 542                              src_address /* changed member */);
 543       icmp0->checksum = ip_csum_fold (sum0);
 544
 545       switch (protocol)
 546         {
 547         case SNAT_PROTOCOL_ICMP:
 548           inner_icmp0 = (icmp46_header_t*)l4_header;
 549           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 550
 551           old_id0 = inner_echo0->identifier;
 552           new_id0 = sm0.port;
 553           inner_echo0->identifier = new_id0;
 554
 555           sum0 = icmp0->checksum;
 556           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 557                                  identifier);
 558           icmp0->checksum = ip_csum_fold (sum0);
 559           break;
 560         case SNAT_PROTOCOL_UDP:
 561         case SNAT_PROTOCOL_TCP:
 562           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 563           new_id0 = sm0.port;
 564           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 565
 566           sum0 = icmp0->checksum;
 567           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 568                                  src_port);
 569           icmp0->checksum = ip_csum_fold (sum0);
 570           break;
 571         default:
 572           ASSERT(0);
 573         }
 574     }
 575
 576 out:
 577   return next0;
 578 }
 579
 580
 581 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 582                                          vlib_buffer_t * b0,
 583                                          ip4_header_t * ip0,
 584                                          icmp46_header_t * icmp0,
 585                                          u32 sw_if_index0,
 586                                          u32 rx_fib_index0,
 587                                          vlib_node_runtime_t * node,
 588                                          u32 next0, f64 now,
 589                                          u32 thread_index,
 590                                          snat_session_t ** p_s0)
 591 {
 592   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 593                       next0, thread_index, p_s0, 0);
 594   snat_session_t * s0 = *p_s0;
 595   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 596     {
 597       /* Accounting */
 598       s0->last_heard = now;
 599       s0->total_pkts++;
 600       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 601       /* Per-user LRU list maintenance */
 602       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 603                          s0->per_user_index);
 604       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 605                           s0->per_user_list_head_index,
 606                           s0->per_user_index);
 607     }
 608   return next0;
 609 }
 610
 611 static snat_session_t *
 612 snat_out2in_unknown_proto (snat_main_t *sm,
 613                            vlib_buffer_t * b,
 614                            ip4_header_t * ip,
 615                            u32 rx_fib_index,
 616                            u32 thread_index,
 617                            f64 now,
 618                            vlib_main_t * vm,
 619                            vlib_node_runtime_t * node)
 620 {
 621   clib_bihash_kv_8_8_t kv, value;
 622   clib_bihash_kv_16_8_t s_kv, s_value;
 623   snat_static_mapping_t *m;
 624   snat_session_key_t m_key;
 625   u32 old_addr, new_addr;
 626   ip_csum_t sum;
 627   nat_ed_ses_key_t key;
 628   snat_session_t * s;
 629   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 630   snat_user_t *u;
 631
 632   old_addr = ip->dst_address.as_u32;
 633
 634   key.l_addr = ip->dst_address;
 635   key.r_addr = ip->src_address;
 636   key.fib_index = rx_fib_index;
 637   key.proto = ip->protocol;
 638   key.r_port = 0;
 639   key.l_port = 0;
 640   s_kv.key[0] = key.as_u64[0];
 641   s_kv.key[1] = key.as_u64[1];
 642
 643   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 644     {
 645       s = pool_elt_at_index (tsm->sessions, s_value.value);
 646       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 647     }
 648   else
 649     {
 650       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 651         {
 652           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 653           return 0;
 654         }
 655
 656       m_key.addr = ip->dst_address;
 657       m_key.port = 0;
 658       m_key.protocol = 0;
 659       m_key.fib_index = rx_fib_index;
 660       kv.key = m_key.as_u64;
 661       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 662         {
 663           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 664           return 0;
 665         }
 666
 667       m = pool_elt_at_index (sm->static_mappings, value.value);
 668
 669       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 670
 671       u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
 672                                   thread_index);
 673       if (!u)
 674         {
 675           clib_warning ("create NAT user failed");
 676           return 0;
 677         }
 678
 679       /* Create a new session */
 680       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 681       if (!s)
 682         {
 683           clib_warning ("create NAT session failed");
 684           return 0;
 685         }
 686
 687       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 688       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 689       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 690       s->outside_address_index = ~0;
 691       s->out2in.addr.as_u32 = old_addr;
 692       s->out2in.fib_index = rx_fib_index;
 693       s->in2out.addr.as_u32 = new_addr;
 694       s->in2out.fib_index = m->fib_index;
 695       s->in2out.port = s->out2in.port = ip->protocol;
 696       u->nstaticsessions++;
 697
 698       /* Add to lookup tables */
 699       s_kv.value = s - tsm->sessions;
 700       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 701         clib_warning ("out2in key add failed");
 702
 703       key.l_addr = ip->dst_address;
 704       key.fib_index = m->fib_index;
 705       s_kv.key[0] = key.as_u64[0];
 706       s_kv.key[1] = key.as_u64[1];
 707       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 708         clib_warning ("in2out key add failed");
 709    }
 710
 711   /* Update IP checksum */
 712   sum = ip->checksum;
 713   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 714   ip->checksum = ip_csum_fold (sum);
 715
 716   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 717
 718   /* Accounting */
 719   s->last_heard = now;
 720   s->total_pkts++;
 721   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 722   /* Per-user LRU list maintenance */
 723   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 724   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 725                       s->per_user_index);
 726
 727   return s;
 728 }
 729
 730 static snat_session_t *
 731 snat_out2in_lb (snat_main_t *sm,
 732                 vlib_buffer_t * b,
 733                 ip4_header_t * ip,
 734                 u32 rx_fib_index,
 735                 u32 thread_index,
 736                 f64 now,
 737                 vlib_main_t * vm,
 738                 vlib_node_runtime_t * node)
 739 {
 740   nat_ed_ses_key_t key;
 741   clib_bihash_kv_16_8_t s_kv, s_value;
 742   udp_header_t *udp = ip4_next_header (ip);
 743   tcp_header_t *tcp = (tcp_header_t *) udp;
 744   snat_session_t *s = 0;
 745   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 746   snat_session_key_t e_key, l_key;
 747   u32 old_addr, new_addr;
 748   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 749   u16 new_port, old_port;
 750   ip_csum_t sum;
 751   snat_user_t *u;
 752   u32 address_index;
 753   snat_session_key_t eh_key;
 754   u8 twice_nat;
 755
 756   old_addr = ip->dst_address.as_u32;
 757
 758   key.l_addr = ip->dst_address;
 759   key.r_addr = ip->src_address;
 760   key.fib_index = rx_fib_index;
 761   key.proto = ip->protocol;
 762   key.r_port = udp->src_port;
 763   key.l_port = udp->dst_port;
 764   s_kv.key[0] = key.as_u64[0];
 765   s_kv.key[1] = key.as_u64[1];
 766
 767   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 768     {
 769       s = pool_elt_at_index (tsm->sessions, s_value.value);
 770     }
 771   else
 772     {
 773       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 774         {
 775           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 776           return 0;
 777         }
 778
 779       e_key.addr = ip->dst_address;
 780       e_key.port = udp->dst_port;
 781       e_key.protocol = proto;
 782       e_key.fib_index = rx_fib_index;
 783       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat))
 784         return 0;
 785
 786       u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
 787                                   thread_index);
 788       if (!u)
 789       {
 790         clib_warning ("create NAT user failed");
 791         return 0;
 792       }
 793
 794       s = nat_session_alloc_or_recycle (sm, u, thread_index);
 795       if (!s)
 796         {
 797           clib_warning ("create NAT session failed");
 798           return 0;
 799         }
 800
 801       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 802       s->ext_host_port = udp->src_port;
 803       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 804       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 805       s->outside_address_index = ~0;
 806       s->out2in = e_key;
 807       s->in2out = l_key;
 808       u->nstaticsessions++;
 809
 810       /* Add to lookup tables */
 811       s_kv.value = s - tsm->sessions;
 812       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 813         clib_warning ("out2in-ed key add failed");
 814
 815       if (twice_nat)
 816         {
 817           eh_key.protocol = proto;
 818           if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
 819                                                    thread_index, &eh_key,
 820                                                    &address_index,
 821                                                    sm->port_per_thread,
 822                                                    sm->per_thread_data[thread_index].snat_thread_index))
 823             {
 824               b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
 825               return 0;
 826             }
 827           key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
 828           key.r_port = s->ext_host_nat_port = eh_key.port;
 829           s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
 830         }
 831       key.l_addr = l_key.addr;
 832       key.fib_index = l_key.fib_index;
 833       key.l_port = l_key.port;
 834       s_kv.key[0] = key.as_u64[0];
 835       s_kv.key[1] = key.as_u64[1];
 836       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 837         clib_warning ("in2out-ed key add failed");
 838     }
 839
 840   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 841
 842   /* Update IP checksum */
 843   sum = ip->checksum;
 844   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 845   if (is_twice_nat_session (s))
 846     sum = ip_csum_update (sum, ip->src_address.as_u32,
 847                           s->ext_host_nat_addr.as_u32, ip4_header_t,
 848                           src_address);
 849   ip->checksum = ip_csum_fold (sum);
 850
 851   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 852     {
 853       old_port = tcp->dst_port;
 854       tcp->dst_port = s->in2out.port;
 855       new_port = tcp->dst_port;
 856
 857       sum = tcp->checksum;
 858       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 859       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 860       if (is_twice_nat_session (s))
 861         {
 862           sum = ip_csum_update (sum, ip->src_address.as_u32,
 863                                 s->ext_host_nat_addr.as_u32, ip4_header_t,
 864                                 dst_address);
 865           sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
 866                                 ip4_header_t, length);
 867           tcp->src_port = s->ext_host_nat_port;
 868           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
 869         }
 870       tcp->checksum = ip_csum_fold(sum);
 871     }
 872   else
 873     {
 874       udp->dst_port = s->in2out.port;
 875       if (is_twice_nat_session (s))
 876         {
 877           udp->src_port = s->ext_host_nat_port;
 878           ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
 879         }
 880       udp->checksum = 0;
 881     }
 882
 883   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 884
 885   /* Accounting */
 886   s->last_heard = now;
 887   s->total_pkts++;
 888   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 889   /* Per-user LRU list maintenance */
 890   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 891   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 892                       s->per_user_index);
 893
 894   return s;
 895 }
 896
 897 static uword
 898 snat_out2in_node_fn (vlib_main_t * vm,
 899                   vlib_node_runtime_t * node,
 900                   vlib_frame_t * frame)
 901 {
 902   u32 n_left_from, * from, * to_next;
 903   snat_out2in_next_t next_index;
 904   u32 pkts_processed = 0;
 905   snat_main_t * sm = &snat_main;
 906   f64 now = vlib_time_now (vm);
 907   u32 thread_index = vlib_get_thread_index ();
 908
 909   from = vlib_frame_vector_args (frame);
 910   n_left_from = frame->n_vectors;
 911   next_index = node->cached_next_index;
 912
 913   while (n_left_from > 0)
 914     {
 915       u32 n_left_to_next;
 916
 917       vlib_get_next_frame (vm, node, next_index,
 918                            to_next, n_left_to_next);
 919
 920       while (n_left_from >= 4 && n_left_to_next >= 2)
 921         {
 922           u32 bi0, bi1;
 923           vlib_buffer_t * b0, * b1;
 924           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 925           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 926           u32 sw_if_index0, sw_if_index1;
 927           ip4_header_t * ip0, *ip1;
 928           ip_csum_t sum0, sum1;
 929           u32 new_addr0, old_addr0;
 930           u16 new_port0, old_port0;
 931           u32 new_addr1, old_addr1;
 932           u16 new_port1, old_port1;
 933           udp_header_t * udp0, * udp1;
 934           tcp_header_t * tcp0, * tcp1;
 935           icmp46_header_t * icmp0, * icmp1;
 936           snat_session_key_t key0, key1, sm0, sm1;
 937           u32 rx_fib_index0, rx_fib_index1;
 938           u32 proto0, proto1;
 939           snat_session_t * s0 = 0, * s1 = 0;
 940           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 941
 942           /* Prefetch next iteration. */
 943           {
 944             vlib_buffer_t * p2, * p3;
 945
 946             p2 = vlib_get_buffer (vm, from[2]);
 947             p3 = vlib_get_buffer (vm, from[3]);
 948
 949             vlib_prefetch_buffer_header (p2, LOAD);
 950             vlib_prefetch_buffer_header (p3, LOAD);
 951
 952             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 953             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 954           }
 955
 956           /* speculatively enqueue b0 and b1 to the current next frame */
 957           to_next[0] = bi0 = from[0];
 958           to_next[1] = bi1 = from[1];
 959           from += 2;
 960           to_next += 2;
 961           n_left_from -= 2;
 962           n_left_to_next -= 2;
 963
 964           b0 = vlib_get_buffer (vm, bi0);
 965           b1 = vlib_get_buffer (vm, bi1);
 966
 967           vnet_buffer (b0)->snat.flags = 0;
 968           vnet_buffer (b1)->snat.flags = 0;
 969
 970           ip0 = vlib_buffer_get_current (b0);
 971           udp0 = ip4_next_header (ip0);
 972           tcp0 = (tcp_header_t *) udp0;
 973           icmp0 = (icmp46_header_t *) udp0;
 974
 975           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 976           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 977                                    sw_if_index0);
 978
 979           if (PREDICT_FALSE(ip0->ttl == 1))
 980             {
 981               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 982               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 983                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 984                                            0);
 985               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 986               goto trace0;
 987             }
 988
 989           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 990
 991           if (PREDICT_FALSE (proto0 == ~0))
 992             {
 993               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
 994                                              thread_index, now, vm, node);
 995               if (!s0)
 996                 next0 = SNAT_OUT2IN_NEXT_DROP;
 997               goto trace0;
 998             }
 999
1000           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1001             {
1002               next0 = icmp_out2in_slow_path
1003                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1004                  next0, now, thread_index, &s0);
1005               goto trace0;
1006             }
1007
1008           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1009             {
1010               next0 = SNAT_OUT2IN_NEXT_REASS;
1011               goto trace0;
1012             }
1013
1014           key0.addr = ip0->dst_address;
1015           key0.port = udp0->dst_port;
1016           key0.protocol = proto0;
1017           key0.fib_index = rx_fib_index0;
1018
1019           kv0.key = key0.as_u64;
1020
1021           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1022                                       &kv0, &value0))
1023             {
1024               /* Try to match static mapping by external address and port,
1025                  destination address and port in packet */
1026               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1027                 {
1028                   if (!sm->forwarding_enabled)
1029                     {
1030                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1031                       /*
1032                        * Send DHCP packets to the ipv4 stack, or we won't
1033                        * be able to use dhcp client on the outside interface
1034                        */
1035                       if (proto0 != SNAT_PROTOCOL_UDP
1036                           || (udp0->dst_port
1037                               != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1038                         next0 = SNAT_OUT2IN_NEXT_DROP;
1039                       goto trace0;
1040                     }
1041                   else
1042                     goto trace0;
1043                 }
1044
1045               /* Create session initiated by host from external network */
1046               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1047                                                      thread_index);
1048               if (!s0)
1049                 {
1050                   next0 = SNAT_OUT2IN_NEXT_DROP;
1051                   goto trace0;
1052                 }
1053             }
1054           else
1055             {
1056               if (PREDICT_FALSE (value0.value == ~0ULL))
1057                 {
1058                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1059                                       now, vm, node);
1060                   if (!s0)
1061                     next0 = SNAT_OUT2IN_NEXT_DROP;
1062                   goto trace0;
1063                 }
1064               else
1065                 {
1066                   s0 = pool_elt_at_index (
1067                     sm->per_thread_data[thread_index].sessions,
1068                     value0.value);
1069                 }
1070             }
1071
1072           old_addr0 = ip0->dst_address.as_u32;
1073           ip0->dst_address = s0->in2out.addr;
1074           new_addr0 = ip0->dst_address.as_u32;
1075           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1076
1077           sum0 = ip0->checksum;
1078           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1079                                  ip4_header_t,
1080                                  dst_address /* changed member */);
1081           ip0->checksum = ip_csum_fold (sum0);
1082
1083           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1084             {
1085               old_port0 = tcp0->dst_port;
1086               tcp0->dst_port = s0->in2out.port;
1087               new_port0 = tcp0->dst_port;
1088
1089               sum0 = tcp0->checksum;
1090               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1091                                      ip4_header_t,
1092                                      dst_address /* changed member */);
1093
1094               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1095                                      ip4_header_t /* cheat */,
1096                                      length /* changed member */);
1097               tcp0->checksum = ip_csum_fold(sum0);
1098             }
1099           else
1100             {
1101               old_port0 = udp0->dst_port;
1102               udp0->dst_port = s0->in2out.port;
1103               udp0->checksum = 0;
1104             }
1105
1106           /* Accounting */
1107           s0->last_heard = now;
1108           s0->total_pkts++;
1109           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1110           /* Per-user LRU list maintenance */
1111           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1112                              s0->per_user_index);
1113           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1114                               s0->per_user_list_head_index,
1115                               s0->per_user_index);
1116         trace0:
1117
1118           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1119                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1120             {
1121               snat_out2in_trace_t *t =
1122                  vlib_add_trace (vm, node, b0, sizeof (*t));
1123               t->sw_if_index = sw_if_index0;
1124               t->next_index = next0;
1125               t->session_index = ~0;
1126               if (s0)
1127                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1128             }
1129
1130           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1131
1132
1133           ip1 = vlib_buffer_get_current (b1);
1134           udp1 = ip4_next_header (ip1);
1135           tcp1 = (tcp_header_t *) udp1;
1136           icmp1 = (icmp46_header_t *) udp1;
1137
1138           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1139           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1140                                    sw_if_index1);
1141
1142           if (PREDICT_FALSE(ip1->ttl == 1))
1143             {
1144               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1145               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1146                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1147                                            0);
1148               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1149               goto trace1;
1150             }
1151
1152           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1153
1154           if (PREDICT_FALSE (proto1 == ~0))
1155             {
1156               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1157                                              thread_index, now, vm, node);
1158               if (!s1)
1159                 next1 = SNAT_OUT2IN_NEXT_DROP;
1160               goto trace1;
1161             }
1162
1163           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1164             {
1165               next1 = icmp_out2in_slow_path
1166                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1167                  next1, now, thread_index, &s1);
1168               goto trace1;
1169             }
1170
1171           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1172             {
1173               next1 = SNAT_OUT2IN_NEXT_REASS;
1174               goto trace1;
1175             }
1176
1177           key1.addr = ip1->dst_address;
1178           key1.port = udp1->dst_port;
1179           key1.protocol = proto1;
1180           key1.fib_index = rx_fib_index1;
1181
1182           kv1.key = key1.as_u64;
1183
1184           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1185                                       &kv1, &value1))
1186             {
1187               /* Try to match static mapping by external address and port,
1188                  destination address and port in packet */
1189               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0))
1190                 {
1191                   if (!sm->forwarding_enabled)
1192                     {
1193                       b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1194                       /*
1195                        * Send DHCP packets to the ipv4 stack, or we won't
1196                        * be able to use dhcp client on the outside interface
1197                        */
1198                       if (proto1 != SNAT_PROTOCOL_UDP
1199                           || (udp1->dst_port
1200                               != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1201                         next1 = SNAT_OUT2IN_NEXT_DROP;
1202                       goto trace1;
1203                     }
1204                   else
1205                     goto trace1;
1206                 }
1207
1208               /* Create session initiated by host from external network */
1209               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1210                                                      thread_index);
1211               if (!s1)
1212                 {
1213                   next1 = SNAT_OUT2IN_NEXT_DROP;
1214                   goto trace1;
1215                 }
1216             }
1217           else
1218             {
1219               if (PREDICT_FALSE (value1.value == ~0ULL))
1220                 {
1221                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1222                                       now, vm, node);
1223                   if (!s1)
1224                     next1 = SNAT_OUT2IN_NEXT_DROP;
1225                   goto trace1;
1226                 }
1227               else
1228                 {
1229                   s1 = pool_elt_at_index (
1230                     sm->per_thread_data[thread_index].sessions,
1231                     value1.value);
1232                 }
1233             }
1234
1235           old_addr1 = ip1->dst_address.as_u32;
1236           ip1->dst_address = s1->in2out.addr;
1237           new_addr1 = ip1->dst_address.as_u32;
1238           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1239
1240           sum1 = ip1->checksum;
1241           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1242                                  ip4_header_t,
1243                                  dst_address /* changed member */);
1244           ip1->checksum = ip_csum_fold (sum1);
1245
1246           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1247             {
1248               old_port1 = tcp1->dst_port;
1249               tcp1->dst_port = s1->in2out.port;
1250               new_port1 = tcp1->dst_port;
1251
1252               sum1 = tcp1->checksum;
1253               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1254                                      ip4_header_t,
1255                                      dst_address /* changed member */);
1256
1257               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1258                                      ip4_header_t /* cheat */,
1259                                      length /* changed member */);
1260               tcp1->checksum = ip_csum_fold(sum1);
1261             }
1262           else
1263             {
1264               old_port1 = udp1->dst_port;
1265               udp1->dst_port = s1->in2out.port;
1266               udp1->checksum = 0;
1267             }
1268
1269           /* Accounting */
1270           s1->last_heard = now;
1271           s1->total_pkts++;
1272           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1273           /* Per-user LRU list maintenance */
1274           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1275                              s1->per_user_index);
1276           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1277                               s1->per_user_list_head_index,
1278                               s1->per_user_index);
1279         trace1:
1280
1281           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1282                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1283             {
1284               snat_out2in_trace_t *t =
1285                  vlib_add_trace (vm, node, b1, sizeof (*t));
1286               t->sw_if_index = sw_if_index1;
1287               t->next_index = next1;
1288               t->session_index = ~0;
1289               if (s1)
1290                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1291             }
1292
1293           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1294
1295           /* verify speculative enqueues, maybe switch current next frame */
1296           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1297                                            to_next, n_left_to_next,
1298                                            bi0, bi1, next0, next1);
1299         }
1300
1301       while (n_left_from > 0 && n_left_to_next > 0)
1302         {
1303           u32 bi0;
1304           vlib_buffer_t * b0;
1305           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1306           u32 sw_if_index0;
1307           ip4_header_t * ip0;
1308           ip_csum_t sum0;
1309           u32 new_addr0, old_addr0;
1310           u16 new_port0, old_port0;
1311           udp_header_t * udp0;
1312           tcp_header_t * tcp0;
1313           icmp46_header_t * icmp0;
1314           snat_session_key_t key0, sm0;
1315           u32 rx_fib_index0;
1316           u32 proto0;
1317           snat_session_t * s0 = 0;
1318           clib_bihash_kv_8_8_t kv0, value0;
1319
1320           /* speculatively enqueue b0 to the current next frame */
1321           bi0 = from[0];
1322           to_next[0] = bi0;
1323           from += 1;
1324           to_next += 1;
1325           n_left_from -= 1;
1326           n_left_to_next -= 1;
1327
1328           b0 = vlib_get_buffer (vm, bi0);
1329
1330           vnet_buffer (b0)->snat.flags = 0;
1331
1332           ip0 = vlib_buffer_get_current (b0);
1333           udp0 = ip4_next_header (ip0);
1334           tcp0 = (tcp_header_t *) udp0;
1335           icmp0 = (icmp46_header_t *) udp0;
1336
1337           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1338           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1339                                    sw_if_index0);
1340
1341           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1342
1343           if (PREDICT_FALSE (proto0 == ~0))
1344             {
1345               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1346                                              thread_index, now, vm, node);
1347               if (!s0)
1348                 next0 = SNAT_OUT2IN_NEXT_DROP;
1349               goto trace00;
1350             }
1351
1352           if (PREDICT_FALSE(ip0->ttl == 1))
1353             {
1354               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1355               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1356                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1357                                            0);
1358               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1359               goto trace00;
1360             }
1361
1362           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1363             {
1364               next0 = icmp_out2in_slow_path
1365                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1366                  next0, now, thread_index, &s0);
1367               goto trace00;
1368             }
1369
1370           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1371             {
1372               next0 = SNAT_OUT2IN_NEXT_REASS;
1373               goto trace00;
1374             }
1375
1376           key0.addr = ip0->dst_address;
1377           key0.port = udp0->dst_port;
1378           key0.protocol = proto0;
1379           key0.fib_index = rx_fib_index0;
1380
1381           kv0.key = key0.as_u64;
1382
1383           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1384                                       &kv0, &value0))
1385             {
1386               /* Try to match static mapping by external address and port,
1387                  destination address and port in packet */
1388               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1389                 {
1390                   if (!sm->forwarding_enabled)
1391                     {
1392                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1393                       /*
1394                        * Send DHCP packets to the ipv4 stack, or we won't
1395                        * be able to use dhcp client on the outside interface
1396                        */
1397                       if (proto0 != SNAT_PROTOCOL_UDP
1398                           || (udp0->dst_port
1399                               != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1400                         next0 = SNAT_OUT2IN_NEXT_DROP;
1401                       goto trace00;
1402                     }
1403                   else
1404                     goto trace00;
1405                 }
1406
1407               /* Create session initiated by host from external network */
1408               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1409                                                      thread_index);
1410               if (!s0)
1411                 {
1412                   next0 = SNAT_OUT2IN_NEXT_DROP;
1413                   goto trace00;
1414                 }
1415             }
1416           else
1417             {
1418               if (PREDICT_FALSE (value0.value == ~0ULL))
1419                 {
1420                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1421                                       now, vm, node);
1422                   if (!s0)
1423                     next0 = SNAT_OUT2IN_NEXT_DROP;
1424                   goto trace00;
1425                 }
1426               else
1427                 {
1428                   s0 = pool_elt_at_index (
1429                     sm->per_thread_data[thread_index].sessions,
1430                     value0.value);
1431                 }
1432             }
1433
1434           old_addr0 = ip0->dst_address.as_u32;
1435           ip0->dst_address = s0->in2out.addr;
1436           new_addr0 = ip0->dst_address.as_u32;
1437           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1438
1439           sum0 = ip0->checksum;
1440           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1441                                  ip4_header_t,
1442                                  dst_address /* changed member */);
1443           ip0->checksum = ip_csum_fold (sum0);
1444
1445           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1446             {
1447               old_port0 = tcp0->dst_port;
1448               tcp0->dst_port = s0->in2out.port;
1449               new_port0 = tcp0->dst_port;
1450
1451               sum0 = tcp0->checksum;
1452               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1453                                      ip4_header_t,
1454                                      dst_address /* changed member */);
1455
1456               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1457                                      ip4_header_t /* cheat */,
1458                                      length /* changed member */);
1459               tcp0->checksum = ip_csum_fold(sum0);
1460             }
1461           else
1462             {
1463               old_port0 = udp0->dst_port;
1464               udp0->dst_port = s0->in2out.port;
1465               udp0->checksum = 0;
1466             }
1467
1468           /* Accounting */
1469           s0->last_heard = now;
1470           s0->total_pkts++;
1471           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1472           /* Per-user LRU list maintenance */
1473           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1474                              s0->per_user_index);
1475           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1476                               s0->per_user_list_head_index,
1477                               s0->per_user_index);
1478         trace00:
1479
1480           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1481                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1482             {
1483               snat_out2in_trace_t *t =
1484                  vlib_add_trace (vm, node, b0, sizeof (*t));
1485               t->sw_if_index = sw_if_index0;
1486               t->next_index = next0;
1487               t->session_index = ~0;
1488               if (s0)
1489                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1490             }
1491
1492           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1493
1494           /* verify speculative enqueue, maybe switch current next frame */
1495           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1496                                            to_next, n_left_to_next,
1497                                            bi0, next0);
1498         }
1499
1500       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1501     }
1502
1503   vlib_node_increment_counter (vm, snat_out2in_node.index,
1504                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1505                                pkts_processed);
1506   return frame->n_vectors;
1507 }
1508
1509 VLIB_REGISTER_NODE (snat_out2in_node) = {
1510   .function = snat_out2in_node_fn,
1511   .name = "nat44-out2in",
1512   .vector_size = sizeof (u32),
1513   .format_trace = format_snat_out2in_trace,
1514   .type = VLIB_NODE_TYPE_INTERNAL,
1515
1516   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1517   .error_strings = snat_out2in_error_strings,
1518
1519   .runtime_data_bytes = sizeof (snat_runtime_t),
1520
1521   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1522
1523   /* edit / add dispositions here */
1524   .next_nodes = {
1525     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1526     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1527     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1528     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1529   },
1530 };
1531 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1532
1533 static uword
1534 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1535                             vlib_node_runtime_t * node,
1536                             vlib_frame_t * frame)
1537 {
1538   u32 n_left_from, *from, *to_next;
1539   snat_out2in_next_t next_index;
1540   u32 pkts_processed = 0;
1541   snat_main_t *sm = &snat_main;
1542   f64 now = vlib_time_now (vm);
1543   u32 thread_index = vlib_get_thread_index ();
1544   snat_main_per_thread_data_t *per_thread_data =
1545     &sm->per_thread_data[thread_index];
1546   u32 *fragments_to_drop = 0;
1547   u32 *fragments_to_loopback = 0;
1548
1549   from = vlib_frame_vector_args (frame);
1550   n_left_from = frame->n_vectors;
1551   next_index = node->cached_next_index;
1552
1553   while (n_left_from > 0)
1554     {
1555       u32 n_left_to_next;
1556
1557       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1558
1559       while (n_left_from > 0 && n_left_to_next > 0)
1560        {
1561           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1562           vlib_buffer_t *b0;
1563           u32 next0;
1564           u8 cached0 = 0;
1565           ip4_header_t *ip0;
1566           nat_reass_ip4_t *reass0;
1567           udp_header_t * udp0;
1568           tcp_header_t * tcp0;
1569           snat_session_key_t key0, sm0;
1570           clib_bihash_kv_8_8_t kv0, value0;
1571           snat_session_t * s0 = 0;
1572           u16 old_port0, new_port0;
1573           ip_csum_t sum0;
1574
1575           /* speculatively enqueue b0 to the current next frame */
1576           bi0 = from[0];
1577           to_next[0] = bi0;
1578           from += 1;
1579           to_next += 1;
1580           n_left_from -= 1;
1581           n_left_to_next -= 1;
1582
1583           b0 = vlib_get_buffer (vm, bi0);
1584           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1585
1586           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1587           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1588                                                                sw_if_index0);
1589
1590           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1591             {
1592               next0 = SNAT_OUT2IN_NEXT_DROP;
1593               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1594               goto trace0;
1595             }
1596
1597           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1598           udp0 = ip4_next_header (ip0);
1599           tcp0 = (tcp_header_t *) udp0;
1600           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1601
1602           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1603                                                  ip0->dst_address,
1604                                                  ip0->fragment_id,
1605                                                  ip0->protocol,
1606                                                  1,
1607                                                  &fragments_to_drop);
1608
1609           if (PREDICT_FALSE (!reass0))
1610             {
1611               next0 = SNAT_OUT2IN_NEXT_DROP;
1612               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1613               goto trace0;
1614             }
1615
1616           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1617             {
1618               key0.addr = ip0->dst_address;
1619               key0.port = udp0->dst_port;
1620               key0.protocol = proto0;
1621               key0.fib_index = rx_fib_index0;
1622               kv0.key = key0.as_u64;
1623
1624               if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1625                 {
1626                   /* Try to match static mapping by external address and port,
1627                      destination address and port in packet */
1628                   if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1629                     {
1630                       if (!sm->forwarding_enabled)
1631                         {
1632                           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1633                           /*
1634                            * Send DHCP packets to the ipv4 stack, or we won't
1635                            * be able to use dhcp client on the outside interface
1636                            */
1637                           if (proto0 != SNAT_PROTOCOL_UDP
1638                               || (udp0->dst_port
1639                                   != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1640                             next0 = SNAT_OUT2IN_NEXT_DROP;
1641                           goto trace0;
1642                         }
1643                       else
1644                         goto trace0;
1645                     }
1646
1647                   /* Create session initiated by host from external network */
1648                   s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1649                                                          thread_index);
1650                   if (!s0)
1651                     {
1652                       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1653                       next0 = SNAT_OUT2IN_NEXT_DROP;
1654                       goto trace0;
1655                     }
1656                   reass0->sess_index = s0 - per_thread_data->sessions;
1657                   reass0->thread_index = thread_index;
1658                 }
1659               else
1660                 {
1661                   s0 = pool_elt_at_index (per_thread_data->sessions,
1662                                           value0.value);
1663                   reass0->sess_index = value0.value;
1664                 }
1665               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1666             }
1667           else
1668             {
1669               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1670                 {
1671                   if (nat_ip4_reass_add_fragment (reass0, bi0))
1672                     {
1673                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1674                       next0 = SNAT_OUT2IN_NEXT_DROP;
1675                       goto trace0;
1676                     }
1677                   cached0 = 1;
1678                   goto trace0;
1679                 }
1680               s0 = pool_elt_at_index (per_thread_data->sessions,
1681                                       reass0->sess_index);
1682             }
1683
1684           old_addr0 = ip0->dst_address.as_u32;
1685           ip0->dst_address = s0->in2out.addr;
1686           new_addr0 = ip0->dst_address.as_u32;
1687           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1688
1689           sum0 = ip0->checksum;
1690           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1691                                  ip4_header_t,
1692                                  dst_address /* changed member */);
1693           ip0->checksum = ip_csum_fold (sum0);
1694
1695           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1696             {
1697               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1698                 {
1699                   old_port0 = tcp0->dst_port;
1700                   tcp0->dst_port = s0->in2out.port;
1701                   new_port0 = tcp0->dst_port;
1702
1703                   sum0 = tcp0->checksum;
1704                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1705                                          ip4_header_t,
1706                                          dst_address /* changed member */);
1707
1708                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1709                                          ip4_header_t /* cheat */,
1710                                          length /* changed member */);
1711                   tcp0->checksum = ip_csum_fold(sum0);
1712                 }
1713               else
1714                 {
1715                   old_port0 = udp0->dst_port;
1716                   udp0->dst_port = s0->in2out.port;
1717                   udp0->checksum = 0;
1718                 }
1719             }
1720
1721           /* Accounting */
1722           s0->last_heard = now;
1723           s0->total_pkts++;
1724           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1725           /* Per-user LRU list maintenance */
1726           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1727                              s0->per_user_index);
1728           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1729                               s0->per_user_list_head_index,
1730                               s0->per_user_index);
1731
1732         trace0:
1733           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1734                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1735             {
1736               nat44_out2in_reass_trace_t *t =
1737                  vlib_add_trace (vm, node, b0, sizeof (*t));
1738               t->cached = cached0;
1739               t->sw_if_index = sw_if_index0;
1740               t->next_index = next0;
1741             }
1742
1743           if (cached0)
1744             {
1745               n_left_to_next++;
1746               to_next--;
1747             }
1748           else
1749             {
1750               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1751
1752               /* verify speculative enqueue, maybe switch current next frame */
1753               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1754                                                to_next, n_left_to_next,
1755                                                bi0, next0);
1756             }
1757
1758           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1759             {
1760               from = vlib_frame_vector_args (frame);
1761               u32 len = vec_len (fragments_to_loopback);
1762               if (len <= VLIB_FRAME_SIZE)
1763                 {
1764                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1765                   n_left_from = len;
1766                   vec_reset_length (fragments_to_loopback);
1767                 }
1768               else
1769                 {
1770                   clib_memcpy (from,
1771                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1772                                sizeof (u32) * VLIB_FRAME_SIZE);
1773                   n_left_from = VLIB_FRAME_SIZE;
1774                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1775                 }
1776             }
1777        }
1778
1779       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1780     }
1781
1782   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1783                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1784                                pkts_processed);
1785
1786   nat_send_all_to_node (vm, fragments_to_drop, node,
1787                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1788                         SNAT_OUT2IN_NEXT_DROP);
1789
1790   vec_free (fragments_to_drop);
1791   vec_free (fragments_to_loopback);
1792   return frame->n_vectors;
1793 }
1794
1795 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1796   .function = nat44_out2in_reass_node_fn,
1797   .name = "nat44-out2in-reass",
1798   .vector_size = sizeof (u32),
1799   .format_trace = format_nat44_out2in_reass_trace,
1800   .type = VLIB_NODE_TYPE_INTERNAL,
1801
1802   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1803   .error_strings = snat_out2in_error_strings,
1804
1805   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1806
1807   /* edit / add dispositions here */
1808   .next_nodes = {
1809     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1810     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1811     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1812     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1813   },
1814 };
1815 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1816                               nat44_out2in_reass_node_fn);
1817
1818 /**************************/
1819 /*** deterministic mode ***/
1820 /**************************/
1821 static uword
1822 snat_det_out2in_node_fn (vlib_main_t * vm,
1823                          vlib_node_runtime_t * node,
1824                          vlib_frame_t * frame)
1825 {
1826   u32 n_left_from, * from, * to_next;
1827   snat_out2in_next_t next_index;
1828   u32 pkts_processed = 0;
1829   snat_main_t * sm = &snat_main;
1830   u32 thread_index = vlib_get_thread_index ();
1831
1832   from = vlib_frame_vector_args (frame);
1833   n_left_from = frame->n_vectors;
1834   next_index = node->cached_next_index;
1835
1836   while (n_left_from > 0)
1837     {
1838       u32 n_left_to_next;
1839
1840       vlib_get_next_frame (vm, node, next_index,
1841                            to_next, n_left_to_next);
1842
1843       while (n_left_from >= 4 && n_left_to_next >= 2)
1844         {
1845           u32 bi0, bi1;
1846           vlib_buffer_t * b0, * b1;
1847           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1848           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1849           u32 sw_if_index0, sw_if_index1;
1850           ip4_header_t * ip0, * ip1;
1851           ip_csum_t sum0, sum1;
1852           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1853           u16 new_port0, old_port0, old_port1, new_port1;
1854           udp_header_t * udp0, * udp1;
1855           tcp_header_t * tcp0, * tcp1;
1856           u32 proto0, proto1;
1857           snat_det_out_key_t key0, key1;
1858           snat_det_map_t * dm0, * dm1;
1859           snat_det_session_t * ses0 = 0, * ses1 = 0;
1860           u32 rx_fib_index0, rx_fib_index1;
1861           icmp46_header_t * icmp0, * icmp1;
1862
1863           /* Prefetch next iteration. */
1864           {
1865             vlib_buffer_t * p2, * p3;
1866
1867             p2 = vlib_get_buffer (vm, from[2]);
1868             p3 = vlib_get_buffer (vm, from[3]);
1869
1870             vlib_prefetch_buffer_header (p2, LOAD);
1871             vlib_prefetch_buffer_header (p3, LOAD);
1872
1873             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1874             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1875           }
1876
1877           /* speculatively enqueue b0 and b1 to the current next frame */
1878           to_next[0] = bi0 = from[0];
1879           to_next[1] = bi1 = from[1];
1880           from += 2;
1881           to_next += 2;
1882           n_left_from -= 2;
1883           n_left_to_next -= 2;
1884
1885           b0 = vlib_get_buffer (vm, bi0);
1886           b1 = vlib_get_buffer (vm, bi1);
1887
1888           ip0 = vlib_buffer_get_current (b0);
1889           udp0 = ip4_next_header (ip0);
1890           tcp0 = (tcp_header_t *) udp0;
1891
1892           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1893
1894           if (PREDICT_FALSE(ip0->ttl == 1))
1895             {
1896               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1897               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1898                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1899                                            0);
1900               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1901               goto trace0;
1902             }
1903
1904           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1905
1906           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1907             {
1908               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1909               icmp0 = (icmp46_header_t *) udp0;
1910
1911               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1912                                   rx_fib_index0, node, next0, thread_index,
1913                                   &ses0, &dm0);
1914               goto trace0;
1915             }
1916
1917           key0.ext_host_addr = ip0->src_address;
1918           key0.ext_host_port = tcp0->src;
1919           key0.out_port = tcp0->dst;
1920
1921           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1922           if (PREDICT_FALSE(!dm0))
1923             {
1924               clib_warning("unknown dst address:  %U",
1925                            format_ip4_address, &ip0->dst_address);
1926               next0 = SNAT_OUT2IN_NEXT_DROP;
1927               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1928               goto trace0;
1929             }
1930
1931           snat_det_reverse(dm0, &ip0->dst_address,
1932                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1933
1934           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1935           if (PREDICT_FALSE(!ses0))
1936             {
1937               clib_warning("no match src %U:%d dst %U:%d for user %U",
1938                            format_ip4_address, &ip0->src_address,
1939                            clib_net_to_host_u16 (tcp0->src),
1940                            format_ip4_address, &ip0->dst_address,
1941                            clib_net_to_host_u16 (tcp0->dst),
1942                            format_ip4_address, &new_addr0);
1943               next0 = SNAT_OUT2IN_NEXT_DROP;
1944               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1945               goto trace0;
1946             }
1947           new_port0 = ses0->in_port;
1948
1949           old_addr0 = ip0->dst_address;
1950           ip0->dst_address = new_addr0;
1951           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1952
1953           sum0 = ip0->checksum;
1954           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1955                                  ip4_header_t,
1956                                  dst_address /* changed member */);
1957           ip0->checksum = ip_csum_fold (sum0);
1958
1959           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1960             {
1961               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1962                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1963               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1964                 snat_det_ses_close(dm0, ses0);
1965
1966               old_port0 = tcp0->dst;
1967               tcp0->dst = new_port0;
1968
1969               sum0 = tcp0->checksum;
1970               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1971                                      ip4_header_t,
1972                                      dst_address /* changed member */);
1973
1974               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1975                                      ip4_header_t /* cheat */,
1976                                      length /* changed member */);
1977               tcp0->checksum = ip_csum_fold(sum0);
1978             }
1979           else
1980             {
1981               old_port0 = udp0->dst_port;
1982               udp0->dst_port = new_port0;
1983               udp0->checksum = 0;
1984             }
1985
1986         trace0:
1987
1988           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1989                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1990             {
1991               snat_out2in_trace_t *t =
1992                  vlib_add_trace (vm, node, b0, sizeof (*t));
1993               t->sw_if_index = sw_if_index0;
1994               t->next_index = next0;
1995               t->session_index = ~0;
1996               if (ses0)
1997                 t->session_index = ses0 - dm0->sessions;
1998             }
1999
2000           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2001
2002           b1 = vlib_get_buffer (vm, bi1);
2003
2004           ip1 = vlib_buffer_get_current (b1);
2005           udp1 = ip4_next_header (ip1);
2006           tcp1 = (tcp_header_t *) udp1;
2007
2008           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2009
2010           if (PREDICT_FALSE(ip1->ttl == 1))
2011             {
2012               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2013               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2014                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2015                                            0);
2016               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2017               goto trace1;
2018             }
2019
2020           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2021
2022           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2023             {
2024               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2025               icmp1 = (icmp46_header_t *) udp1;
2026
2027               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2028                                   rx_fib_index1, node, next1, thread_index,
2029                                   &ses1, &dm1);
2030               goto trace1;
2031             }
2032
2033           key1.ext_host_addr = ip1->src_address;
2034           key1.ext_host_port = tcp1->src;
2035           key1.out_port = tcp1->dst;
2036
2037           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2038           if (PREDICT_FALSE(!dm1))
2039             {
2040               clib_warning("unknown dst address:  %U",
2041                            format_ip4_address, &ip1->dst_address);
2042               next1 = SNAT_OUT2IN_NEXT_DROP;
2043               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2044               goto trace1;
2045             }
2046
2047           snat_det_reverse(dm1, &ip1->dst_address,
2048                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
2049
2050           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2051           if (PREDICT_FALSE(!ses1))
2052             {
2053               clib_warning("no match src %U:%d dst %U:%d for user %U",
2054                            format_ip4_address, &ip1->src_address,
2055                            clib_net_to_host_u16 (tcp1->src),
2056                            format_ip4_address, &ip1->dst_address,
2057                            clib_net_to_host_u16 (tcp1->dst),
2058                            format_ip4_address, &new_addr1);
2059               next1 = SNAT_OUT2IN_NEXT_DROP;
2060               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2061               goto trace1;
2062             }
2063           new_port1 = ses1->in_port;
2064
2065           old_addr1 = ip1->dst_address;
2066           ip1->dst_address = new_addr1;
2067           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2068
2069           sum1 = ip1->checksum;
2070           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2071                                  ip4_header_t,
2072                                  dst_address /* changed member */);
2073           ip1->checksum = ip_csum_fold (sum1);
2074
2075           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2076             {
2077               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2078                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2079               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2080                 snat_det_ses_close(dm1, ses1);
2081
2082               old_port1 = tcp1->dst;
2083               tcp1->dst = new_port1;
2084
2085               sum1 = tcp1->checksum;
2086               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2087                                      ip4_header_t,
2088                                      dst_address /* changed member */);
2089
2090               sum1 = ip_csum_update (sum1, old_port1, new_port1,
2091                                      ip4_header_t /* cheat */,
2092                                      length /* changed member */);
2093               tcp1->checksum = ip_csum_fold(sum1);
2094             }
2095           else
2096             {
2097               old_port1 = udp1->dst_port;
2098               udp1->dst_port = new_port1;
2099               udp1->checksum = 0;
2100             }
2101
2102         trace1:
2103
2104           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2105                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2106             {
2107               snat_out2in_trace_t *t =
2108                  vlib_add_trace (vm, node, b1, sizeof (*t));
2109               t->sw_if_index = sw_if_index1;
2110               t->next_index = next1;
2111               t->session_index = ~0;
2112               if (ses1)
2113                 t->session_index = ses1 - dm1->sessions;
2114             }
2115
2116           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2117
2118           /* verify speculative enqueues, maybe switch current next frame */
2119           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2120                                            to_next, n_left_to_next,
2121                                            bi0, bi1, next0, next1);
2122          }
2123
2124       while (n_left_from > 0 && n_left_to_next > 0)
2125         {
2126           u32 bi0;
2127           vlib_buffer_t * b0;
2128           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2129           u32 sw_if_index0;
2130           ip4_header_t * ip0;
2131           ip_csum_t sum0;
2132           ip4_address_t new_addr0, old_addr0;
2133           u16 new_port0, old_port0;
2134           udp_header_t * udp0;
2135           tcp_header_t * tcp0;
2136           u32 proto0;
2137           snat_det_out_key_t key0;
2138           snat_det_map_t * dm0;
2139           snat_det_session_t * ses0 = 0;
2140           u32 rx_fib_index0;
2141           icmp46_header_t * icmp0;
2142
2143           /* speculatively enqueue b0 to the current next frame */
2144           bi0 = from[0];
2145           to_next[0] = bi0;
2146           from += 1;
2147           to_next += 1;
2148           n_left_from -= 1;
2149           n_left_to_next -= 1;
2150
2151           b0 = vlib_get_buffer (vm, bi0);
2152
2153           ip0 = vlib_buffer_get_current (b0);
2154           udp0 = ip4_next_header (ip0);
2155           tcp0 = (tcp_header_t *) udp0;
2156
2157           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2158
2159           if (PREDICT_FALSE(ip0->ttl == 1))
2160             {
2161               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2162               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2163                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2164                                            0);
2165               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2166               goto trace00;
2167             }
2168
2169           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2170
2171           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2172             {
2173               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2174               icmp0 = (icmp46_header_t *) udp0;
2175
2176               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2177                                   rx_fib_index0, node, next0, thread_index,
2178                                   &ses0, &dm0);
2179               goto trace00;
2180             }
2181
2182           key0.ext_host_addr = ip0->src_address;
2183           key0.ext_host_port = tcp0->src;
2184           key0.out_port = tcp0->dst;
2185
2186           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2187           if (PREDICT_FALSE(!dm0))
2188             {
2189               clib_warning("unknown dst address:  %U",
2190                            format_ip4_address, &ip0->dst_address);
2191               next0 = SNAT_OUT2IN_NEXT_DROP;
2192               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2193               goto trace00;
2194             }
2195
2196           snat_det_reverse(dm0, &ip0->dst_address,
2197                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
2198
2199           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2200           if (PREDICT_FALSE(!ses0))
2201             {
2202               clib_warning("no match src %U:%d dst %U:%d for user %U",
2203                            format_ip4_address, &ip0->src_address,
2204                            clib_net_to_host_u16 (tcp0->src),
2205                            format_ip4_address, &ip0->dst_address,
2206                            clib_net_to_host_u16 (tcp0->dst),
2207                            format_ip4_address, &new_addr0);
2208               next0 = SNAT_OUT2IN_NEXT_DROP;
2209               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2210               goto trace00;
2211             }
2212           new_port0 = ses0->in_port;
2213
2214           old_addr0 = ip0->dst_address;
2215           ip0->dst_address = new_addr0;
2216           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2217
2218           sum0 = ip0->checksum;
2219           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2220                                  ip4_header_t,
2221                                  dst_address /* changed member */);
2222           ip0->checksum = ip_csum_fold (sum0);
2223
2224           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2225             {
2226               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2227                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2228               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2229                 snat_det_ses_close(dm0, ses0);
2230
2231               old_port0 = tcp0->dst;
2232               tcp0->dst = new_port0;
2233
2234               sum0 = tcp0->checksum;
2235               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2236                                      ip4_header_t,
2237                                      dst_address /* changed member */);
2238
2239               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2240                                      ip4_header_t /* cheat */,
2241                                      length /* changed member */);
2242               tcp0->checksum = ip_csum_fold(sum0);
2243             }
2244           else
2245             {
2246               old_port0 = udp0->dst_port;
2247               udp0->dst_port = new_port0;
2248               udp0->checksum = 0;
2249             }
2250
2251         trace00:
2252
2253           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2254                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2255             {
2256               snat_out2in_trace_t *t =
2257                  vlib_add_trace (vm, node, b0, sizeof (*t));
2258               t->sw_if_index = sw_if_index0;
2259               t->next_index = next0;
2260               t->session_index = ~0;
2261               if (ses0)
2262                 t->session_index = ses0 - dm0->sessions;
2263             }
2264
2265           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2266
2267           /* verify speculative enqueue, maybe switch current next frame */
2268           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2269                                            to_next, n_left_to_next,
2270                                            bi0, next0);
2271         }
2272
2273       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2274     }
2275
2276   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2277                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2278                                pkts_processed);
2279   return frame->n_vectors;
2280 }
2281
2282 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2283   .function = snat_det_out2in_node_fn,
2284   .name = "nat44-det-out2in",
2285   .vector_size = sizeof (u32),
2286   .format_trace = format_snat_out2in_trace,
2287   .type = VLIB_NODE_TYPE_INTERNAL,
2288
2289   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2290   .error_strings = snat_out2in_error_strings,
2291
2292   .runtime_data_bytes = sizeof (snat_runtime_t),
2293
2294   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2295
2296   /* edit / add dispositions here */
2297   .next_nodes = {
2298     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2299     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2300     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2301     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2302   },
2303 };
2304 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2305
2306 /**
2307  * Get address and port values to be used for ICMP packet translation
2308  * and create session if needed
2309  *
2310  * @param[in,out] sm             NAT main
2311  * @param[in,out] node           NAT node runtime
2312  * @param[in] thread_index       thread index
2313  * @param[in,out] b0             buffer containing packet to be translated
2314  * @param[out] p_proto           protocol used for matching
2315  * @param[out] p_value           address and port after NAT translation
2316  * @param[out] p_dont_translate  if packet should not be translated
2317  * @param d                      optional parameter
2318  * @param e                      optional parameter
2319  */
2320 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2321                           u32 thread_index, vlib_buffer_t *b0,
2322                           ip4_header_t *ip0, u8 *p_proto,
2323                           snat_session_key_t *p_value,
2324                           u8 *p_dont_translate, void *d, void *e)
2325 {
2326   icmp46_header_t *icmp0;
2327   u32 sw_if_index0;
2328   u8 protocol;
2329   snat_det_out_key_t key0;
2330   u8 dont_translate = 0;
2331   u32 next0 = ~0;
2332   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2333   ip4_header_t *inner_ip0;
2334   void *l4_header = 0;
2335   icmp46_header_t *inner_icmp0;
2336   snat_det_map_t * dm0 = 0;
2337   ip4_address_t new_addr0 = {{0}};
2338   snat_det_session_t * ses0 = 0;
2339   ip4_address_t out_addr;
2340
2341   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2342   echo0 = (icmp_echo_header_t *)(icmp0+1);
2343   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2344
2345   if (!icmp_is_error_message (icmp0))
2346     {
2347       protocol = SNAT_PROTOCOL_ICMP;
2348       key0.ext_host_addr = ip0->src_address;
2349       key0.ext_host_port = 0;
2350       key0.out_port = echo0->identifier;
2351       out_addr = ip0->dst_address;
2352     }
2353   else
2354     {
2355       inner_ip0 = (ip4_header_t *)(echo0+1);
2356       l4_header = ip4_next_header (inner_ip0);
2357       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2358       key0.ext_host_addr = inner_ip0->dst_address;
2359       out_addr = inner_ip0->src_address;
2360       switch (protocol)
2361         {
2362         case SNAT_PROTOCOL_ICMP:
2363           inner_icmp0 = (icmp46_header_t*)l4_header;
2364           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2365           key0.ext_host_port = 0;
2366           key0.out_port = inner_echo0->identifier;
2367           break;
2368         case SNAT_PROTOCOL_UDP:
2369         case SNAT_PROTOCOL_TCP:
2370           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2371           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2372           break;
2373         default:
2374           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2375           next0 = SNAT_OUT2IN_NEXT_DROP;
2376           goto out;
2377         }
2378     }
2379
2380   dm0 = snat_det_map_by_out(sm, &out_addr);
2381   if (PREDICT_FALSE(!dm0))
2382     {
2383       /* Don't NAT packet aimed at the intfc address */
2384       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2385                                           ip0->dst_address.as_u32)))
2386         {
2387           dont_translate = 1;
2388           goto out;
2389         }
2390       clib_warning("unknown dst address:  %U",
2391                    format_ip4_address, &ip0->dst_address);
2392       goto out;
2393     }
2394
2395   snat_det_reverse(dm0, &ip0->dst_address,
2396                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2397
2398   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2399   if (PREDICT_FALSE(!ses0))
2400     {
2401       /* Don't NAT packet aimed at the intfc address */
2402       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2403                                           ip0->dst_address.as_u32)))
2404         {
2405           dont_translate = 1;
2406           goto out;
2407         }
2408       clib_warning("no match src %U:%d dst %U:%d for user %U",
2409                    format_ip4_address, &key0.ext_host_addr,
2410                    clib_net_to_host_u16 (key0.ext_host_port),
2411                    format_ip4_address, &out_addr,
2412                    clib_net_to_host_u16 (key0.out_port),
2413                    format_ip4_address, &new_addr0);
2414       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2415       next0 = SNAT_OUT2IN_NEXT_DROP;
2416       goto out;
2417     }
2418
2419   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2420                     !icmp_is_error_message (icmp0)))
2421     {
2422       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2423       next0 = SNAT_OUT2IN_NEXT_DROP;
2424       goto out;
2425     }
2426
2427   goto out;
2428
2429 out:
2430   *p_proto = protocol;
2431   if (ses0)
2432     {
2433       p_value->addr = new_addr0;
2434       p_value->fib_index = sm->inside_fib_index;
2435       p_value->port = ses0->in_port;
2436     }
2437   *p_dont_translate = dont_translate;
2438   if (d)
2439     *(snat_det_session_t**)d = ses0;
2440   if (e)
2441     *(snat_det_map_t**)e = dm0;
2442   return next0;
2443 }
2444
2445 /**********************/
2446 /*** worker handoff ***/
2447 /**********************/
2448 static uword
2449 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2450                                vlib_node_runtime_t * node,
2451                                vlib_frame_t * frame)
2452 {
2453   snat_main_t *sm = &snat_main;
2454   vlib_thread_main_t *tm = vlib_get_thread_main ();
2455   u32 n_left_from, *from, *to_next = 0;
2456   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2457   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2458     = 0;
2459   vlib_frame_queue_elt_t *hf = 0;
2460   vlib_frame_t *f = 0;
2461   int i;
2462   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2463   u32 next_worker_index = 0;
2464   u32 current_worker_index = ~0;
2465   u32 thread_index = vlib_get_thread_index ();
2466
2467   ASSERT (vec_len (sm->workers));
2468
2469   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2470     {
2471       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2472
2473       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2474                                sm->first_worker_index + sm->num_workers - 1,
2475                                (vlib_frame_queue_t *) (~0));
2476     }
2477
2478   from = vlib_frame_vector_args (frame);
2479   n_left_from = frame->n_vectors;
2480
2481   while (n_left_from > 0)
2482     {
2483       u32 bi0;
2484       vlib_buffer_t *b0;
2485       u32 sw_if_index0;
2486       u32 rx_fib_index0;
2487       ip4_header_t * ip0;
2488       u8 do_handoff;
2489
2490       bi0 = from[0];
2491       from += 1;
2492       n_left_from -= 1;
2493
2494       b0 = vlib_get_buffer (vm, bi0);
2495
2496       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2497       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2498
2499       ip0 = vlib_buffer_get_current (b0);
2500
2501       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2502
2503       if (PREDICT_FALSE (next_worker_index != thread_index))
2504         {
2505           do_handoff = 1;
2506
2507           if (next_worker_index != current_worker_index)
2508             {
2509               if (hf)
2510                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2511
2512               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2513                                                       next_worker_index,
2514                                                       handoff_queue_elt_by_worker_index);
2515
2516               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2517               to_next_worker = &hf->buffer_index[hf->n_vectors];
2518               current_worker_index = next_worker_index;
2519             }
2520
2521           /* enqueue to correct worker thread */
2522           to_next_worker[0] = bi0;
2523           to_next_worker++;
2524           n_left_to_next_worker--;
2525
2526           if (n_left_to_next_worker == 0)
2527             {
2528               hf->n_vectors = VLIB_FRAME_SIZE;
2529               vlib_put_frame_queue_elt (hf);
2530               current_worker_index = ~0;
2531               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2532               hf = 0;
2533             }
2534         }
2535       else
2536         {
2537           do_handoff = 0;
2538           /* if this is 1st frame */
2539           if (!f)
2540             {
2541               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2542               to_next = vlib_frame_vector_args (f);
2543             }
2544
2545           to_next[0] = bi0;
2546           to_next += 1;
2547           f->n_vectors++;
2548         }
2549
2550       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2551                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2552         {
2553           snat_out2in_worker_handoff_trace_t *t =
2554             vlib_add_trace (vm, node, b0, sizeof (*t));
2555           t->next_worker_index = next_worker_index;
2556           t->do_handoff = do_handoff;
2557         }
2558     }
2559
2560   if (f)
2561     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2562
2563   if (hf)
2564     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2565
2566   /* Ship frames to the worker nodes */
2567   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2568     {
2569       if (handoff_queue_elt_by_worker_index[i])
2570         {
2571           hf = handoff_queue_elt_by_worker_index[i];
2572           /*
2573            * It works better to let the handoff node
2574            * rate-adapt, always ship the handoff queue element.
2575            */
2576           if (1 || hf->n_vectors == hf->last_n_vectors)
2577             {
2578               vlib_put_frame_queue_elt (hf);
2579               handoff_queue_elt_by_worker_index[i] = 0;
2580             }
2581           else
2582             hf->last_n_vectors = hf->n_vectors;
2583         }
2584       congested_handoff_queue_by_worker_index[i] =
2585         (vlib_frame_queue_t *) (~0);
2586     }
2587   hf = 0;
2588   current_worker_index = ~0;
2589   return frame->n_vectors;
2590 }
2591
2592 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2593   .function = snat_out2in_worker_handoff_fn,
2594   .name = "nat44-out2in-worker-handoff",
2595   .vector_size = sizeof (u32),
2596   .format_trace = format_snat_out2in_worker_handoff_trace,
2597   .type = VLIB_NODE_TYPE_INTERNAL,
2598
2599   .n_next_nodes = 1,
2600
2601   .next_nodes = {
2602     [0] = "error-drop",
2603   },
2604 };
2605
2606 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2607
2608 static uword
2609 snat_out2in_fast_node_fn (vlib_main_t * vm,
2610                           vlib_node_runtime_t * node,
2611                           vlib_frame_t * frame)
2612 {
2613   u32 n_left_from, * from, * to_next;
2614   snat_out2in_next_t next_index;
2615   u32 pkts_processed = 0;
2616   snat_main_t * sm = &snat_main;
2617
2618   from = vlib_frame_vector_args (frame);
2619   n_left_from = frame->n_vectors;
2620   next_index = node->cached_next_index;
2621
2622   while (n_left_from > 0)
2623     {
2624       u32 n_left_to_next;
2625
2626       vlib_get_next_frame (vm, node, next_index,
2627                            to_next, n_left_to_next);
2628
2629       while (n_left_from > 0 && n_left_to_next > 0)
2630         {
2631           u32 bi0;
2632           vlib_buffer_t * b0;
2633           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2634           u32 sw_if_index0;
2635           ip4_header_t * ip0;
2636           ip_csum_t sum0;
2637           u32 new_addr0, old_addr0;
2638           u16 new_port0, old_port0;
2639           udp_header_t * udp0;
2640           tcp_header_t * tcp0;
2641           icmp46_header_t * icmp0;
2642           snat_session_key_t key0, sm0;
2643           u32 proto0;
2644           u32 rx_fib_index0;
2645
2646           /* speculatively enqueue b0 to the current next frame */
2647           bi0 = from[0];
2648           to_next[0] = bi0;
2649           from += 1;
2650           to_next += 1;
2651           n_left_from -= 1;
2652           n_left_to_next -= 1;
2653
2654           b0 = vlib_get_buffer (vm, bi0);
2655
2656           ip0 = vlib_buffer_get_current (b0);
2657           udp0 = ip4_next_header (ip0);
2658           tcp0 = (tcp_header_t *) udp0;
2659           icmp0 = (icmp46_header_t *) udp0;
2660
2661           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2662           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2663
2664           vnet_feature_next (sw_if_index0, &next0, b0);
2665
2666           if (PREDICT_FALSE(ip0->ttl == 1))
2667             {
2668               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2669               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2670                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2671                                            0);
2672               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2673               goto trace00;
2674             }
2675
2676           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2677
2678           if (PREDICT_FALSE (proto0 == ~0))
2679               goto trace00;
2680
2681           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2682             {
2683               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2684                                   rx_fib_index0, node, next0, ~0, 0, 0);
2685               goto trace00;
2686             }
2687
2688           key0.addr = ip0->dst_address;
2689           key0.port = udp0->dst_port;
2690           key0.fib_index = rx_fib_index0;
2691
2692           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2693             {
2694               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2695               goto trace00;
2696             }
2697
2698           new_addr0 = sm0.addr.as_u32;
2699           new_port0 = sm0.port;
2700           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2701           old_addr0 = ip0->dst_address.as_u32;
2702           ip0->dst_address.as_u32 = new_addr0;
2703
2704           sum0 = ip0->checksum;
2705           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2706                                  ip4_header_t,
2707                                  dst_address /* changed member */);
2708           ip0->checksum = ip_csum_fold (sum0);
2709
2710           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2711             {
2712                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2713                 {
2714                   old_port0 = tcp0->dst_port;
2715                   tcp0->dst_port = new_port0;
2716
2717                   sum0 = tcp0->checksum;
2718                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2719                                          ip4_header_t,
2720                                          dst_address /* changed member */);
2721
2722                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2723                                          ip4_header_t /* cheat */,
2724                                          length /* changed member */);
2725                   tcp0->checksum = ip_csum_fold(sum0);
2726                 }
2727               else
2728                 {
2729                   old_port0 = udp0->dst_port;
2730                   udp0->dst_port = new_port0;
2731                   udp0->checksum = 0;
2732                 }
2733             }
2734           else
2735             {
2736               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2737                 {
2738                   sum0 = tcp0->checksum;
2739                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2740                                          ip4_header_t,
2741                                          dst_address /* changed member */);
2742
2743                   tcp0->checksum = ip_csum_fold(sum0);
2744                 }
2745             }
2746
2747         trace00:
2748
2749           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2750                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2751             {
2752               snat_out2in_trace_t *t =
2753                  vlib_add_trace (vm, node, b0, sizeof (*t));
2754               t->sw_if_index = sw_if_index0;
2755               t->next_index = next0;
2756             }
2757
2758           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2759
2760           /* verify speculative enqueue, maybe switch current next frame */
2761           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2762                                            to_next, n_left_to_next,
2763                                            bi0, next0);
2764         }
2765
2766       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2767     }
2768
2769   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2770                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2771                                pkts_processed);
2772   return frame->n_vectors;
2773 }
2774
2775 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2776   .function = snat_out2in_fast_node_fn,
2777   .name = "nat44-out2in-fast",
2778   .vector_size = sizeof (u32),
2779   .format_trace = format_snat_out2in_fast_trace,
2780   .type = VLIB_NODE_TYPE_INTERNAL,
2781
2782   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2783   .error_strings = snat_out2in_error_strings,
2784
2785   .runtime_data_bytes = sizeof (snat_runtime_t),
2786
2787   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2788
2789   /* edit / add dispositions here */
2790   .next_nodes = {
2791     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2792     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2793     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2794     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2795   },
2796 };
2797 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);