src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27 #include <snat/snat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     SNAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out SNAT session key.
 121  * @param out2in Out2in SNAT session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140
 141   user_key.addr = in2out.addr;
 142   user_key.fib_index = in2out.fib_index;
 143   kv0.key = user_key.as_u64;
 144
 145   /* Ever heard of the "user" = inside ip4 address before? */
 146   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 147     {
 148       /* no, make a new one */
 149       pool_get (sm->per_thread_data[thread_index].users, u);
 150       memset (u, 0, sizeof (*u));
 151       u->addr = in2out.addr;
 152       u->fib_index = in2out.fib_index;
 153
 154       pool_get (sm->per_thread_data[thread_index].list_pool,
 155                 per_user_list_head_elt);
 156
 157       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 158         sm->per_thread_data[thread_index].list_pool;
 159
 160       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 161                        u->sessions_per_user_list_head_index);
 162
 163       kv0.value = u - sm->per_thread_data[thread_index].users;
 164
 165       /* add user */
 166       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 167
 168       /* add non-traslated packets worker lookup */
 169       kv0.value = thread_index;
 170       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 171     }
 172   else
 173     {
 174       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 175                              value0.value);
 176     }
 177
 178   pool_get (sm->per_thread_data[thread_index].sessions, s);
 179   memset (s, 0, sizeof (*s));
 180
 181   s->outside_address_index = ~0;
 182   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 183   u->nstaticsessions++;
 184
 185   /* Create list elts */
 186   pool_get (sm->per_thread_data[thread_index].list_pool,
 187             per_user_translation_list_elt);
 188   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 189                    per_user_translation_list_elt -
 190                    sm->per_thread_data[thread_index].list_pool);
 191
 192   per_user_translation_list_elt->value =
 193     s - sm->per_thread_data[thread_index].sessions;
 194   s->per_user_index =
 195     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 196   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 197
 198   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 199                       s->per_user_list_head_index,
 200                       per_user_translation_list_elt -
 201                       sm->per_thread_data[thread_index].list_pool);
 202
 203   s->in2out = in2out;
 204   s->out2in = out2in;
 205   s->in2out.protocol = out2in.protocol;
 206
 207   /* Add to translation hashes */
 208   kv0.key = s->in2out.as_u64;
 209   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 210   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 211       clib_warning ("in2out key add failed");
 212
 213   kv0.key = s->out2in.as_u64;
 214   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 215
 216   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 217       clib_warning ("out2in key add failed");
 218
 219   /* log NAT event */
 220   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 221                                       s->out2in.addr.as_u32,
 222                                       s->in2out.protocol,
 223                                       s->in2out.port,
 224                                       s->out2in.port,
 225                                       s->in2out.fib_index);
 226    return s;
 227 }
 228
 229 static_always_inline
 230 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 231                                  snat_session_key_t *p_key0)
 232 {
 233   icmp46_header_t *icmp0;
 234   snat_session_key_t key0;
 235   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 236   ip4_header_t *inner_ip0;
 237   void *l4_header = 0;
 238   icmp46_header_t *inner_icmp0;
 239
 240   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 241   echo0 = (icmp_echo_header_t *)(icmp0+1);
 242
 243   if (!icmp_is_error_message (icmp0))
 244     {
 245       key0.protocol = SNAT_PROTOCOL_ICMP;
 246       key0.addr = ip0->dst_address;
 247       key0.port = echo0->identifier;
 248     }
 249   else
 250     {
 251       inner_ip0 = (ip4_header_t *)(echo0+1);
 252       l4_header = ip4_next_header (inner_ip0);
 253       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 254       key0.addr = inner_ip0->src_address;
 255       switch (key0.protocol)
 256         {
 257         case SNAT_PROTOCOL_ICMP:
 258           inner_icmp0 = (icmp46_header_t*)l4_header;
 259           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 260           key0.port = inner_echo0->identifier;
 261           break;
 262         case SNAT_PROTOCOL_UDP:
 263         case SNAT_PROTOCOL_TCP:
 264           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 265           break;
 266         default:
 267           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 268         }
 269     }
 270   *p_key0 = key0;
 271   return -1; /* success */
 272 }
 273
 274 /**
 275  * Get address and port values to be used for packet SNAT translation
 276  * and create session if needed
 277  *
 278  * @param[in,out] sm             SNAT main
 279  * @param[in,out] node           SNAT node runtime
 280  * @param[in] thread_index       thread index
 281  * @param[in,out] b0             buffer containing packet to be translated
 282  * @param[out] p_proto           protocol used for matching
 283  * @param[out] p_value           address and port after NAT translation
 284  * @param[out] p_dont_translate  if packet should not be translated
 285  * @param d                      optional parameter
 286  */
 287 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 288                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 289                            snat_session_key_t *p_value,
 290                            u8 *p_dont_translate, void *d)
 291 {
 292   ip4_header_t *ip0;
 293   icmp46_header_t *icmp0;
 294   u32 sw_if_index0;
 295   u32 rx_fib_index0;
 296   snat_session_key_t key0;
 297   snat_session_key_t sm0;
 298   snat_session_t *s0 = 0;
 299   u8 dont_translate = 0;
 300   clib_bihash_kv_8_8_t kv0, value0;
 301   u32 next0 = ~0;
 302   int err;
 303
 304   ip0 = vlib_buffer_get_current (b0);
 305   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 306   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 307   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 308
 309   err = icmp_get_key (ip0, &key0);
 310   if (err != -1)
 311     {
 312       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 313       next0 = SNAT_OUT2IN_NEXT_DROP;
 314       goto out;
 315     }
 316   key0.fib_index = rx_fib_index0;
 317
 318   kv0.key = key0.as_u64;
 319
 320   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 321     {
 322       /* Try to match static mapping by external address and port,
 323          destination address and port in packet */
 324       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 325         {
 326           /* Don't NAT packet aimed at the intfc address */
 327           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 328                                               ip0->dst_address.as_u32)))
 329             {
 330               dont_translate = 1;
 331               goto out;
 332             }
 333           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 334           next0 = SNAT_OUT2IN_NEXT_DROP;
 335           goto out;
 336         }
 337
 338       if (icmp_is_error_message (icmp0))
 339         {
 340           next0 = SNAT_OUT2IN_NEXT_DROP;
 341           goto out;
 342         }
 343
 344       /* Create session initiated by host from external network */
 345       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 346                                              node, thread_index);
 347
 348       if (!s0)
 349         {
 350           next0 = SNAT_OUT2IN_NEXT_DROP;
 351           goto out;
 352         }
 353     }
 354   else
 355     s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 356                             value0.value);
 357
 358   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 359                     !icmp_is_error_message (icmp0)))
 360     {
 361       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 362       next0 = SNAT_OUT2IN_NEXT_DROP;
 363       goto out;
 364     }
 365
 366 out:
 367   *p_proto = key0.protocol;
 368   if (s0)
 369     *p_value = s0->in2out;
 370   *p_dont_translate = dont_translate;
 371   if (d)
 372     *(snat_session_t**)d = s0;
 373   return next0;
 374 }
 375
 376 /**
 377  * Get address and port values to be used for packet SNAT translation
 378  *
 379  * @param[in] sm                 SNAT main
 380  * @param[in,out] node           SNAT node runtime
 381  * @param[in] thread_index       thread index
 382  * @param[in,out] b0             buffer containing packet to be translated
 383  * @param[out] p_proto           protocol used for matching
 384  * @param[out] p_value           address and port after NAT translation
 385  * @param[out] p_dont_translate  if packet should not be translated
 386  * @param d                      optional parameter
 387  */
 388 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 389                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 390                            snat_session_key_t *p_value,
 391                            u8 *p_dont_translate, void *d)
 392 {
 393   ip4_header_t *ip0;
 394   icmp46_header_t *icmp0;
 395   u32 sw_if_index0;
 396   u32 rx_fib_index0;
 397   snat_session_key_t key0;
 398   snat_session_key_t sm0;
 399   u8 dont_translate = 0;
 400   u32 next0 = ~0;
 401   int err;
 402
 403   ip0 = vlib_buffer_get_current (b0);
 404   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 405   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 406   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 407
 408   err = icmp_get_key (ip0, &key0);
 409   if (err != -1)
 410     {
 411       b0->error = node->errors[err];
 412       next0 = SNAT_OUT2IN_NEXT_DROP;
 413       goto out2;
 414     }
 415   key0.fib_index = rx_fib_index0;
 416
 417   if (snat_static_mapping_match(sm, key0, &sm0, 1))
 418     {
 419       /* Don't NAT packet aimed at the intfc address */
 420       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 421         {
 422           dont_translate = 1;
 423           goto out;
 424         }
 425       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 426       next0 = SNAT_OUT2IN_NEXT_DROP;
 427       goto out;
 428     }
 429
 430   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 431                     !icmp_is_error_message (icmp0)))
 432     {
 433       if (icmp0->type != ICMP4_echo_request || key0.port != sm0.port)
 434         {
 435           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 436           next0 = SNAT_OUT2IN_NEXT_DROP;
 437           goto out;
 438         }
 439     }
 440
 441 out:
 442   *p_value = sm0;
 443 out2:
 444   *p_proto = key0.protocol;
 445   *p_dont_translate = dont_translate;
 446   return next0;
 447 }
 448
 449 static inline u32 icmp_out2in (snat_main_t *sm,
 450                                vlib_buffer_t * b0,
 451                                ip4_header_t * ip0,
 452                                icmp46_header_t * icmp0,
 453                                u32 sw_if_index0,
 454                                u32 rx_fib_index0,
 455                                vlib_node_runtime_t * node,
 456                                u32 next0,
 457                                u32 thread_index,
 458                                void *d)
 459 {
 460   snat_session_key_t sm0;
 461   u8 protocol;
 462   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 463   ip4_header_t *inner_ip0 = 0;
 464   void *l4_header = 0;
 465   icmp46_header_t *inner_icmp0;
 466   u8 dont_translate;
 467   u32 new_addr0, old_addr0;
 468   u16 old_id0, new_id0;
 469   ip_csum_t sum0;
 470   u16 checksum0;
 471   u32 next0_tmp;
 472
 473   echo0 = (icmp_echo_header_t *)(icmp0+1);
 474
 475   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 476                                        &protocol, &sm0, &dont_translate, d);
 477   if (next0_tmp != ~0)
 478     next0 = next0_tmp;
 479   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 480     goto out;
 481
 482   sum0 = ip_incremental_checksum (0, icmp0,
 483                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 484   checksum0 = ~ip_csum_fold (sum0);
 485   if (checksum0 != 0 && checksum0 != 0xffff)
 486     {
 487       next0 = SNAT_OUT2IN_NEXT_DROP;
 488       goto out;
 489     }
 490
 491   old_addr0 = ip0->dst_address.as_u32;
 492   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 493   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 494
 495   sum0 = ip0->checksum;
 496   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 497                          dst_address /* changed member */);
 498   ip0->checksum = ip_csum_fold (sum0);
 499
 500   if (!icmp_is_error_message (icmp0))
 501     {
 502       new_id0 = sm0.port;
 503       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 504         {
 505           old_id0 = echo0->identifier;
 506           new_id0 = sm0.port;
 507           echo0->identifier = new_id0;
 508
 509           sum0 = icmp0->checksum;
 510           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 511                                  identifier /* changed member */);
 512           icmp0->checksum = ip_csum_fold (sum0);
 513         }
 514     }
 515   else
 516     {
 517       inner_ip0 = (ip4_header_t *)(echo0+1);
 518       l4_header = ip4_next_header (inner_ip0);
 519
 520       if (!ip4_header_checksum_is_valid (inner_ip0))
 521         {
 522           next0 = SNAT_OUT2IN_NEXT_DROP;
 523           goto out;
 524         }
 525
 526       old_addr0 = inner_ip0->src_address.as_u32;
 527       inner_ip0->src_address = sm0.addr;
 528       new_addr0 = inner_ip0->src_address.as_u32;
 529
 530       sum0 = icmp0->checksum;
 531       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 532                              src_address /* changed member */);
 533       icmp0->checksum = ip_csum_fold (sum0);
 534
 535       switch (protocol)
 536         {
 537         case SNAT_PROTOCOL_ICMP:
 538           inner_icmp0 = (icmp46_header_t*)l4_header;
 539           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 540
 541           old_id0 = inner_echo0->identifier;
 542           new_id0 = sm0.port;
 543           inner_echo0->identifier = new_id0;
 544
 545           sum0 = icmp0->checksum;
 546           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 547                                  identifier);
 548           icmp0->checksum = ip_csum_fold (sum0);
 549           break;
 550         case SNAT_PROTOCOL_UDP:
 551         case SNAT_PROTOCOL_TCP:
 552           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 553           new_id0 = sm0.port;
 554           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 555
 556           sum0 = icmp0->checksum;
 557           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 558                                  src_port);
 559           icmp0->checksum = ip_csum_fold (sum0);
 560           break;
 561         default:
 562           ASSERT(0);
 563         }
 564     }
 565
 566 out:
 567   return next0;
 568 }
 569
 570
 571 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 572                                          vlib_buffer_t * b0,
 573                                          ip4_header_t * ip0,
 574                                          icmp46_header_t * icmp0,
 575                                          u32 sw_if_index0,
 576                                          u32 rx_fib_index0,
 577                                          vlib_node_runtime_t * node,
 578                                          u32 next0, f64 now,
 579                                          u32 thread_index,
 580                                          snat_session_t ** p_s0)
 581 {
 582   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 583                       next0, thread_index, p_s0);
 584   snat_session_t * s0 = *p_s0;
 585   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 586     {
 587       /* Accounting */
 588       s0->last_heard = now;
 589       s0->total_pkts++;
 590       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 591       /* Per-user LRU list maintenance for dynamic translation */
 592       if (!snat_is_session_static (s0))
 593         {
 594           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 595                              s0->per_user_index);
 596           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 597                               s0->per_user_list_head_index,
 598                               s0->per_user_index);
 599         }
 600     }
 601   return next0;
 602 }
 603
 604 static uword
 605 snat_out2in_node_fn (vlib_main_t * vm,
 606                   vlib_node_runtime_t * node,
 607                   vlib_frame_t * frame)
 608 {
 609   u32 n_left_from, * from, * to_next;
 610   snat_out2in_next_t next_index;
 611   u32 pkts_processed = 0;
 612   snat_main_t * sm = &snat_main;
 613   f64 now = vlib_time_now (vm);
 614   u32 thread_index = vlib_get_thread_index ();
 615
 616   from = vlib_frame_vector_args (frame);
 617   n_left_from = frame->n_vectors;
 618   next_index = node->cached_next_index;
 619
 620   while (n_left_from > 0)
 621     {
 622       u32 n_left_to_next;
 623
 624       vlib_get_next_frame (vm, node, next_index,
 625                            to_next, n_left_to_next);
 626
 627       while (n_left_from >= 4 && n_left_to_next >= 2)
 628         {
 629           u32 bi0, bi1;
 630           vlib_buffer_t * b0, * b1;
 631           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 632           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 633           u32 sw_if_index0, sw_if_index1;
 634           ip4_header_t * ip0, *ip1;
 635           ip_csum_t sum0, sum1;
 636           u32 new_addr0, old_addr0;
 637           u16 new_port0, old_port0;
 638           u32 new_addr1, old_addr1;
 639           u16 new_port1, old_port1;
 640           udp_header_t * udp0, * udp1;
 641           tcp_header_t * tcp0, * tcp1;
 642           icmp46_header_t * icmp0, * icmp1;
 643           snat_session_key_t key0, key1, sm0, sm1;
 644           u32 rx_fib_index0, rx_fib_index1;
 645           u32 proto0, proto1;
 646           snat_session_t * s0 = 0, * s1 = 0;
 647           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 648
 649           /* Prefetch next iteration. */
 650           {
 651             vlib_buffer_t * p2, * p3;
 652
 653             p2 = vlib_get_buffer (vm, from[2]);
 654             p3 = vlib_get_buffer (vm, from[3]);
 655
 656             vlib_prefetch_buffer_header (p2, LOAD);
 657             vlib_prefetch_buffer_header (p3, LOAD);
 658
 659             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 660             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 661           }
 662
 663           /* speculatively enqueue b0 and b1 to the current next frame */
 664           to_next[0] = bi0 = from[0];
 665           to_next[1] = bi1 = from[1];
 666           from += 2;
 667           to_next += 2;
 668           n_left_from -= 2;
 669           n_left_to_next -= 2;
 670
 671           b0 = vlib_get_buffer (vm, bi0);
 672           b1 = vlib_get_buffer (vm, bi1);
 673
 674           ip0 = vlib_buffer_get_current (b0);
 675           udp0 = ip4_next_header (ip0);
 676           tcp0 = (tcp_header_t *) udp0;
 677           icmp0 = (icmp46_header_t *) udp0;
 678
 679           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 680           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 681                                    sw_if_index0);
 682
 683           if (PREDICT_FALSE(ip0->ttl == 1))
 684             {
 685               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 686               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 687                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 688                                            0);
 689               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 690               goto trace0;
 691             }
 692
 693           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 694
 695           if (PREDICT_FALSE (proto0 == ~0))
 696               goto trace0;
 697
 698           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 699             {
 700               next0 = icmp_out2in_slow_path
 701                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 702                  next0, now, thread_index, &s0);
 703               goto trace0;
 704             }
 705
 706           key0.addr = ip0->dst_address;
 707           key0.port = udp0->dst_port;
 708           key0.protocol = proto0;
 709           key0.fib_index = rx_fib_index0;
 710
 711           kv0.key = key0.as_u64;
 712
 713           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 714             {
 715               /* Try to match static mapping by external address and port,
 716                  destination address and port in packet */
 717               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 718                 {
 719                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 720                   /*
 721                    * Send DHCP packets to the ipv4 stack, or we won't
 722                    * be able to use dhcp client on the outside interface
 723                    */
 724                   if (proto0 != SNAT_PROTOCOL_UDP
 725                       || (udp0->dst_port
 726                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 727                     next0 = SNAT_OUT2IN_NEXT_DROP;
 728                   goto trace0;
 729                 }
 730
 731               /* Create session initiated by host from external network */
 732               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 733                                                      thread_index);
 734               if (!s0)
 735                 {
 736                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 737                   next0 = SNAT_OUT2IN_NEXT_DROP;
 738                   goto trace0;
 739                 }
 740             }
 741           else
 742             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 743                                     value0.value);
 744
 745           old_addr0 = ip0->dst_address.as_u32;
 746           ip0->dst_address = s0->in2out.addr;
 747           new_addr0 = ip0->dst_address.as_u32;
 748           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 749
 750           sum0 = ip0->checksum;
 751           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 752                                  ip4_header_t,
 753                                  dst_address /* changed member */);
 754           ip0->checksum = ip_csum_fold (sum0);
 755
 756           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 757             {
 758               old_port0 = tcp0->dst_port;
 759               tcp0->dst_port = s0->in2out.port;
 760               new_port0 = tcp0->dst_port;
 761
 762               sum0 = tcp0->checksum;
 763               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 764                                      ip4_header_t,
 765                                      dst_address /* changed member */);
 766
 767               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 768                                      ip4_header_t /* cheat */,
 769                                      length /* changed member */);
 770               tcp0->checksum = ip_csum_fold(sum0);
 771             }
 772           else
 773             {
 774               old_port0 = udp0->dst_port;
 775               udp0->dst_port = s0->in2out.port;
 776               udp0->checksum = 0;
 777             }
 778
 779           /* Accounting */
 780           s0->last_heard = now;
 781           s0->total_pkts++;
 782           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 783           /* Per-user LRU list maintenance for dynamic translation */
 784           if (!snat_is_session_static (s0))
 785             {
 786               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 787                                  s0->per_user_index);
 788               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 789                                   s0->per_user_list_head_index,
 790                                   s0->per_user_index);
 791             }
 792         trace0:
 793
 794           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 795                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 796             {
 797               snat_out2in_trace_t *t =
 798                  vlib_add_trace (vm, node, b0, sizeof (*t));
 799               t->sw_if_index = sw_if_index0;
 800               t->next_index = next0;
 801               t->session_index = ~0;
 802               if (s0)
 803                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 804             }
 805
 806           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 807
 808
 809           ip1 = vlib_buffer_get_current (b1);
 810           udp1 = ip4_next_header (ip1);
 811           tcp1 = (tcp_header_t *) udp1;
 812           icmp1 = (icmp46_header_t *) udp1;
 813
 814           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 815           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 816                                    sw_if_index1);
 817
 818           if (PREDICT_FALSE(ip1->ttl == 1))
 819             {
 820               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 821               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 822                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 823                                            0);
 824               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 825               goto trace1;
 826             }
 827
 828           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 829
 830           if (PREDICT_FALSE (proto1 == ~0))
 831               goto trace1;
 832
 833           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 834             {
 835               next1 = icmp_out2in_slow_path
 836                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 837                  next1, now, thread_index, &s1);
 838               goto trace1;
 839             }
 840
 841           key1.addr = ip1->dst_address;
 842           key1.port = udp1->dst_port;
 843           key1.protocol = proto1;
 844           key1.fib_index = rx_fib_index1;
 845
 846           kv1.key = key1.as_u64;
 847
 848           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 849             {
 850               /* Try to match static mapping by external address and port,
 851                  destination address and port in packet */
 852               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 853                 {
 854                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 855                   /*
 856                    * Send DHCP packets to the ipv4 stack, or we won't
 857                    * be able to use dhcp client on the outside interface
 858                    */
 859                   if (proto1 != SNAT_PROTOCOL_UDP
 860                       || (udp1->dst_port
 861                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 862                     next1 = SNAT_OUT2IN_NEXT_DROP;
 863                   goto trace1;
 864                 }
 865
 866               /* Create session initiated by host from external network */
 867               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 868                                                      thread_index);
 869               if (!s1)
 870                 {
 871                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 872                   next1 = SNAT_OUT2IN_NEXT_DROP;
 873                   goto trace1;
 874                 }
 875             }
 876           else
 877             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 878                                     value1.value);
 879
 880           old_addr1 = ip1->dst_address.as_u32;
 881           ip1->dst_address = s1->in2out.addr;
 882           new_addr1 = ip1->dst_address.as_u32;
 883           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 884
 885           sum1 = ip1->checksum;
 886           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 887                                  ip4_header_t,
 888                                  dst_address /* changed member */);
 889           ip1->checksum = ip_csum_fold (sum1);
 890
 891           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 892             {
 893               old_port1 = tcp1->dst_port;
 894               tcp1->dst_port = s1->in2out.port;
 895               new_port1 = tcp1->dst_port;
 896
 897               sum1 = tcp1->checksum;
 898               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 899                                      ip4_header_t,
 900                                      dst_address /* changed member */);
 901
 902               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 903                                      ip4_header_t /* cheat */,
 904                                      length /* changed member */);
 905               tcp1->checksum = ip_csum_fold(sum1);
 906             }
 907           else
 908             {
 909               old_port1 = udp1->dst_port;
 910               udp1->dst_port = s1->in2out.port;
 911               udp1->checksum = 0;
 912             }
 913
 914           /* Accounting */
 915           s1->last_heard = now;
 916           s1->total_pkts++;
 917           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 918           /* Per-user LRU list maintenance for dynamic translation */
 919           if (!snat_is_session_static (s1))
 920             {
 921               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 922                                  s1->per_user_index);
 923               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 924                                   s1->per_user_list_head_index,
 925                                   s1->per_user_index);
 926             }
 927         trace1:
 928
 929           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 930                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 931             {
 932               snat_out2in_trace_t *t =
 933                  vlib_add_trace (vm, node, b1, sizeof (*t));
 934               t->sw_if_index = sw_if_index1;
 935               t->next_index = next1;
 936               t->session_index = ~0;
 937               if (s1)
 938                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
 939             }
 940
 941           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 942
 943           /* verify speculative enqueues, maybe switch current next frame */
 944           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 945                                            to_next, n_left_to_next,
 946                                            bi0, bi1, next0, next1);
 947         }
 948
 949       while (n_left_from > 0 && n_left_to_next > 0)
 950         {
 951           u32 bi0;
 952           vlib_buffer_t * b0;
 953           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 954           u32 sw_if_index0;
 955           ip4_header_t * ip0;
 956           ip_csum_t sum0;
 957           u32 new_addr0, old_addr0;
 958           u16 new_port0, old_port0;
 959           udp_header_t * udp0;
 960           tcp_header_t * tcp0;
 961           icmp46_header_t * icmp0;
 962           snat_session_key_t key0, sm0;
 963           u32 rx_fib_index0;
 964           u32 proto0;
 965           snat_session_t * s0 = 0;
 966           clib_bihash_kv_8_8_t kv0, value0;
 967
 968           /* speculatively enqueue b0 to the current next frame */
 969           bi0 = from[0];
 970           to_next[0] = bi0;
 971           from += 1;
 972           to_next += 1;
 973           n_left_from -= 1;
 974           n_left_to_next -= 1;
 975
 976           b0 = vlib_get_buffer (vm, bi0);
 977
 978           ip0 = vlib_buffer_get_current (b0);
 979           udp0 = ip4_next_header (ip0);
 980           tcp0 = (tcp_header_t *) udp0;
 981           icmp0 = (icmp46_header_t *) udp0;
 982
 983           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 984           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 985                                    sw_if_index0);
 986
 987           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 988
 989           if (PREDICT_FALSE (proto0 == ~0))
 990               goto trace00;
 991
 992           if (PREDICT_FALSE(ip0->ttl == 1))
 993             {
 994               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 995               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 996                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 997                                            0);
 998               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 999               goto trace00;
1000             }
1001
1002           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1003             {
1004               next0 = icmp_out2in_slow_path
1005                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1006                  next0, now, thread_index, &s0);
1007               goto trace00;
1008             }
1009
1010           key0.addr = ip0->dst_address;
1011           key0.port = udp0->dst_port;
1012           key0.protocol = proto0;
1013           key0.fib_index = rx_fib_index0;
1014
1015           kv0.key = key0.as_u64;
1016
1017           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1018             {
1019               /* Try to match static mapping by external address and port,
1020                  destination address and port in packet */
1021               if (snat_static_mapping_match(sm, key0, &sm0, 1))
1022                 {
1023                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1024                   /*
1025                    * Send DHCP packets to the ipv4 stack, or we won't
1026                    * be able to use dhcp client on the outside interface
1027                    */
1028                   if (proto0 != SNAT_PROTOCOL_UDP
1029                       || (udp0->dst_port
1030                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1031
1032                     next0 = SNAT_OUT2IN_NEXT_DROP;
1033                   goto trace00;
1034                 }
1035
1036               /* Create session initiated by host from external network */
1037               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1038                                                      thread_index);
1039               if (!s0)
1040                 {
1041                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1042                     next0 = SNAT_OUT2IN_NEXT_DROP;
1043                   goto trace00;
1044                 }
1045             }
1046           else
1047             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1048                                     value0.value);
1049
1050           old_addr0 = ip0->dst_address.as_u32;
1051           ip0->dst_address = s0->in2out.addr;
1052           new_addr0 = ip0->dst_address.as_u32;
1053           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1054
1055           sum0 = ip0->checksum;
1056           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1057                                  ip4_header_t,
1058                                  dst_address /* changed member */);
1059           ip0->checksum = ip_csum_fold (sum0);
1060
1061           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1062             {
1063               old_port0 = tcp0->dst_port;
1064               tcp0->dst_port = s0->in2out.port;
1065               new_port0 = tcp0->dst_port;
1066
1067               sum0 = tcp0->checksum;
1068               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1069                                      ip4_header_t,
1070                                      dst_address /* changed member */);
1071
1072               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1073                                      ip4_header_t /* cheat */,
1074                                      length /* changed member */);
1075               tcp0->checksum = ip_csum_fold(sum0);
1076             }
1077           else
1078             {
1079               old_port0 = udp0->dst_port;
1080               udp0->dst_port = s0->in2out.port;
1081               udp0->checksum = 0;
1082             }
1083
1084           /* Accounting */
1085           s0->last_heard = now;
1086           s0->total_pkts++;
1087           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1088           /* Per-user LRU list maintenance for dynamic translation */
1089           if (!snat_is_session_static (s0))
1090             {
1091               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1092                                  s0->per_user_index);
1093               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1094                                   s0->per_user_list_head_index,
1095                                   s0->per_user_index);
1096             }
1097         trace00:
1098
1099           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1100                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1101             {
1102               snat_out2in_trace_t *t =
1103                  vlib_add_trace (vm, node, b0, sizeof (*t));
1104               t->sw_if_index = sw_if_index0;
1105               t->next_index = next0;
1106               t->session_index = ~0;
1107               if (s0)
1108                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1109             }
1110
1111           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1112
1113           /* verify speculative enqueue, maybe switch current next frame */
1114           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1115                                            to_next, n_left_to_next,
1116                                            bi0, next0);
1117         }
1118
1119       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1120     }
1121
1122   vlib_node_increment_counter (vm, snat_out2in_node.index,
1123                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1124                                pkts_processed);
1125   return frame->n_vectors;
1126 }
1127
1128 VLIB_REGISTER_NODE (snat_out2in_node) = {
1129   .function = snat_out2in_node_fn,
1130   .name = "snat-out2in",
1131   .vector_size = sizeof (u32),
1132   .format_trace = format_snat_out2in_trace,
1133   .type = VLIB_NODE_TYPE_INTERNAL,
1134
1135   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1136   .error_strings = snat_out2in_error_strings,
1137
1138   .runtime_data_bytes = sizeof (snat_runtime_t),
1139
1140   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1141
1142   /* edit / add dispositions here */
1143   .next_nodes = {
1144     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1145     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1146     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1147   },
1148 };
1149 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1150
1151 /**************************/
1152 /*** deterministic mode ***/
1153 /**************************/
1154 static uword
1155 snat_det_out2in_node_fn (vlib_main_t * vm,
1156                          vlib_node_runtime_t * node,
1157                          vlib_frame_t * frame)
1158 {
1159   u32 n_left_from, * from, * to_next;
1160   snat_out2in_next_t next_index;
1161   u32 pkts_processed = 0;
1162   snat_main_t * sm = &snat_main;
1163
1164   from = vlib_frame_vector_args (frame);
1165   n_left_from = frame->n_vectors;
1166   next_index = node->cached_next_index;
1167
1168   while (n_left_from > 0)
1169     {
1170       u32 n_left_to_next;
1171
1172       vlib_get_next_frame (vm, node, next_index,
1173                            to_next, n_left_to_next);
1174
1175       while (n_left_from >= 4 && n_left_to_next >= 2)
1176         {
1177           u32 bi0, bi1;
1178           vlib_buffer_t * b0, * b1;
1179           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1180           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1181           u32 sw_if_index0, sw_if_index1;
1182           ip4_header_t * ip0, * ip1;
1183           ip_csum_t sum0, sum1;
1184           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1185           u16 new_port0, old_port0, old_port1, new_port1;
1186           udp_header_t * udp0, * udp1;
1187           tcp_header_t * tcp0, * tcp1;
1188           u32 proto0, proto1;
1189           snat_det_out_key_t key0, key1;
1190           snat_det_map_t * dm0, * dm1;
1191           snat_det_session_t * ses0 = 0, * ses1 = 0;
1192
1193           /* Prefetch next iteration. */
1194           {
1195             vlib_buffer_t * p2, * p3;
1196
1197             p2 = vlib_get_buffer (vm, from[2]);
1198             p3 = vlib_get_buffer (vm, from[3]);
1199
1200             vlib_prefetch_buffer_header (p2, LOAD);
1201             vlib_prefetch_buffer_header (p3, LOAD);
1202
1203             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1204             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1205           }
1206
1207           /* speculatively enqueue b0 and b1 to the current next frame */
1208           to_next[0] = bi0 = from[0];
1209           to_next[1] = bi1 = from[1];
1210           from += 2;
1211           to_next += 2;
1212           n_left_from -= 2;
1213           n_left_to_next -= 2;
1214
1215           b0 = vlib_get_buffer (vm, bi0);
1216           b1 = vlib_get_buffer (vm, bi1);
1217
1218           ip0 = vlib_buffer_get_current (b0);
1219           udp0 = ip4_next_header (ip0);
1220           tcp0 = (tcp_header_t *) udp0;
1221
1222           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1223
1224           if (PREDICT_FALSE(ip0->ttl == 1))
1225             {
1226               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1227               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1228                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1229                                            0);
1230               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1231               goto trace0;
1232             }
1233
1234           key0.ext_host_addr = ip0->src_address;
1235           key0.ext_host_port = tcp0->src;
1236           key0.out_port = tcp0->dst;
1237
1238           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1239           if (PREDICT_FALSE(!dm0))
1240             {
1241               clib_warning("unknown dst address:  %U",
1242                            format_ip4_address, &ip0->dst_address);
1243               next0 = SNAT_OUT2IN_NEXT_DROP;
1244               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1245               goto trace0;
1246             }
1247
1248           snat_det_reverse(dm0, &ip0->dst_address,
1249                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1250
1251           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1252           if (PREDICT_FALSE(!ses0))
1253             {
1254               clib_warning("no match src %U:%d dst %U:%d for user %U",
1255                            format_ip4_address, &ip0->src_address,
1256                            clib_net_to_host_u16 (tcp0->src),
1257                            format_ip4_address, &ip0->dst_address,
1258                            clib_net_to_host_u16 (tcp0->dst),
1259                            format_ip4_address, &new_addr0);
1260               next0 = SNAT_OUT2IN_NEXT_DROP;
1261               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1262               goto trace0;
1263             }
1264           new_port0 = ses0->in_port;
1265
1266           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1267
1268           old_addr0 = ip0->dst_address;
1269           ip0->dst_address = new_addr0;
1270           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1271
1272           sum0 = ip0->checksum;
1273           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1274                                  ip4_header_t,
1275                                  dst_address /* changed member */);
1276           ip0->checksum = ip_csum_fold (sum0);
1277
1278           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1279             {
1280               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1281                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1282               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1283                 snat_det_ses_close(dm0, ses0);
1284
1285               old_port0 = tcp0->dst;
1286               tcp0->dst = new_port0;
1287
1288               sum0 = tcp0->checksum;
1289               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1290                                      ip4_header_t,
1291                                      dst_address /* changed member */);
1292
1293               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1294                                      ip4_header_t /* cheat */,
1295                                      length /* changed member */);
1296               tcp0->checksum = ip_csum_fold(sum0);
1297             }
1298           else
1299             {
1300               old_port0 = udp0->dst_port;
1301               udp0->dst_port = new_port0;
1302               udp0->checksum = 0;
1303             }
1304
1305         trace0:
1306
1307           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1308                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1309             {
1310               snat_out2in_trace_t *t =
1311                  vlib_add_trace (vm, node, b0, sizeof (*t));
1312               t->sw_if_index = sw_if_index0;
1313               t->next_index = next0;
1314               t->session_index = ~0;
1315               if (ses0)
1316                 t->session_index = ses0 - dm0->sessions;
1317             }
1318
1319           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1320
1321           b1 = vlib_get_buffer (vm, bi1);
1322
1323           ip1 = vlib_buffer_get_current (b1);
1324           udp1 = ip4_next_header (ip1);
1325           tcp1 = (tcp_header_t *) udp1;
1326
1327           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1328
1329           if (PREDICT_FALSE(ip1->ttl == 1))
1330             {
1331               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1332               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1333                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1334                                            0);
1335               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1336               goto trace1;
1337             }
1338
1339           key1.ext_host_addr = ip1->src_address;
1340           key1.ext_host_port = tcp1->src;
1341           key1.out_port = tcp1->dst;
1342
1343           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1344           if (PREDICT_FALSE(!dm1))
1345             {
1346               clib_warning("unknown dst address:  %U",
1347                            format_ip4_address, &ip1->dst_address);
1348               next1 = SNAT_OUT2IN_NEXT_DROP;
1349               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1350               goto trace1;
1351             }
1352
1353           snat_det_reverse(dm1, &ip1->dst_address,
1354                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1355
1356           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1357           if (PREDICT_FALSE(!ses1))
1358             {
1359               clib_warning("no match src %U:%d dst %U:%d for user %U",
1360                            format_ip4_address, &ip1->src_address,
1361                            clib_net_to_host_u16 (tcp1->src),
1362                            format_ip4_address, &ip1->dst_address,
1363                            clib_net_to_host_u16 (tcp1->dst),
1364                            format_ip4_address, &new_addr1);
1365               next1 = SNAT_OUT2IN_NEXT_DROP;
1366               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1367               goto trace1;
1368             }
1369           new_port1 = ses1->in_port;
1370
1371           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1372
1373           old_addr1 = ip1->dst_address;
1374           ip1->dst_address = new_addr1;
1375           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1376
1377           sum1 = ip1->checksum;
1378           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1379                                  ip4_header_t,
1380                                  dst_address /* changed member */);
1381           ip1->checksum = ip_csum_fold (sum1);
1382
1383           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1384             {
1385               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1386                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1387               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1388                 snat_det_ses_close(dm1, ses1);
1389
1390               old_port1 = tcp1->dst;
1391               tcp1->dst = new_port1;
1392
1393               sum1 = tcp1->checksum;
1394               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1395                                      ip4_header_t,
1396                                      dst_address /* changed member */);
1397
1398               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1399                                      ip4_header_t /* cheat */,
1400                                      length /* changed member */);
1401               tcp1->checksum = ip_csum_fold(sum1);
1402             }
1403           else
1404             {
1405               old_port1 = udp1->dst_port;
1406               udp1->dst_port = new_port1;
1407               udp1->checksum = 0;
1408             }
1409
1410         trace1:
1411
1412           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1413                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1414             {
1415               snat_out2in_trace_t *t =
1416                  vlib_add_trace (vm, node, b1, sizeof (*t));
1417               t->sw_if_index = sw_if_index1;
1418               t->next_index = next1;
1419               t->session_index = ~0;
1420               if (ses1)
1421                 t->session_index = ses1 - dm1->sessions;
1422             }
1423
1424           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1425
1426           /* verify speculative enqueues, maybe switch current next frame */
1427           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1428                                            to_next, n_left_to_next,
1429                                            bi0, bi1, next0, next1);
1430          }
1431
1432       while (n_left_from > 0 && n_left_to_next > 0)
1433         {
1434           u32 bi0;
1435           vlib_buffer_t * b0;
1436           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1437           u32 sw_if_index0;
1438           ip4_header_t * ip0;
1439           ip_csum_t sum0;
1440           ip4_address_t new_addr0, old_addr0;
1441           u16 new_port0, old_port0;
1442           udp_header_t * udp0;
1443           tcp_header_t * tcp0;
1444           u32 proto0;
1445           snat_det_out_key_t key0;
1446           snat_det_map_t * dm0;
1447           snat_det_session_t * ses0 = 0;
1448
1449           /* speculatively enqueue b0 to the current next frame */
1450           bi0 = from[0];
1451           to_next[0] = bi0;
1452           from += 1;
1453           to_next += 1;
1454           n_left_from -= 1;
1455           n_left_to_next -= 1;
1456
1457           b0 = vlib_get_buffer (vm, bi0);
1458
1459           ip0 = vlib_buffer_get_current (b0);
1460           udp0 = ip4_next_header (ip0);
1461           tcp0 = (tcp_header_t *) udp0;
1462
1463           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1464
1465           if (PREDICT_FALSE(ip0->ttl == 1))
1466             {
1467               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1468               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1469                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1470                                            0);
1471               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1472               goto trace00;
1473             }
1474
1475           key0.ext_host_addr = ip0->src_address;
1476           key0.ext_host_port = tcp0->src;
1477           key0.out_port = tcp0->dst;
1478
1479           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1480           if (PREDICT_FALSE(!dm0))
1481             {
1482               clib_warning("unknown dst address:  %U",
1483                            format_ip4_address, &ip0->dst_address);
1484               next0 = SNAT_OUT2IN_NEXT_DROP;
1485               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1486               goto trace00;
1487             }
1488
1489           snat_det_reverse(dm0, &ip0->dst_address,
1490                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1491
1492           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1493           if (PREDICT_FALSE(!ses0))
1494             {
1495               clib_warning("no match src %U:%d dst %U:%d for user %U",
1496                            format_ip4_address, &ip0->src_address,
1497                            clib_net_to_host_u16 (tcp0->src),
1498                            format_ip4_address, &ip0->dst_address,
1499                            clib_net_to_host_u16 (tcp0->dst),
1500                            format_ip4_address, &new_addr0);
1501               next0 = SNAT_OUT2IN_NEXT_DROP;
1502               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1503               goto trace00;
1504             }
1505           new_port0 = ses0->in_port;
1506
1507           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1508
1509           old_addr0 = ip0->dst_address;
1510           ip0->dst_address = new_addr0;
1511           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1512
1513           sum0 = ip0->checksum;
1514           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1515                                  ip4_header_t,
1516                                  dst_address /* changed member */);
1517           ip0->checksum = ip_csum_fold (sum0);
1518
1519           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1520             {
1521               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1522                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1523               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1524                 snat_det_ses_close(dm0, ses0);
1525
1526               old_port0 = tcp0->dst;
1527               tcp0->dst = new_port0;
1528
1529               sum0 = tcp0->checksum;
1530               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1531                                      ip4_header_t,
1532                                      dst_address /* changed member */);
1533
1534               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1535                                      ip4_header_t /* cheat */,
1536                                      length /* changed member */);
1537               tcp0->checksum = ip_csum_fold(sum0);
1538             }
1539           else
1540             {
1541               old_port0 = udp0->dst_port;
1542               udp0->dst_port = new_port0;
1543               udp0->checksum = 0;
1544             }
1545
1546         trace00:
1547
1548           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1549                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1550             {
1551               snat_out2in_trace_t *t =
1552                  vlib_add_trace (vm, node, b0, sizeof (*t));
1553               t->sw_if_index = sw_if_index0;
1554               t->next_index = next0;
1555               t->session_index = ~0;
1556               if (ses0)
1557                 t->session_index = ses0 - dm0->sessions;
1558             }
1559
1560           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1561
1562           /* verify speculative enqueue, maybe switch current next frame */
1563           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1564                                            to_next, n_left_to_next,
1565                                            bi0, next0);
1566         }
1567
1568       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1569     }
1570
1571   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1572                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1573                                pkts_processed);
1574   return frame->n_vectors;
1575 }
1576
1577 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1578   .function = snat_det_out2in_node_fn,
1579   .name = "snat-det-out2in",
1580   .vector_size = sizeof (u32),
1581   .format_trace = format_snat_out2in_trace,
1582   .type = VLIB_NODE_TYPE_INTERNAL,
1583
1584   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1585   .error_strings = snat_out2in_error_strings,
1586
1587   .runtime_data_bytes = sizeof (snat_runtime_t),
1588
1589   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1590
1591   /* edit / add dispositions here */
1592   .next_nodes = {
1593     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1594     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1595     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1596   },
1597 };
1598 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1599
1600 /**********************/
1601 /*** worker handoff ***/
1602 /**********************/
1603 static uword
1604 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1605                                vlib_node_runtime_t * node,
1606                                vlib_frame_t * frame)
1607 {
1608   snat_main_t *sm = &snat_main;
1609   vlib_thread_main_t *tm = vlib_get_thread_main ();
1610   u32 n_left_from, *from, *to_next = 0;
1611   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1612   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1613     = 0;
1614   vlib_frame_queue_elt_t *hf = 0;
1615   vlib_frame_t *f = 0;
1616   int i;
1617   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1618   u32 next_worker_index = 0;
1619   u32 current_worker_index = ~0;
1620   u32 thread_index = vlib_get_thread_index ();
1621
1622   ASSERT (vec_len (sm->workers));
1623
1624   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1625     {
1626       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1627
1628       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1629                                sm->first_worker_index + sm->num_workers - 1,
1630                                (vlib_frame_queue_t *) (~0));
1631     }
1632
1633   from = vlib_frame_vector_args (frame);
1634   n_left_from = frame->n_vectors;
1635
1636   while (n_left_from > 0)
1637     {
1638       u32 bi0;
1639       vlib_buffer_t *b0;
1640       u32 sw_if_index0;
1641       u32 rx_fib_index0;
1642       ip4_header_t * ip0;
1643       u8 do_handoff;
1644
1645       bi0 = from[0];
1646       from += 1;
1647       n_left_from -= 1;
1648
1649       b0 = vlib_get_buffer (vm, bi0);
1650
1651       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1652       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1653
1654       ip0 = vlib_buffer_get_current (b0);
1655
1656       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1657
1658       if (PREDICT_FALSE (next_worker_index != thread_index))
1659         {
1660           do_handoff = 1;
1661
1662           if (next_worker_index != current_worker_index)
1663             {
1664               if (hf)
1665                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1666
1667               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1668                                                       next_worker_index,
1669                                                       handoff_queue_elt_by_worker_index);
1670
1671               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1672               to_next_worker = &hf->buffer_index[hf->n_vectors];
1673               current_worker_index = next_worker_index;
1674             }
1675
1676           /* enqueue to correct worker thread */
1677           to_next_worker[0] = bi0;
1678           to_next_worker++;
1679           n_left_to_next_worker--;
1680
1681           if (n_left_to_next_worker == 0)
1682             {
1683               hf->n_vectors = VLIB_FRAME_SIZE;
1684               vlib_put_frame_queue_elt (hf);
1685               current_worker_index = ~0;
1686               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1687               hf = 0;
1688             }
1689         }
1690       else
1691         {
1692           do_handoff = 0;
1693           /* if this is 1st frame */
1694           if (!f)
1695             {
1696               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1697               to_next = vlib_frame_vector_args (f);
1698             }
1699
1700           to_next[0] = bi0;
1701           to_next += 1;
1702           f->n_vectors++;
1703         }
1704
1705       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1706                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1707         {
1708           snat_out2in_worker_handoff_trace_t *t =
1709             vlib_add_trace (vm, node, b0, sizeof (*t));
1710           t->next_worker_index = next_worker_index;
1711           t->do_handoff = do_handoff;
1712         }
1713     }
1714
1715   if (f)
1716     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1717
1718   if (hf)
1719     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1720
1721   /* Ship frames to the worker nodes */
1722   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1723     {
1724       if (handoff_queue_elt_by_worker_index[i])
1725         {
1726           hf = handoff_queue_elt_by_worker_index[i];
1727           /*
1728            * It works better to let the handoff node
1729            * rate-adapt, always ship the handoff queue element.
1730            */
1731           if (1 || hf->n_vectors == hf->last_n_vectors)
1732             {
1733               vlib_put_frame_queue_elt (hf);
1734               handoff_queue_elt_by_worker_index[i] = 0;
1735             }
1736           else
1737             hf->last_n_vectors = hf->n_vectors;
1738         }
1739       congested_handoff_queue_by_worker_index[i] =
1740         (vlib_frame_queue_t *) (~0);
1741     }
1742   hf = 0;
1743   current_worker_index = ~0;
1744   return frame->n_vectors;
1745 }
1746
1747 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1748   .function = snat_out2in_worker_handoff_fn,
1749   .name = "snat-out2in-worker-handoff",
1750   .vector_size = sizeof (u32),
1751   .format_trace = format_snat_out2in_worker_handoff_trace,
1752   .type = VLIB_NODE_TYPE_INTERNAL,
1753
1754   .n_next_nodes = 1,
1755
1756   .next_nodes = {
1757     [0] = "error-drop",
1758   },
1759 };
1760
1761 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1762
1763 static uword
1764 snat_out2in_fast_node_fn (vlib_main_t * vm,
1765                           vlib_node_runtime_t * node,
1766                           vlib_frame_t * frame)
1767 {
1768   u32 n_left_from, * from, * to_next;
1769   snat_out2in_next_t next_index;
1770   u32 pkts_processed = 0;
1771   snat_main_t * sm = &snat_main;
1772
1773   from = vlib_frame_vector_args (frame);
1774   n_left_from = frame->n_vectors;
1775   next_index = node->cached_next_index;
1776
1777   while (n_left_from > 0)
1778     {
1779       u32 n_left_to_next;
1780
1781       vlib_get_next_frame (vm, node, next_index,
1782                            to_next, n_left_to_next);
1783
1784       while (n_left_from > 0 && n_left_to_next > 0)
1785         {
1786           u32 bi0;
1787           vlib_buffer_t * b0;
1788           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1789           u32 sw_if_index0;
1790           ip4_header_t * ip0;
1791           ip_csum_t sum0;
1792           u32 new_addr0, old_addr0;
1793           u16 new_port0, old_port0;
1794           udp_header_t * udp0;
1795           tcp_header_t * tcp0;
1796           icmp46_header_t * icmp0;
1797           snat_session_key_t key0, sm0;
1798           u32 proto0;
1799           u32 rx_fib_index0;
1800
1801           /* speculatively enqueue b0 to the current next frame */
1802           bi0 = from[0];
1803           to_next[0] = bi0;
1804           from += 1;
1805           to_next += 1;
1806           n_left_from -= 1;
1807           n_left_to_next -= 1;
1808
1809           b0 = vlib_get_buffer (vm, bi0);
1810
1811           ip0 = vlib_buffer_get_current (b0);
1812           udp0 = ip4_next_header (ip0);
1813           tcp0 = (tcp_header_t *) udp0;
1814           icmp0 = (icmp46_header_t *) udp0;
1815
1816           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1817           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1818
1819           vnet_feature_next (sw_if_index0, &next0, b0);
1820
1821           if (PREDICT_FALSE(ip0->ttl == 1))
1822             {
1823               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1824               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1825                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1826                                            0);
1827               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1828               goto trace00;
1829             }
1830
1831           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1832
1833           if (PREDICT_FALSE (proto0 == ~0))
1834               goto trace00;
1835
1836           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1837             {
1838               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1839                                   rx_fib_index0, node, next0, ~0, 0);
1840               goto trace00;
1841             }
1842
1843           key0.addr = ip0->dst_address;
1844           key0.port = udp0->dst_port;
1845           key0.fib_index = rx_fib_index0;
1846
1847           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1848             {
1849               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1850               goto trace00;
1851             }
1852
1853           new_addr0 = sm0.addr.as_u32;
1854           new_port0 = sm0.port;
1855           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1856           old_addr0 = ip0->dst_address.as_u32;
1857           ip0->dst_address.as_u32 = new_addr0;
1858
1859           sum0 = ip0->checksum;
1860           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1861                                  ip4_header_t,
1862                                  dst_address /* changed member */);
1863           ip0->checksum = ip_csum_fold (sum0);
1864
1865           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1866             {
1867                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1868                 {
1869                   old_port0 = tcp0->dst_port;
1870                   tcp0->dst_port = new_port0;
1871
1872                   sum0 = tcp0->checksum;
1873                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1874                                          ip4_header_t,
1875                                          dst_address /* changed member */);
1876
1877                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1878                                          ip4_header_t /* cheat */,
1879                                          length /* changed member */);
1880                   tcp0->checksum = ip_csum_fold(sum0);
1881                 }
1882               else
1883                 {
1884                   old_port0 = udp0->dst_port;
1885                   udp0->dst_port = new_port0;
1886                   udp0->checksum = 0;
1887                 }
1888             }
1889           else
1890             {
1891               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1892                 {
1893                   sum0 = tcp0->checksum;
1894                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1895                                          ip4_header_t,
1896                                          dst_address /* changed member */);
1897
1898                   tcp0->checksum = ip_csum_fold(sum0);
1899                 }
1900             }
1901
1902         trace00:
1903
1904           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1905                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1906             {
1907               snat_out2in_trace_t *t =
1908                  vlib_add_trace (vm, node, b0, sizeof (*t));
1909               t->sw_if_index = sw_if_index0;
1910               t->next_index = next0;
1911             }
1912
1913           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1914
1915           /* verify speculative enqueue, maybe switch current next frame */
1916           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1917                                            to_next, n_left_to_next,
1918                                            bi0, next0);
1919         }
1920
1921       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1922     }
1923
1924   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1925                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1926                                pkts_processed);
1927   return frame->n_vectors;
1928 }
1929
1930 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1931   .function = snat_out2in_fast_node_fn,
1932   .name = "snat-out2in-fast",
1933   .vector_size = sizeof (u32),
1934   .format_trace = format_snat_out2in_fast_trace,
1935   .type = VLIB_NODE_TYPE_INTERNAL,
1936
1937   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1938   .error_strings = snat_out2in_error_strings,
1939
1940   .runtime_data_bytes = sizeof (snat_runtime_t),
1941
1942   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1943
1944   /* edit / add dispositions here */
1945   .next_nodes = {
1946     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1947     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1948     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1949   },
1950 };
1951 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);