src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27 #include <snat/snat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     SNAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out SNAT session key.
 121  * @param out2in Out2in SNAT session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140
 141   user_key.addr = in2out.addr;
 142   user_key.fib_index = in2out.fib_index;
 143   kv0.key = user_key.as_u64;
 144
 145   /* Ever heard of the "user" = inside ip4 address before? */
 146   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 147     {
 148       /* no, make a new one */
 149       pool_get (sm->per_thread_data[thread_index].users, u);
 150       memset (u, 0, sizeof (*u));
 151       u->addr = in2out.addr;
 152       u->fib_index = in2out.fib_index;
 153
 154       pool_get (sm->per_thread_data[thread_index].list_pool,
 155                 per_user_list_head_elt);
 156
 157       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 158         sm->per_thread_data[thread_index].list_pool;
 159
 160       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 161                        u->sessions_per_user_list_head_index);
 162
 163       kv0.value = u - sm->per_thread_data[thread_index].users;
 164
 165       /* add user */
 166       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 167
 168       /* add non-traslated packets worker lookup */
 169       kv0.value = thread_index;
 170       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 171     }
 172   else
 173     {
 174       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 175                              value0.value);
 176     }
 177
 178   pool_get (sm->per_thread_data[thread_index].sessions, s);
 179   memset (s, 0, sizeof (*s));
 180
 181   s->outside_address_index = ~0;
 182   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 183   u->nstaticsessions++;
 184
 185   /* Create list elts */
 186   pool_get (sm->per_thread_data[thread_index].list_pool,
 187             per_user_translation_list_elt);
 188   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 189                    per_user_translation_list_elt -
 190                    sm->per_thread_data[thread_index].list_pool);
 191
 192   per_user_translation_list_elt->value =
 193     s - sm->per_thread_data[thread_index].sessions;
 194   s->per_user_index =
 195     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 196   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 197
 198   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 199                       s->per_user_list_head_index,
 200                       per_user_translation_list_elt -
 201                       sm->per_thread_data[thread_index].list_pool);
 202
 203   s->in2out = in2out;
 204   s->out2in = out2in;
 205   s->in2out.protocol = out2in.protocol;
 206
 207   /* Add to translation hashes */
 208   kv0.key = s->in2out.as_u64;
 209   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 210   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 211       clib_warning ("in2out key add failed");
 212
 213   kv0.key = s->out2in.as_u64;
 214   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 215
 216   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 217       clib_warning ("out2in key add failed");
 218
 219   /* log NAT event */
 220   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 221                                       s->out2in.addr.as_u32,
 222                                       s->in2out.protocol,
 223                                       s->in2out.port,
 224                                       s->out2in.port,
 225                                       s->in2out.fib_index);
 226    return s;
 227 }
 228
 229 static_always_inline
 230 snat_out2in_error_t icmp_get_key(icmp46_header_t *icmp0,
 231                                  snat_session_key_t *p_key0)
 232 {
 233   snat_session_key_t key0;
 234   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 235   ip4_header_t *inner_ip0;
 236   void *l4_header = 0;
 237   icmp46_header_t *inner_icmp0;
 238
 239   echo0 = (icmp_echo_header_t *)(icmp0+1);
 240
 241   if (!icmp_is_error_message (icmp0))
 242     {
 243       key0.protocol = SNAT_PROTOCOL_ICMP;
 244       key0.port = echo0->identifier;
 245     }
 246   else
 247     {
 248       inner_ip0 = (ip4_header_t *)(echo0+1);
 249       l4_header = ip4_next_header (inner_ip0);
 250       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 251       switch (key0.protocol)
 252         {
 253         case SNAT_PROTOCOL_ICMP:
 254           inner_icmp0 = (icmp46_header_t*)l4_header;
 255           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 256           key0.port = inner_echo0->identifier;
 257           break;
 258         case SNAT_PROTOCOL_UDP:
 259         case SNAT_PROTOCOL_TCP:
 260           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 261           break;
 262         default:
 263           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 264         }
 265     }
 266   *p_key0 = key0;
 267   return -1; /* success */
 268 }
 269
 270 static_always_inline u8
 271 is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0,
 272                   u32 ip4_addr)
 273 {
 274   snat_runtime_t *rt = (snat_runtime_t *) node->runtime_data;
 275   ip4_address_t * first_int_addr;
 276
 277   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 278     {
 279       first_int_addr =
 280         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 281                                      0 /* just want the address */);
 282       rt->cached_sw_if_index = sw_if_index0;
 283       if (first_int_addr)
 284         rt->cached_ip4_address = first_int_addr->as_u32;
 285       else
 286         rt->cached_ip4_address = 0;
 287     }
 288
 289   if (PREDICT_FALSE(ip4_addr == rt->cached_ip4_address))
 290     return 1;
 291   else
 292     return 0;
 293 }
 294
 295 /**
 296  * Get address and port values to be used for packet SNAT translation
 297  * and create session if needed
 298  *
 299  * @param[in,out] sm             SNAT main
 300  * @param[in,out] node           SNAT node runtime
 301  * @param[in] thread_index       thread index
 302  * @param[in,out] b0             buffer containing packet to be translated
 303  * @param[out] p_key             address and port before NAT translation
 304  * @param[out] p_value           address and port after NAT translation
 305  * @param[out] p_dont_translate  if packet should not be translated
 306  * @param d                      optional parameter
 307  */
 308 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 309                            u32 thread_index, vlib_buffer_t *b0,
 310                            snat_session_key_t *p_key,
 311                            snat_session_key_t *p_value,
 312                            u8 *p_dont_translate, void *d)
 313 {
 314   ip4_header_t *ip0;
 315   icmp46_header_t *icmp0;
 316   u32 sw_if_index0;
 317   u32 rx_fib_index0;
 318   snat_session_key_t key0;
 319   snat_session_key_t sm0;
 320   snat_session_t *s0 = 0;
 321   u8 dont_translate = 0;
 322   clib_bihash_kv_8_8_t kv0, value0;
 323   u32 next0 = ~0;
 324   int err;
 325
 326   ip0 = vlib_buffer_get_current (b0);
 327   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 328   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 329   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 330
 331   err = icmp_get_key (icmp0, &key0);
 332   if (err != -1)
 333     {
 334       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 335       next0 = SNAT_OUT2IN_NEXT_DROP;
 336       goto out;
 337     }
 338   key0.addr = ip0->dst_address;
 339   key0.fib_index = rx_fib_index0;
 340
 341   kv0.key = key0.as_u64;
 342
 343   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 344     {
 345       /* Try to match static mapping by external address and port,
 346          destination address and port in packet */
 347       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 348         {
 349           /* Don't NAT packet aimed at the intfc address */
 350           if (is_interface_addr(sm, node, sw_if_index0,
 351                                 ip0->dst_address.as_u32))
 352             {
 353               dont_translate = 1;
 354               goto out;
 355             }
 356           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 357           next0 = SNAT_OUT2IN_NEXT_DROP;
 358           goto out;
 359         }
 360
 361       if (icmp_is_error_message (icmp0))
 362         {
 363           next0 = SNAT_OUT2IN_NEXT_DROP;
 364           goto out;
 365         }
 366
 367       /* Create session initiated by host from external network */
 368       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 369                                              node, thread_index);
 370
 371       if (!s0)
 372         {
 373           next0 = SNAT_OUT2IN_NEXT_DROP;
 374           goto out;
 375         }
 376     }
 377   else
 378     s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 379                             value0.value);
 380
 381 out:
 382   *p_key = key0;
 383   if (s0)
 384     *p_value = s0->in2out;
 385   *p_dont_translate = dont_translate;
 386   if (d)
 387     *(snat_session_t**)d = s0;
 388   return next0;
 389 }
 390
 391 /**
 392  * Get address and port values to be used for packet SNAT translation
 393  *
 394  * @param[in] sm                 SNAT main
 395  * @param[in,out] node           SNAT node runtime
 396  * @param[in] thread_index       thread index
 397  * @param[in,out] b0             buffer containing packet to be translated
 398  * @param[out] p_key             address and port before NAT translation
 399  * @param[out] p_value           address and port after NAT translation
 400  * @param[out] p_dont_translate  if packet should not be translated
 401  * @param d                      optional parameter
 402  */
 403 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 404                            u32 thread_index, vlib_buffer_t *b0,
 405                            snat_session_key_t *p_key,
 406                            snat_session_key_t *p_value,
 407                            u8 *p_dont_translate, void *d)
 408 {
 409   ip4_header_t *ip0;
 410   icmp46_header_t *icmp0;
 411   u32 sw_if_index0;
 412   u32 rx_fib_index0;
 413   snat_session_key_t key0;
 414   snat_session_key_t sm0;
 415   u8 dont_translate = 0;
 416   u32 next0 = ~0;
 417   int err;
 418
 419   ip0 = vlib_buffer_get_current (b0);
 420   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 421   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 422   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 423
 424   err = icmp_get_key (icmp0, &key0);
 425   if (err != -1)
 426     {
 427       b0->error = node->errors[err];
 428       next0 = SNAT_OUT2IN_NEXT_DROP;
 429       goto out2;
 430     }
 431   key0.addr = ip0->dst_address;
 432   key0.fib_index = rx_fib_index0;
 433
 434   if (snat_static_mapping_match(sm, key0, &sm0, 1))
 435     {
 436       /* Don't NAT packet aimed at the intfc address */
 437       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 438         {
 439           dont_translate = 1;
 440           goto out;
 441         }
 442       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 443       next0 = SNAT_OUT2IN_NEXT_DROP;
 444       goto out;
 445     }
 446
 447 out:
 448   *p_value = sm0;
 449 out2:
 450   *p_key = key0;
 451   *p_dont_translate = dont_translate;
 452   return next0;
 453 }
 454
 455 static inline u32 icmp_out2in (snat_main_t *sm,
 456                                vlib_buffer_t * b0,
 457                                ip4_header_t * ip0,
 458                                icmp46_header_t * icmp0,
 459                                u32 sw_if_index0,
 460                                u32 rx_fib_index0,
 461                                vlib_node_runtime_t * node,
 462                                u32 next0,
 463                                u32 thread_index,
 464                                void *d)
 465 {
 466   snat_session_key_t key0, sm0;
 467   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 468   ip4_header_t *inner_ip0 = 0;
 469   void *l4_header = 0;
 470   icmp46_header_t *inner_icmp0;
 471   u8 dont_translate;
 472   u32 new_addr0, old_addr0;
 473   u16 old_id0, new_id0;
 474   ip_csum_t sum0;
 475   u16 checksum0;
 476   u32 next0_tmp;
 477
 478   echo0 = (icmp_echo_header_t *)(icmp0+1);
 479
 480   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 481                                        &key0, &sm0, &dont_translate, d);
 482   if (next0_tmp != ~0)
 483     next0 = next0_tmp;
 484   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 485     goto out;
 486
 487   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 488                     !icmp_is_error_message (icmp0)))
 489     {
 490       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 491       next0 = SNAT_OUT2IN_NEXT_DROP;
 492       goto out;
 493     }
 494
 495   sum0 = ip_incremental_checksum (0, icmp0,
 496                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 497   checksum0 = ~ip_csum_fold (sum0);
 498   if (checksum0 != 0 && checksum0 != 0xffff)
 499     {
 500       next0 = SNAT_OUT2IN_NEXT_DROP;
 501       goto out;
 502     }
 503
 504   old_addr0 = ip0->dst_address.as_u32;
 505   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 506   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 507
 508   sum0 = ip0->checksum;
 509   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 510                          dst_address /* changed member */);
 511   ip0->checksum = ip_csum_fold (sum0);
 512
 513   if (!icmp_is_error_message (icmp0))
 514     {
 515       new_id0 = sm0.port;
 516       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 517         {
 518           old_id0 = echo0->identifier;
 519           new_id0 = sm0.port;
 520           echo0->identifier = new_id0;
 521
 522           sum0 = icmp0->checksum;
 523           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 524                                  identifier /* changed member */);
 525           icmp0->checksum = ip_csum_fold (sum0);
 526         }
 527     }
 528   else
 529     {
 530       inner_ip0 = (ip4_header_t *)(echo0+1);
 531       l4_header = ip4_next_header (inner_ip0);
 532
 533       if (!ip4_header_checksum_is_valid (inner_ip0))
 534         {
 535           next0 = SNAT_OUT2IN_NEXT_DROP;
 536           goto out;
 537         }
 538
 539       old_addr0 = inner_ip0->src_address.as_u32;
 540       inner_ip0->src_address = sm0.addr;
 541       new_addr0 = inner_ip0->src_address.as_u32;
 542
 543       sum0 = icmp0->checksum;
 544       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 545                              src_address /* changed member */);
 546       icmp0->checksum = ip_csum_fold (sum0);
 547
 548       switch (key0.protocol)
 549         {
 550         case SNAT_PROTOCOL_ICMP:
 551           inner_icmp0 = (icmp46_header_t*)l4_header;
 552           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 553
 554           old_id0 = inner_echo0->identifier;
 555           new_id0 = sm0.port;
 556           inner_echo0->identifier = new_id0;
 557
 558           sum0 = icmp0->checksum;
 559           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 560                                  identifier);
 561           icmp0->checksum = ip_csum_fold (sum0);
 562           break;
 563         case SNAT_PROTOCOL_UDP:
 564         case SNAT_PROTOCOL_TCP:
 565           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 566           new_id0 = sm0.port;
 567           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 568
 569           sum0 = icmp0->checksum;
 570           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 571                                  src_port);
 572           icmp0->checksum = ip_csum_fold (sum0);
 573           break;
 574         default:
 575           ASSERT(0);
 576         }
 577     }
 578
 579 out:
 580   return next0;
 581 }
 582
 583
 584 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 585                                          vlib_buffer_t * b0,
 586                                          ip4_header_t * ip0,
 587                                          icmp46_header_t * icmp0,
 588                                          u32 sw_if_index0,
 589                                          u32 rx_fib_index0,
 590                                          vlib_node_runtime_t * node,
 591                                          u32 next0, f64 now,
 592                                          u32 thread_index,
 593                                          snat_session_t ** p_s0)
 594 {
 595   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 596                       next0, thread_index, p_s0);
 597   snat_session_t * s0 = *p_s0;
 598   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 599     {
 600       /* Accounting */
 601       s0->last_heard = now;
 602       s0->total_pkts++;
 603       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 604       /* Per-user LRU list maintenance for dynamic translation */
 605       if (!snat_is_session_static (s0))
 606         {
 607           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 608                              s0->per_user_index);
 609           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 610                               s0->per_user_list_head_index,
 611                               s0->per_user_index);
 612         }
 613     }
 614   return next0;
 615 }
 616
 617 static uword
 618 snat_out2in_node_fn (vlib_main_t * vm,
 619                   vlib_node_runtime_t * node,
 620                   vlib_frame_t * frame)
 621 {
 622   u32 n_left_from, * from, * to_next;
 623   snat_out2in_next_t next_index;
 624   u32 pkts_processed = 0;
 625   snat_main_t * sm = &snat_main;
 626   f64 now = vlib_time_now (vm);
 627   u32 thread_index = vlib_get_thread_index ();
 628
 629   from = vlib_frame_vector_args (frame);
 630   n_left_from = frame->n_vectors;
 631   next_index = node->cached_next_index;
 632
 633   while (n_left_from > 0)
 634     {
 635       u32 n_left_to_next;
 636
 637       vlib_get_next_frame (vm, node, next_index,
 638                            to_next, n_left_to_next);
 639
 640       while (n_left_from >= 4 && n_left_to_next >= 2)
 641         {
 642           u32 bi0, bi1;
 643           vlib_buffer_t * b0, * b1;
 644           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 645           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 646           u32 sw_if_index0, sw_if_index1;
 647           ip4_header_t * ip0, *ip1;
 648           ip_csum_t sum0, sum1;
 649           u32 new_addr0, old_addr0;
 650           u16 new_port0, old_port0;
 651           u32 new_addr1, old_addr1;
 652           u16 new_port1, old_port1;
 653           udp_header_t * udp0, * udp1;
 654           tcp_header_t * tcp0, * tcp1;
 655           icmp46_header_t * icmp0, * icmp1;
 656           snat_session_key_t key0, key1, sm0, sm1;
 657           u32 rx_fib_index0, rx_fib_index1;
 658           u32 proto0, proto1;
 659           snat_session_t * s0 = 0, * s1 = 0;
 660           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 661
 662           /* Prefetch next iteration. */
 663           {
 664             vlib_buffer_t * p2, * p3;
 665
 666             p2 = vlib_get_buffer (vm, from[2]);
 667             p3 = vlib_get_buffer (vm, from[3]);
 668
 669             vlib_prefetch_buffer_header (p2, LOAD);
 670             vlib_prefetch_buffer_header (p3, LOAD);
 671
 672             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 673             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 674           }
 675
 676           /* speculatively enqueue b0 and b1 to the current next frame */
 677           to_next[0] = bi0 = from[0];
 678           to_next[1] = bi1 = from[1];
 679           from += 2;
 680           to_next += 2;
 681           n_left_from -= 2;
 682           n_left_to_next -= 2;
 683
 684           b0 = vlib_get_buffer (vm, bi0);
 685           b1 = vlib_get_buffer (vm, bi1);
 686
 687           ip0 = vlib_buffer_get_current (b0);
 688           udp0 = ip4_next_header (ip0);
 689           tcp0 = (tcp_header_t *) udp0;
 690           icmp0 = (icmp46_header_t *) udp0;
 691
 692           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 693           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 694                                    sw_if_index0);
 695
 696           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 697
 698           if (PREDICT_FALSE (proto0 == ~0))
 699               goto trace0;
 700
 701           if (PREDICT_FALSE(ip0->ttl == 1))
 702             {
 703               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 704               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 705                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 706                                            0);
 707               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 708               goto trace0;
 709             }
 710
 711           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 712             {
 713               next0 = icmp_out2in_slow_path
 714                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 715                  next0, now, thread_index, &s0);
 716               goto trace0;
 717             }
 718
 719           key0.addr = ip0->dst_address;
 720           key0.port = udp0->dst_port;
 721           key0.protocol = proto0;
 722           key0.fib_index = rx_fib_index0;
 723
 724           kv0.key = key0.as_u64;
 725
 726           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 727             {
 728               /* Try to match static mapping by external address and port,
 729                  destination address and port in packet */
 730               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 731                 {
 732                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 733                   /*
 734                    * Send DHCP packets to the ipv4 stack, or we won't
 735                    * be able to use dhcp client on the outside interface
 736                    */
 737                   if (proto0 != SNAT_PROTOCOL_UDP
 738                       || (udp0->dst_port
 739                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 740                     next0 = SNAT_OUT2IN_NEXT_DROP;
 741                   goto trace0;
 742                 }
 743
 744               /* Create session initiated by host from external network */
 745               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 746                                                      thread_index);
 747               if (!s0)
 748                 {
 749                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 750                   next0 = SNAT_OUT2IN_NEXT_DROP;
 751                   goto trace0;
 752                 }
 753             }
 754           else
 755             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 756                                     value0.value);
 757
 758           old_addr0 = ip0->dst_address.as_u32;
 759           ip0->dst_address = s0->in2out.addr;
 760           new_addr0 = ip0->dst_address.as_u32;
 761           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 762
 763           sum0 = ip0->checksum;
 764           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 765                                  ip4_header_t,
 766                                  dst_address /* changed member */);
 767           ip0->checksum = ip_csum_fold (sum0);
 768
 769           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 770             {
 771               old_port0 = tcp0->dst_port;
 772               tcp0->dst_port = s0->in2out.port;
 773               new_port0 = tcp0->dst_port;
 774
 775               sum0 = tcp0->checksum;
 776               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 777                                      ip4_header_t,
 778                                      dst_address /* changed member */);
 779
 780               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 781                                      ip4_header_t /* cheat */,
 782                                      length /* changed member */);
 783               tcp0->checksum = ip_csum_fold(sum0);
 784             }
 785           else
 786             {
 787               old_port0 = udp0->dst_port;
 788               udp0->dst_port = s0->in2out.port;
 789               udp0->checksum = 0;
 790             }
 791
 792           /* Accounting */
 793           s0->last_heard = now;
 794           s0->total_pkts++;
 795           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 796           /* Per-user LRU list maintenance for dynamic translation */
 797           if (!snat_is_session_static (s0))
 798             {
 799               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 800                                  s0->per_user_index);
 801               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 802                                   s0->per_user_list_head_index,
 803                                   s0->per_user_index);
 804             }
 805         trace0:
 806
 807           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 808                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 809             {
 810               snat_out2in_trace_t *t =
 811                  vlib_add_trace (vm, node, b0, sizeof (*t));
 812               t->sw_if_index = sw_if_index0;
 813               t->next_index = next0;
 814               t->session_index = ~0;
 815               if (s0)
 816                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 817             }
 818
 819           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 820
 821
 822           ip1 = vlib_buffer_get_current (b1);
 823           udp1 = ip4_next_header (ip1);
 824           tcp1 = (tcp_header_t *) udp1;
 825           icmp1 = (icmp46_header_t *) udp1;
 826
 827           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 828           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 829                                    sw_if_index1);
 830
 831           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 832
 833           if (PREDICT_FALSE (proto1 == ~0))
 834               goto trace1;
 835
 836           if (PREDICT_FALSE(ip0->ttl == 1))
 837             {
 838               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 839               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 840                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 841                                            0);
 842               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 843               goto trace1;
 844             }
 845
 846           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 847             {
 848               next1 = icmp_out2in_slow_path
 849                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 850                  next1, now, thread_index, &s1);
 851               goto trace1;
 852             }
 853
 854           key1.addr = ip1->dst_address;
 855           key1.port = udp1->dst_port;
 856           key1.protocol = proto1;
 857           key1.fib_index = rx_fib_index1;
 858
 859           kv1.key = key1.as_u64;
 860
 861           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 862             {
 863               /* Try to match static mapping by external address and port,
 864                  destination address and port in packet */
 865               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 866                 {
 867                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 868                   /*
 869                    * Send DHCP packets to the ipv4 stack, or we won't
 870                    * be able to use dhcp client on the outside interface
 871                    */
 872                   if (proto1 != SNAT_PROTOCOL_UDP
 873                       || (udp1->dst_port
 874                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 875                     next1 = SNAT_OUT2IN_NEXT_DROP;
 876                   goto trace1;
 877                 }
 878
 879               /* Create session initiated by host from external network */
 880               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 881                                                      thread_index);
 882               if (!s1)
 883                 {
 884                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 885                   next1 = SNAT_OUT2IN_NEXT_DROP;
 886                   goto trace1;
 887                 }
 888             }
 889           else
 890             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 891                                     value1.value);
 892
 893           old_addr1 = ip1->dst_address.as_u32;
 894           ip1->dst_address = s1->in2out.addr;
 895           new_addr1 = ip1->dst_address.as_u32;
 896           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 897
 898           sum1 = ip1->checksum;
 899           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 900                                  ip4_header_t,
 901                                  dst_address /* changed member */);
 902           ip1->checksum = ip_csum_fold (sum1);
 903
 904           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 905             {
 906               old_port1 = tcp1->dst_port;
 907               tcp1->dst_port = s1->in2out.port;
 908               new_port1 = tcp1->dst_port;
 909
 910               sum1 = tcp1->checksum;
 911               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 912                                      ip4_header_t,
 913                                      dst_address /* changed member */);
 914
 915               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 916                                      ip4_header_t /* cheat */,
 917                                      length /* changed member */);
 918               tcp1->checksum = ip_csum_fold(sum1);
 919             }
 920           else
 921             {
 922               old_port1 = udp1->dst_port;
 923               udp1->dst_port = s1->in2out.port;
 924               udp1->checksum = 0;
 925             }
 926
 927           /* Accounting */
 928           s1->last_heard = now;
 929           s1->total_pkts++;
 930           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 931           /* Per-user LRU list maintenance for dynamic translation */
 932           if (!snat_is_session_static (s1))
 933             {
 934               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 935                                  s1->per_user_index);
 936               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 937                                   s1->per_user_list_head_index,
 938                                   s1->per_user_index);
 939             }
 940         trace1:
 941
 942           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 943                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 944             {
 945               snat_out2in_trace_t *t =
 946                  vlib_add_trace (vm, node, b1, sizeof (*t));
 947               t->sw_if_index = sw_if_index1;
 948               t->next_index = next1;
 949               t->session_index = ~0;
 950               if (s1)
 951                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
 952             }
 953
 954           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 955
 956           /* verify speculative enqueues, maybe switch current next frame */
 957           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 958                                            to_next, n_left_to_next,
 959                                            bi0, bi1, next0, next1);
 960         }
 961
 962       while (n_left_from > 0 && n_left_to_next > 0)
 963         {
 964           u32 bi0;
 965           vlib_buffer_t * b0;
 966           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 967           u32 sw_if_index0;
 968           ip4_header_t * ip0;
 969           ip_csum_t sum0;
 970           u32 new_addr0, old_addr0;
 971           u16 new_port0, old_port0;
 972           udp_header_t * udp0;
 973           tcp_header_t * tcp0;
 974           icmp46_header_t * icmp0;
 975           snat_session_key_t key0, sm0;
 976           u32 rx_fib_index0;
 977           u32 proto0;
 978           snat_session_t * s0 = 0;
 979           clib_bihash_kv_8_8_t kv0, value0;
 980
 981           /* speculatively enqueue b0 to the current next frame */
 982           bi0 = from[0];
 983           to_next[0] = bi0;
 984           from += 1;
 985           to_next += 1;
 986           n_left_from -= 1;
 987           n_left_to_next -= 1;
 988
 989           b0 = vlib_get_buffer (vm, bi0);
 990
 991           ip0 = vlib_buffer_get_current (b0);
 992           udp0 = ip4_next_header (ip0);
 993           tcp0 = (tcp_header_t *) udp0;
 994           icmp0 = (icmp46_header_t *) udp0;
 995
 996           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 997           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 998                                    sw_if_index0);
 999
1000           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1001
1002           if (PREDICT_FALSE (proto0 == ~0))
1003               goto trace00;
1004
1005           if (PREDICT_FALSE(ip0->ttl == 1))
1006             {
1007               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1008               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1009                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1010                                            0);
1011               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1012               goto trace00;
1013             }
1014
1015           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1016             {
1017               next0 = icmp_out2in_slow_path
1018                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1019                  next0, now, thread_index, &s0);
1020               goto trace00;
1021             }
1022
1023           key0.addr = ip0->dst_address;
1024           key0.port = udp0->dst_port;
1025           key0.protocol = proto0;
1026           key0.fib_index = rx_fib_index0;
1027
1028           kv0.key = key0.as_u64;
1029
1030           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1031             {
1032               /* Try to match static mapping by external address and port,
1033                  destination address and port in packet */
1034               if (snat_static_mapping_match(sm, key0, &sm0, 1))
1035                 {
1036                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1037                   /*
1038                    * Send DHCP packets to the ipv4 stack, or we won't
1039                    * be able to use dhcp client on the outside interface
1040                    */
1041                   if (proto0 != SNAT_PROTOCOL_UDP
1042                       || (udp0->dst_port
1043                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1044
1045                     next0 = SNAT_OUT2IN_NEXT_DROP;
1046                   goto trace00;
1047                 }
1048
1049               /* Create session initiated by host from external network */
1050               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1051                                                      thread_index);
1052               if (!s0)
1053                 {
1054                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1055                     next0 = SNAT_OUT2IN_NEXT_DROP;
1056                   goto trace00;
1057                 }
1058             }
1059           else
1060             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1061                                     value0.value);
1062
1063           old_addr0 = ip0->dst_address.as_u32;
1064           ip0->dst_address = s0->in2out.addr;
1065           new_addr0 = ip0->dst_address.as_u32;
1066           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1067
1068           sum0 = ip0->checksum;
1069           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1070                                  ip4_header_t,
1071                                  dst_address /* changed member */);
1072           ip0->checksum = ip_csum_fold (sum0);
1073
1074           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1075             {
1076               old_port0 = tcp0->dst_port;
1077               tcp0->dst_port = s0->in2out.port;
1078               new_port0 = tcp0->dst_port;
1079
1080               sum0 = tcp0->checksum;
1081               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1082                                      ip4_header_t,
1083                                      dst_address /* changed member */);
1084
1085               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1086                                      ip4_header_t /* cheat */,
1087                                      length /* changed member */);
1088               tcp0->checksum = ip_csum_fold(sum0);
1089             }
1090           else
1091             {
1092               old_port0 = udp0->dst_port;
1093               udp0->dst_port = s0->in2out.port;
1094               udp0->checksum = 0;
1095             }
1096
1097           /* Accounting */
1098           s0->last_heard = now;
1099           s0->total_pkts++;
1100           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1101           /* Per-user LRU list maintenance for dynamic translation */
1102           if (!snat_is_session_static (s0))
1103             {
1104               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1105                                  s0->per_user_index);
1106               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1107                                   s0->per_user_list_head_index,
1108                                   s0->per_user_index);
1109             }
1110         trace00:
1111
1112           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1113                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1114             {
1115               snat_out2in_trace_t *t =
1116                  vlib_add_trace (vm, node, b0, sizeof (*t));
1117               t->sw_if_index = sw_if_index0;
1118               t->next_index = next0;
1119               t->session_index = ~0;
1120               if (s0)
1121                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1122             }
1123
1124           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1125
1126           /* verify speculative enqueue, maybe switch current next frame */
1127           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1128                                            to_next, n_left_to_next,
1129                                            bi0, next0);
1130         }
1131
1132       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1133     }
1134
1135   vlib_node_increment_counter (vm, snat_out2in_node.index,
1136                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1137                                pkts_processed);
1138   return frame->n_vectors;
1139 }
1140
1141 VLIB_REGISTER_NODE (snat_out2in_node) = {
1142   .function = snat_out2in_node_fn,
1143   .name = "snat-out2in",
1144   .vector_size = sizeof (u32),
1145   .format_trace = format_snat_out2in_trace,
1146   .type = VLIB_NODE_TYPE_INTERNAL,
1147
1148   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1149   .error_strings = snat_out2in_error_strings,
1150
1151   .runtime_data_bytes = sizeof (snat_runtime_t),
1152
1153   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1154
1155   /* edit / add dispositions here */
1156   .next_nodes = {
1157     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1158     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1159     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1160   },
1161 };
1162 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1163
1164 /**************************/
1165 /*** deterministic mode ***/
1166 /**************************/
1167 static uword
1168 snat_det_out2in_node_fn (vlib_main_t * vm,
1169                          vlib_node_runtime_t * node,
1170                          vlib_frame_t * frame)
1171 {
1172   u32 n_left_from, * from, * to_next;
1173   snat_out2in_next_t next_index;
1174   u32 pkts_processed = 0;
1175   snat_main_t * sm = &snat_main;
1176
1177   from = vlib_frame_vector_args (frame);
1178   n_left_from = frame->n_vectors;
1179   next_index = node->cached_next_index;
1180
1181   while (n_left_from > 0)
1182     {
1183       u32 n_left_to_next;
1184
1185       vlib_get_next_frame (vm, node, next_index,
1186                            to_next, n_left_to_next);
1187
1188       while (n_left_from >= 4 && n_left_to_next >= 2)
1189         {
1190           u32 bi0, bi1;
1191           vlib_buffer_t * b0, * b1;
1192           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1193           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1194           u32 sw_if_index0, sw_if_index1;
1195           ip4_header_t * ip0, * ip1;
1196           ip_csum_t sum0, sum1;
1197           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1198           u16 new_port0, old_port0, old_port1, new_port1;
1199           udp_header_t * udp0, * udp1;
1200           tcp_header_t * tcp0, * tcp1;
1201           u32 proto0, proto1;
1202           snat_det_out_key_t key0, key1;
1203           snat_det_map_t * dm0, * dm1;
1204           snat_det_session_t * ses0 = 0, * ses1 = 0;
1205
1206           /* Prefetch next iteration. */
1207           {
1208             vlib_buffer_t * p2, * p3;
1209
1210             p2 = vlib_get_buffer (vm, from[2]);
1211             p3 = vlib_get_buffer (vm, from[3]);
1212
1213             vlib_prefetch_buffer_header (p2, LOAD);
1214             vlib_prefetch_buffer_header (p3, LOAD);
1215
1216             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1217             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1218           }
1219
1220           /* speculatively enqueue b0 and b1 to the current next frame */
1221           to_next[0] = bi0 = from[0];
1222           to_next[1] = bi1 = from[1];
1223           from += 2;
1224           to_next += 2;
1225           n_left_from -= 2;
1226           n_left_to_next -= 2;
1227
1228           b0 = vlib_get_buffer (vm, bi0);
1229           b1 = vlib_get_buffer (vm, bi1);
1230
1231           ip0 = vlib_buffer_get_current (b0);
1232           udp0 = ip4_next_header (ip0);
1233           tcp0 = (tcp_header_t *) udp0;
1234
1235           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1236
1237           key0.ext_host_addr = ip0->src_address;
1238           key0.ext_host_port = tcp0->src;
1239           key0.out_port = tcp0->dst;
1240
1241           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1242           if (PREDICT_FALSE(!dm0))
1243             {
1244               clib_warning("unknown dst address:  %U",
1245                            format_ip4_address, &ip0->dst_address);
1246               next0 = SNAT_OUT2IN_NEXT_DROP;
1247               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1248               goto trace0;
1249             }
1250
1251           snat_det_reverse(dm0, &ip0->dst_address,
1252                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1253
1254           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1255           if (PREDICT_FALSE(!ses0))
1256             {
1257               clib_warning("no match src %U:%d dst %U:%d for user %U",
1258                            format_ip4_address, &ip0->src_address,
1259                            clib_net_to_host_u16 (tcp0->src),
1260                            format_ip4_address, &ip0->dst_address,
1261                            clib_net_to_host_u16 (tcp0->dst),
1262                            format_ip4_address, &new_addr0);
1263               next0 = SNAT_OUT2IN_NEXT_DROP;
1264               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1265               goto trace0;
1266             }
1267           new_port0 = ses0->in_port;
1268
1269           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1270
1271           old_addr0 = ip0->dst_address;
1272           ip0->dst_address = new_addr0;
1273           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1274
1275           sum0 = ip0->checksum;
1276           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1277                                  ip4_header_t,
1278                                  dst_address /* changed member */);
1279           ip0->checksum = ip_csum_fold (sum0);
1280
1281           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1282             {
1283               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1284                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1285               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1286                 snat_det_ses_close(dm0, ses0);
1287
1288               old_port0 = tcp0->dst;
1289               tcp0->dst = new_port0;
1290
1291               sum0 = tcp0->checksum;
1292               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1293                                      ip4_header_t,
1294                                      dst_address /* changed member */);
1295
1296               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1297                                      ip4_header_t /* cheat */,
1298                                      length /* changed member */);
1299               tcp0->checksum = ip_csum_fold(sum0);
1300             }
1301           else
1302             {
1303               old_port0 = udp0->dst_port;
1304               udp0->dst_port = new_port0;
1305               udp0->checksum = 0;
1306             }
1307
1308         trace0:
1309
1310           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1311                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1312             {
1313               snat_out2in_trace_t *t =
1314                  vlib_add_trace (vm, node, b0, sizeof (*t));
1315               t->sw_if_index = sw_if_index0;
1316               t->next_index = next0;
1317               t->session_index = ~0;
1318               if (ses0)
1319                 t->session_index = ses0 - dm0->sessions;
1320             }
1321
1322           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1323
1324           b1 = vlib_get_buffer (vm, bi1);
1325
1326           ip1 = vlib_buffer_get_current (b1);
1327           udp1 = ip4_next_header (ip1);
1328           tcp1 = (tcp_header_t *) udp1;
1329
1330           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1331
1332           key1.ext_host_addr = ip1->src_address;
1333           key1.ext_host_port = tcp1->src;
1334           key1.out_port = tcp1->dst;
1335
1336           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1337           if (PREDICT_FALSE(!dm1))
1338             {
1339               clib_warning("unknown dst address:  %U",
1340                            format_ip4_address, &ip1->dst_address);
1341               next1 = SNAT_OUT2IN_NEXT_DROP;
1342               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1343               goto trace1;
1344             }
1345
1346           snat_det_reverse(dm1, &ip1->dst_address,
1347                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1348
1349           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1350           if (PREDICT_FALSE(!ses1))
1351             {
1352               clib_warning("no match src %U:%d dst %U:%d for user %U",
1353                            format_ip4_address, &ip1->src_address,
1354                            clib_net_to_host_u16 (tcp1->src),
1355                            format_ip4_address, &ip1->dst_address,
1356                            clib_net_to_host_u16 (tcp1->dst),
1357                            format_ip4_address, &new_addr1);
1358               next1 = SNAT_OUT2IN_NEXT_DROP;
1359               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1360               goto trace1;
1361             }
1362           new_port1 = ses1->in_port;
1363
1364           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1365
1366           old_addr1 = ip1->dst_address;
1367           ip1->dst_address = new_addr1;
1368           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1369
1370           sum1 = ip1->checksum;
1371           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1372                                  ip4_header_t,
1373                                  dst_address /* changed member */);
1374           ip1->checksum = ip_csum_fold (sum1);
1375
1376           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1377             {
1378               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1379                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1380               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1381                 snat_det_ses_close(dm1, ses1);
1382
1383               old_port1 = tcp1->dst;
1384               tcp1->dst = new_port1;
1385
1386               sum1 = tcp1->checksum;
1387               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1388                                      ip4_header_t,
1389                                      dst_address /* changed member */);
1390
1391               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1392                                      ip4_header_t /* cheat */,
1393                                      length /* changed member */);
1394               tcp1->checksum = ip_csum_fold(sum1);
1395             }
1396           else
1397             {
1398               old_port1 = udp1->dst_port;
1399               udp1->dst_port = new_port1;
1400               udp1->checksum = 0;
1401             }
1402
1403         trace1:
1404
1405           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1406                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1407             {
1408               snat_out2in_trace_t *t =
1409                  vlib_add_trace (vm, node, b1, sizeof (*t));
1410               t->sw_if_index = sw_if_index1;
1411               t->next_index = next1;
1412               t->session_index = ~0;
1413               if (ses1)
1414                 t->session_index = ses1 - dm1->sessions;
1415             }
1416
1417           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1418
1419           /* verify speculative enqueues, maybe switch current next frame */
1420           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1421                                            to_next, n_left_to_next,
1422                                            bi0, bi1, next0, next1);
1423          }
1424
1425       while (n_left_from > 0 && n_left_to_next > 0)
1426         {
1427           u32 bi0;
1428           vlib_buffer_t * b0;
1429           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1430           u32 sw_if_index0;
1431           ip4_header_t * ip0;
1432           ip_csum_t sum0;
1433           ip4_address_t new_addr0, old_addr0;
1434           u16 new_port0, old_port0;
1435           udp_header_t * udp0;
1436           tcp_header_t * tcp0;
1437           u32 proto0;
1438           snat_det_out_key_t key0;
1439           snat_det_map_t * dm0;
1440           snat_det_session_t * ses0 = 0;
1441
1442           /* speculatively enqueue b0 to the current next frame */
1443           bi0 = from[0];
1444           to_next[0] = bi0;
1445           from += 1;
1446           to_next += 1;
1447           n_left_from -= 1;
1448           n_left_to_next -= 1;
1449
1450           b0 = vlib_get_buffer (vm, bi0);
1451
1452           ip0 = vlib_buffer_get_current (b0);
1453           udp0 = ip4_next_header (ip0);
1454           tcp0 = (tcp_header_t *) udp0;
1455
1456           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1457
1458           key0.ext_host_addr = ip0->src_address;
1459           key0.ext_host_port = tcp0->src;
1460           key0.out_port = tcp0->dst;
1461
1462           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1463           if (PREDICT_FALSE(!dm0))
1464             {
1465               clib_warning("unknown dst address:  %U",
1466                            format_ip4_address, &ip0->dst_address);
1467               next0 = SNAT_OUT2IN_NEXT_DROP;
1468               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1469               goto trace00;
1470             }
1471
1472           snat_det_reverse(dm0, &ip0->dst_address,
1473                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1474
1475           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1476           if (PREDICT_FALSE(!ses0))
1477             {
1478               clib_warning("no match src %U:%d dst %U:%d for user %U",
1479                            format_ip4_address, &ip0->src_address,
1480                            clib_net_to_host_u16 (tcp0->src),
1481                            format_ip4_address, &ip0->dst_address,
1482                            clib_net_to_host_u16 (tcp0->dst),
1483                            format_ip4_address, &new_addr0);
1484               next0 = SNAT_OUT2IN_NEXT_DROP;
1485               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1486               goto trace00;
1487             }
1488           new_port0 = ses0->in_port;
1489
1490           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1491
1492           old_addr0 = ip0->dst_address;
1493           ip0->dst_address = new_addr0;
1494           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1495
1496           sum0 = ip0->checksum;
1497           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1498                                  ip4_header_t,
1499                                  dst_address /* changed member */);
1500           ip0->checksum = ip_csum_fold (sum0);
1501
1502           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1503             {
1504               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1505                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1506               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1507                 snat_det_ses_close(dm0, ses0);
1508
1509               old_port0 = tcp0->dst;
1510               tcp0->dst = new_port0;
1511
1512               sum0 = tcp0->checksum;
1513               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1514                                      ip4_header_t,
1515                                      dst_address /* changed member */);
1516
1517               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1518                                      ip4_header_t /* cheat */,
1519                                      length /* changed member */);
1520               tcp0->checksum = ip_csum_fold(sum0);
1521             }
1522           else
1523             {
1524               old_port0 = udp0->dst_port;
1525               udp0->dst_port = new_port0;
1526               udp0->checksum = 0;
1527             }
1528
1529         trace00:
1530
1531           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1532                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1533             {
1534               snat_out2in_trace_t *t =
1535                  vlib_add_trace (vm, node, b0, sizeof (*t));
1536               t->sw_if_index = sw_if_index0;
1537               t->next_index = next0;
1538               t->session_index = ~0;
1539               if (ses0)
1540                 t->session_index = ses0 - dm0->sessions;
1541             }
1542
1543           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1544
1545           /* verify speculative enqueue, maybe switch current next frame */
1546           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1547                                            to_next, n_left_to_next,
1548                                            bi0, next0);
1549         }
1550
1551       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1552     }
1553
1554   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1555                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1556                                pkts_processed);
1557   return frame->n_vectors;
1558 }
1559
1560 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1561   .function = snat_det_out2in_node_fn,
1562   .name = "snat-det-out2in",
1563   .vector_size = sizeof (u32),
1564   .format_trace = format_snat_out2in_trace,
1565   .type = VLIB_NODE_TYPE_INTERNAL,
1566
1567   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1568   .error_strings = snat_out2in_error_strings,
1569
1570   .runtime_data_bytes = sizeof (snat_runtime_t),
1571
1572   .n_next_nodes = 2,
1573
1574   /* edit / add dispositions here */
1575   .next_nodes = {
1576     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1577     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1578   },
1579 };
1580 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1581
1582 /**********************/
1583 /*** worker handoff ***/
1584 /**********************/
1585 static uword
1586 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1587                                vlib_node_runtime_t * node,
1588                                vlib_frame_t * frame)
1589 {
1590   snat_main_t *sm = &snat_main;
1591   vlib_thread_main_t *tm = vlib_get_thread_main ();
1592   u32 n_left_from, *from, *to_next = 0;
1593   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1594   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1595     = 0;
1596   vlib_frame_queue_elt_t *hf = 0;
1597   vlib_frame_t *f = 0;
1598   int i;
1599   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1600   u32 next_worker_index = 0;
1601   u32 current_worker_index = ~0;
1602   u32 thread_index = vlib_get_thread_index ();
1603
1604   ASSERT (vec_len (sm->workers));
1605
1606   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1607     {
1608       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1609
1610       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1611                                sm->first_worker_index + sm->num_workers - 1,
1612                                (vlib_frame_queue_t *) (~0));
1613     }
1614
1615   from = vlib_frame_vector_args (frame);
1616   n_left_from = frame->n_vectors;
1617
1618   while (n_left_from > 0)
1619     {
1620       u32 bi0;
1621       vlib_buffer_t *b0;
1622       u32 sw_if_index0;
1623       u32 rx_fib_index0;
1624       ip4_header_t * ip0;
1625       u8 do_handoff;
1626
1627       bi0 = from[0];
1628       from += 1;
1629       n_left_from -= 1;
1630
1631       b0 = vlib_get_buffer (vm, bi0);
1632
1633       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1634       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1635
1636       ip0 = vlib_buffer_get_current (b0);
1637
1638       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1639
1640       if (PREDICT_FALSE (next_worker_index != thread_index))
1641         {
1642           do_handoff = 1;
1643
1644           if (next_worker_index != current_worker_index)
1645             {
1646               if (hf)
1647                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1648
1649               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1650                                                       next_worker_index,
1651                                                       handoff_queue_elt_by_worker_index);
1652
1653               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1654               to_next_worker = &hf->buffer_index[hf->n_vectors];
1655               current_worker_index = next_worker_index;
1656             }
1657
1658           /* enqueue to correct worker thread */
1659           to_next_worker[0] = bi0;
1660           to_next_worker++;
1661           n_left_to_next_worker--;
1662
1663           if (n_left_to_next_worker == 0)
1664             {
1665               hf->n_vectors = VLIB_FRAME_SIZE;
1666               vlib_put_frame_queue_elt (hf);
1667               current_worker_index = ~0;
1668               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1669               hf = 0;
1670             }
1671         }
1672       else
1673         {
1674           do_handoff = 0;
1675           /* if this is 1st frame */
1676           if (!f)
1677             {
1678               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1679               to_next = vlib_frame_vector_args (f);
1680             }
1681
1682           to_next[0] = bi0;
1683           to_next += 1;
1684           f->n_vectors++;
1685         }
1686
1687       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1688                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1689         {
1690           snat_out2in_worker_handoff_trace_t *t =
1691             vlib_add_trace (vm, node, b0, sizeof (*t));
1692           t->next_worker_index = next_worker_index;
1693           t->do_handoff = do_handoff;
1694         }
1695     }
1696
1697   if (f)
1698     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1699
1700   if (hf)
1701     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1702
1703   /* Ship frames to the worker nodes */
1704   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1705     {
1706       if (handoff_queue_elt_by_worker_index[i])
1707         {
1708           hf = handoff_queue_elt_by_worker_index[i];
1709           /*
1710            * It works better to let the handoff node
1711            * rate-adapt, always ship the handoff queue element.
1712            */
1713           if (1 || hf->n_vectors == hf->last_n_vectors)
1714             {
1715               vlib_put_frame_queue_elt (hf);
1716               handoff_queue_elt_by_worker_index[i] = 0;
1717             }
1718           else
1719             hf->last_n_vectors = hf->n_vectors;
1720         }
1721       congested_handoff_queue_by_worker_index[i] =
1722         (vlib_frame_queue_t *) (~0);
1723     }
1724   hf = 0;
1725   current_worker_index = ~0;
1726   return frame->n_vectors;
1727 }
1728
1729 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1730   .function = snat_out2in_worker_handoff_fn,
1731   .name = "snat-out2in-worker-handoff",
1732   .vector_size = sizeof (u32),
1733   .format_trace = format_snat_out2in_worker_handoff_trace,
1734   .type = VLIB_NODE_TYPE_INTERNAL,
1735
1736   .n_next_nodes = 1,
1737
1738   .next_nodes = {
1739     [0] = "error-drop",
1740   },
1741 };
1742
1743 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1744
1745 static uword
1746 snat_out2in_fast_node_fn (vlib_main_t * vm,
1747                           vlib_node_runtime_t * node,
1748                           vlib_frame_t * frame)
1749 {
1750   u32 n_left_from, * from, * to_next;
1751   snat_out2in_next_t next_index;
1752   u32 pkts_processed = 0;
1753   snat_main_t * sm = &snat_main;
1754
1755   from = vlib_frame_vector_args (frame);
1756   n_left_from = frame->n_vectors;
1757   next_index = node->cached_next_index;
1758
1759   while (n_left_from > 0)
1760     {
1761       u32 n_left_to_next;
1762
1763       vlib_get_next_frame (vm, node, next_index,
1764                            to_next, n_left_to_next);
1765
1766       while (n_left_from > 0 && n_left_to_next > 0)
1767         {
1768           u32 bi0;
1769           vlib_buffer_t * b0;
1770           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1771           u32 sw_if_index0;
1772           ip4_header_t * ip0;
1773           ip_csum_t sum0;
1774           u32 new_addr0, old_addr0;
1775           u16 new_port0, old_port0;
1776           udp_header_t * udp0;
1777           tcp_header_t * tcp0;
1778           icmp46_header_t * icmp0;
1779           snat_session_key_t key0, sm0;
1780           u32 proto0;
1781           u32 rx_fib_index0;
1782
1783           /* speculatively enqueue b0 to the current next frame */
1784           bi0 = from[0];
1785           to_next[0] = bi0;
1786           from += 1;
1787           to_next += 1;
1788           n_left_from -= 1;
1789           n_left_to_next -= 1;
1790
1791           b0 = vlib_get_buffer (vm, bi0);
1792
1793           ip0 = vlib_buffer_get_current (b0);
1794           udp0 = ip4_next_header (ip0);
1795           tcp0 = (tcp_header_t *) udp0;
1796           icmp0 = (icmp46_header_t *) udp0;
1797
1798           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1799           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1800
1801           vnet_feature_next (sw_if_index0, &next0, b0);
1802
1803           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1804
1805           if (PREDICT_FALSE (proto0 == ~0))
1806               goto trace00;
1807
1808           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1809             {
1810               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1811                                   rx_fib_index0, node, next0, ~0, 0);
1812               goto trace00;
1813             }
1814
1815           key0.addr = ip0->dst_address;
1816           key0.port = udp0->dst_port;
1817           key0.fib_index = rx_fib_index0;
1818
1819           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1820             {
1821               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1822               goto trace00;
1823             }
1824
1825           new_addr0 = sm0.addr.as_u32;
1826           new_port0 = sm0.port;
1827           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1828           old_addr0 = ip0->dst_address.as_u32;
1829           ip0->dst_address.as_u32 = new_addr0;
1830
1831           sum0 = ip0->checksum;
1832           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1833                                  ip4_header_t,
1834                                  dst_address /* changed member */);
1835           ip0->checksum = ip_csum_fold (sum0);
1836
1837           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1838             {
1839                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1840                 {
1841                   old_port0 = tcp0->dst_port;
1842                   tcp0->dst_port = new_port0;
1843
1844                   sum0 = tcp0->checksum;
1845                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1846                                          ip4_header_t,
1847                                          dst_address /* changed member */);
1848
1849                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1850                                          ip4_header_t /* cheat */,
1851                                          length /* changed member */);
1852                   tcp0->checksum = ip_csum_fold(sum0);
1853                 }
1854               else
1855                 {
1856                   old_port0 = udp0->dst_port;
1857                   udp0->dst_port = new_port0;
1858                   udp0->checksum = 0;
1859                 }
1860             }
1861           else
1862             {
1863               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1864                 {
1865                   sum0 = tcp0->checksum;
1866                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1867                                          ip4_header_t,
1868                                          dst_address /* changed member */);
1869
1870                   tcp0->checksum = ip_csum_fold(sum0);
1871                 }
1872             }
1873
1874         trace00:
1875
1876           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1877                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1878             {
1879               snat_out2in_trace_t *t =
1880                  vlib_add_trace (vm, node, b0, sizeof (*t));
1881               t->sw_if_index = sw_if_index0;
1882               t->next_index = next0;
1883             }
1884
1885           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1886
1887           /* verify speculative enqueue, maybe switch current next frame */
1888           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1889                                            to_next, n_left_to_next,
1890                                            bi0, next0);
1891         }
1892
1893       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1894     }
1895
1896   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1897                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1898                                pkts_processed);
1899   return frame->n_vectors;
1900 }
1901
1902 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1903   .function = snat_out2in_fast_node_fn,
1904   .name = "snat-out2in-fast",
1905   .vector_size = sizeof (u32),
1906   .format_trace = format_snat_out2in_fast_trace,
1907   .type = VLIB_NODE_TYPE_INTERNAL,
1908
1909   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1910   .error_strings = snat_out2in_error_strings,
1911
1912   .runtime_data_bytes = sizeof (snat_runtime_t),
1913
1914   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1915
1916   /* edit / add dispositions here */
1917   .next_nodes = {
1918     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1919     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1920     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1921   },
1922 };
1923 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);