src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25
  26 #include <vppinfra/hash.h>
  27 #include <vppinfra/error.h>
  28 #include <vppinfra/elog.h>
  29
  30 typedef struct {
  31   u32 sw_if_index;
  32   u32 next_index;
  33   u32 session_index;
  34 } snat_out2in_trace_t;
  35
  36 typedef struct {
  37   u32 next_worker_index;
  38   u8 do_handoff;
  39 } snat_out2in_worker_handoff_trace_t;
  40
  41 /* packet trace format function */
  42 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  43 {
  44   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  45   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  46   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  47
  48   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  49               t->sw_if_index, t->next_index, t->session_index);
  50   return s;
  51 }
  52
  53 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  54 {
  55   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  56   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  57   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  58
  59   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  60               t->sw_if_index, t->next_index);
  61   return s;
  62 }
  63
  64 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  65 {
  66   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  67   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  68   snat_out2in_worker_handoff_trace_t * t =
  69     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  70   char * m;
  71
  72   m = t->do_handoff ? "next worker" : "same worker";
  73   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  74
  75   return s;
  76 }
  77
  78 vlib_node_registration_t snat_out2in_node;
  79 vlib_node_registration_t snat_out2in_fast_node;
  80 vlib_node_registration_t snat_out2in_worker_handoff_node;
  81
  82 #define foreach_snat_out2in_error                       \
  83 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  84 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  85 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  86 _(NO_TRANSLATION, "No translation")
  87
  88 typedef enum {
  89 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  90   foreach_snat_out2in_error
  91 #undef _
  92   SNAT_OUT2IN_N_ERROR,
  93 } snat_out2in_error_t;
  94
  95 static char * snat_out2in_error_strings[] = {
  96 #define _(sym,string) string,
  97   foreach_snat_out2in_error
  98 #undef _
  99 };
 100
 101 typedef enum {
 102   SNAT_OUT2IN_NEXT_DROP,
 103   SNAT_OUT2IN_NEXT_LOOKUP,
 104   SNAT_OUT2IN_N_NEXT,
 105 } snat_out2in_next_t;
 106
 107 /**
 108  * @brief Create session for static mapping.
 109  *
 110  * Create NAT session initiated by host from external network with static
 111  * mapping.
 112  *
 113  * @param sm     SNAT main.
 114  * @param b0     Vlib buffer.
 115  * @param in2out In2out SNAT session key.
 116  * @param out2in Out2in SNAT session key.
 117  * @param node   Vlib node.
 118  *
 119  * @returns SNAT session if successfully created otherwise 0.
 120  */
 121 static inline snat_session_t *
 122 create_session_for_static_mapping (snat_main_t *sm,
 123                                    vlib_buffer_t *b0,
 124                                    snat_session_key_t in2out,
 125                                    snat_session_key_t out2in,
 126                                    vlib_node_runtime_t * node,
 127                                    u32 cpu_index)
 128 {
 129   snat_user_t *u;
 130   snat_user_key_t user_key;
 131   snat_session_t *s;
 132   clib_bihash_kv_8_8_t kv0, value0;
 133   dlist_elt_t * per_user_translation_list_elt;
 134   dlist_elt_t * per_user_list_head_elt;
 135
 136   user_key.addr = in2out.addr;
 137   user_key.fib_index = in2out.fib_index;
 138   kv0.key = user_key.as_u64;
 139
 140   /* Ever heard of the "user" = inside ip4 address before? */
 141   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 142     {
 143       /* no, make a new one */
 144       pool_get (sm->per_thread_data[cpu_index].users, u);
 145       memset (u, 0, sizeof (*u));
 146       u->addr = in2out.addr;
 147
 148       pool_get (sm->per_thread_data[cpu_index].list_pool,
 149                 per_user_list_head_elt);
 150
 151       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 152         sm->per_thread_data[cpu_index].list_pool;
 153
 154       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 155                        u->sessions_per_user_list_head_index);
 156
 157       kv0.value = u - sm->per_thread_data[cpu_index].users;
 158
 159       /* add user */
 160       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 161
 162       /* add non-traslated packets worker lookup */
 163       kv0.value = cpu_index;
 164       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 165     }
 166   else
 167     {
 168       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 169                              value0.value);
 170     }
 171
 172   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 173   memset (s, 0, sizeof (*s));
 174
 175   s->outside_address_index = ~0;
 176   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 177   u->nstaticsessions++;
 178
 179   /* Create list elts */
 180   pool_get (sm->per_thread_data[cpu_index].list_pool,
 181             per_user_translation_list_elt);
 182   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 183                    per_user_translation_list_elt -
 184                    sm->per_thread_data[cpu_index].list_pool);
 185
 186   per_user_translation_list_elt->value =
 187     s - sm->per_thread_data[cpu_index].sessions;
 188   s->per_user_index =
 189     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 190   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 191
 192   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 193                       s->per_user_list_head_index,
 194                       per_user_translation_list_elt -
 195                       sm->per_thread_data[cpu_index].list_pool);
 196
 197   s->in2out = in2out;
 198   s->out2in = out2in;
 199   s->in2out.protocol = out2in.protocol;
 200
 201   /* Add to translation hashes */
 202   kv0.key = s->in2out.as_u64;
 203   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 204   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 209
 210   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   return s;
 214 }
 215
 216 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 217                                          vlib_buffer_t * b0,
 218                                          ip4_header_t * ip0,
 219                                          icmp46_header_t * icmp0,
 220                                          u32 sw_if_index0,
 221                                          u32 rx_fib_index0,
 222                                          vlib_node_runtime_t * node,
 223                                          u32 next0, f64 now,
 224                                          u32 cpu_index)
 225 {
 226   snat_session_key_t key0, sm0;
 227   icmp_echo_header_t *echo0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229   snat_session_t * s0;
 230   u32 new_addr0, old_addr0;
 231   u16 old_id0, new_id0;
 232   ip_csum_t sum0;
 233   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 234
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   key0.addr = ip0->dst_address;
 238   key0.port = echo0->identifier;
 239   key0.protocol = SNAT_PROTOCOL_ICMP;
 240   key0.fib_index = rx_fib_index0;
 241
 242   kv0.key = key0.as_u64;
 243
 244   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 245     {
 246       /* Try to match static mapping by external address and port,
 247          destination address and port in packet */
 248       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 249         {
 250            ip4_address_t * first_int_addr;
 251
 252           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 253             {
 254               first_int_addr =
 255                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 256                                              0 /* just want the address */);
 257               rt->cached_sw_if_index = sw_if_index0;
 258               if (first_int_addr)
 259                 rt->cached_ip4_address = first_int_addr->as_u32;
 260               else
 261                 rt->cached_ip4_address = 0;
 262             }
 263
 264           /* Don't NAT packet aimed at the intfc address */
 265           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 266                             rt->cached_ip4_address))
 267             return next0;
 268
 269           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 270           return SNAT_OUT2IN_NEXT_DROP;
 271         }
 272
 273       /* Create session initiated by host from external network */
 274       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 275                                              node, cpu_index);
 276       if (!s0)
 277         return SNAT_OUT2IN_NEXT_DROP;
 278     }
 279   else
 280     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 281                             value0.value);
 282
 283   old_addr0 = ip0->dst_address.as_u32;
 284   ip0->dst_address = s0->in2out.addr;
 285   new_addr0 = ip0->dst_address.as_u32;
 286   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 287
 288   sum0 = ip0->checksum;
 289   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 290                          ip4_header_t,
 291                          dst_address /* changed member */);
 292   ip0->checksum = ip_csum_fold (sum0);
 293
 294   old_id0 = echo0->identifier;
 295   new_id0 = s0->in2out.port;
 296   echo0->identifier = new_id0;
 297
 298   sum0 = icmp0->checksum;
 299   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 300                          identifier);
 301   icmp0->checksum = ip_csum_fold (sum0);
 302
 303   /* Accounting */
 304   s0->last_heard = now;
 305   s0->total_pkts++;
 306   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 307   /* Per-user LRU list maintenance for dynamic translation */
 308   if (!snat_is_session_static (s0))
 309     {
 310       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 311                          s0->per_user_index);
 312       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 313                           s0->per_user_list_head_index,
 314                           s0->per_user_index);
 315     }
 316
 317   return next0;
 318 }
 319
 320 static uword
 321 snat_out2in_node_fn (vlib_main_t * vm,
 322                   vlib_node_runtime_t * node,
 323                   vlib_frame_t * frame)
 324 {
 325   u32 n_left_from, * from, * to_next;
 326   snat_out2in_next_t next_index;
 327   u32 pkts_processed = 0;
 328   snat_main_t * sm = &snat_main;
 329   f64 now = vlib_time_now (vm);
 330   u32 cpu_index = os_get_cpu_number ();
 331
 332   from = vlib_frame_vector_args (frame);
 333   n_left_from = frame->n_vectors;
 334   next_index = node->cached_next_index;
 335
 336   while (n_left_from > 0)
 337     {
 338       u32 n_left_to_next;
 339
 340       vlib_get_next_frame (vm, node, next_index,
 341                            to_next, n_left_to_next);
 342
 343       while (n_left_from >= 4 && n_left_to_next >= 2)
 344         {
 345           u32 bi0, bi1;
 346           vlib_buffer_t * b0, * b1;
 347           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 348           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 349           u32 sw_if_index0, sw_if_index1;
 350           ip4_header_t * ip0, *ip1;
 351           ip_csum_t sum0, sum1;
 352           u32 new_addr0, old_addr0;
 353           u16 new_port0, old_port0;
 354           u32 new_addr1, old_addr1;
 355           u16 new_port1, old_port1;
 356           udp_header_t * udp0, * udp1;
 357           tcp_header_t * tcp0, * tcp1;
 358           icmp46_header_t * icmp0, * icmp1;
 359           snat_session_key_t key0, key1, sm0, sm1;
 360           u32 rx_fib_index0, rx_fib_index1;
 361           u32 proto0, proto1;
 362           snat_session_t * s0 = 0, * s1 = 0;
 363           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 364
 365           /* Prefetch next iteration. */
 366           {
 367             vlib_buffer_t * p2, * p3;
 368
 369             p2 = vlib_get_buffer (vm, from[2]);
 370             p3 = vlib_get_buffer (vm, from[3]);
 371
 372             vlib_prefetch_buffer_header (p2, LOAD);
 373             vlib_prefetch_buffer_header (p3, LOAD);
 374
 375             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 376             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 377           }
 378
 379           /* speculatively enqueue b0 and b1 to the current next frame */
 380           to_next[0] = bi0 = from[0];
 381           to_next[1] = bi1 = from[1];
 382           from += 2;
 383           to_next += 2;
 384           n_left_from -= 2;
 385           n_left_to_next -= 2;
 386
 387           b0 = vlib_get_buffer (vm, bi0);
 388           b1 = vlib_get_buffer (vm, bi1);
 389
 390           ip0 = vlib_buffer_get_current (b0);
 391           udp0 = ip4_next_header (ip0);
 392           tcp0 = (tcp_header_t *) udp0;
 393           icmp0 = (icmp46_header_t *) udp0;
 394
 395           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 396           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 397                                    sw_if_index0);
 398
 399           proto0 = ~0;
 400           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 401             ? SNAT_PROTOCOL_UDP : proto0;
 402           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 403             ? SNAT_PROTOCOL_TCP : proto0;
 404           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 405             ? SNAT_PROTOCOL_ICMP : proto0;
 406
 407           if (PREDICT_FALSE (proto0 == ~0))
 408               goto trace0;
 409
 410           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 411             {
 412               next0 = icmp_out2in_slow_path
 413                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 414                  next0, now, cpu_index);
 415               goto trace0;
 416             }
 417
 418           key0.addr = ip0->dst_address;
 419           key0.port = udp0->dst_port;
 420           key0.protocol = proto0;
 421           key0.fib_index = rx_fib_index0;
 422
 423           kv0.key = key0.as_u64;
 424
 425           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 426             {
 427               /* Try to match static mapping by external address and port,
 428                  destination address and port in packet */
 429               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 430                 {
 431                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 432                   goto trace0;
 433                 }
 434
 435               /* Create session initiated by host from external network */
 436               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 437                                                      cpu_index);
 438               if (!s0)
 439                 goto trace0;
 440             }
 441           else
 442             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 443                                     value0.value);
 444
 445           old_addr0 = ip0->dst_address.as_u32;
 446           ip0->dst_address = s0->in2out.addr;
 447           new_addr0 = ip0->dst_address.as_u32;
 448           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 449
 450           sum0 = ip0->checksum;
 451           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 452                                  ip4_header_t,
 453                                  dst_address /* changed member */);
 454           ip0->checksum = ip_csum_fold (sum0);
 455
 456           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 457             {
 458               old_port0 = tcp0->ports.dst;
 459               tcp0->ports.dst = s0->in2out.port;
 460               new_port0 = tcp0->ports.dst;
 461
 462               sum0 = tcp0->checksum;
 463               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 464                                      ip4_header_t,
 465                                      dst_address /* changed member */);
 466
 467               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 468                                      ip4_header_t /* cheat */,
 469                                      length /* changed member */);
 470               tcp0->checksum = ip_csum_fold(sum0);
 471             }
 472           else
 473             {
 474               old_port0 = udp0->dst_port;
 475               udp0->dst_port = s0->in2out.port;
 476               udp0->checksum = 0;
 477             }
 478
 479           /* Accounting */
 480           s0->last_heard = now;
 481           s0->total_pkts++;
 482           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 483           /* Per-user LRU list maintenance for dynamic translation */
 484           if (!snat_is_session_static (s0))
 485             {
 486               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 487                                  s0->per_user_index);
 488               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 489                                   s0->per_user_list_head_index,
 490                                   s0->per_user_index);
 491             }
 492         trace0:
 493
 494           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 495                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 496             {
 497               snat_out2in_trace_t *t =
 498                  vlib_add_trace (vm, node, b0, sizeof (*t));
 499               t->sw_if_index = sw_if_index0;
 500               t->next_index = next0;
 501               t->session_index = ~0;
 502               if (s0)
 503                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 504             }
 505
 506           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 507
 508
 509           ip1 = vlib_buffer_get_current (b1);
 510           udp1 = ip4_next_header (ip1);
 511           tcp1 = (tcp_header_t *) udp1;
 512           icmp1 = (icmp46_header_t *) udp1;
 513
 514           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 515           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 516                                    sw_if_index1);
 517
 518           proto1 = ~0;
 519           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 520             ? SNAT_PROTOCOL_UDP : proto1;
 521           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 522             ? SNAT_PROTOCOL_TCP : proto1;
 523           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 524             ? SNAT_PROTOCOL_ICMP : proto1;
 525
 526           if (PREDICT_FALSE (proto1 == ~0))
 527               goto trace1;
 528
 529           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 530             {
 531               next1 = icmp_out2in_slow_path
 532                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 533                  next1, now, cpu_index);
 534               goto trace1;
 535             }
 536
 537           key1.addr = ip1->dst_address;
 538           key1.port = udp1->dst_port;
 539           key1.protocol = proto1;
 540           key1.fib_index = rx_fib_index1;
 541
 542           kv1.key = key1.as_u64;
 543
 544           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 545             {
 546               /* Try to match static mapping by external address and port,
 547                  destination address and port in packet */
 548               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 549                 {
 550                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 551                   goto trace1;
 552                 }
 553
 554               /* Create session initiated by host from external network */
 555               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 556                                                      cpu_index);
 557               if (!s1)
 558                 goto trace1;
 559             }
 560           else
 561             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 562                                     value1.value);
 563
 564           old_addr1 = ip1->dst_address.as_u32;
 565           ip1->dst_address = s1->in2out.addr;
 566           new_addr1 = ip1->dst_address.as_u32;
 567           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 568
 569           sum1 = ip1->checksum;
 570           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 571                                  ip4_header_t,
 572                                  dst_address /* changed member */);
 573           ip1->checksum = ip_csum_fold (sum1);
 574
 575           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 576             {
 577               old_port1 = tcp1->ports.dst;
 578               tcp1->ports.dst = s1->in2out.port;
 579               new_port1 = tcp1->ports.dst;
 580
 581               sum1 = tcp1->checksum;
 582               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 583                                      ip4_header_t,
 584                                      dst_address /* changed member */);
 585
 586               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 587                                      ip4_header_t /* cheat */,
 588                                      length /* changed member */);
 589               tcp1->checksum = ip_csum_fold(sum1);
 590             }
 591           else
 592             {
 593               old_port1 = udp1->dst_port;
 594               udp1->dst_port = s1->in2out.port;
 595               udp1->checksum = 0;
 596             }
 597
 598           /* Accounting */
 599           s1->last_heard = now;
 600           s1->total_pkts++;
 601           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 602           /* Per-user LRU list maintenance for dynamic translation */
 603           if (!snat_is_session_static (s1))
 604             {
 605               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 606                                  s1->per_user_index);
 607               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 608                                   s1->per_user_list_head_index,
 609                                   s1->per_user_index);
 610             }
 611         trace1:
 612
 613           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 614                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 615             {
 616               snat_out2in_trace_t *t =
 617                  vlib_add_trace (vm, node, b1, sizeof (*t));
 618               t->sw_if_index = sw_if_index1;
 619               t->next_index = next1;
 620               t->session_index = ~0;
 621               if (s1)
 622                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 623             }
 624
 625           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 626
 627           /* verify speculative enqueues, maybe switch current next frame */
 628           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 629                                            to_next, n_left_to_next,
 630                                            bi0, bi1, next0, next1);
 631         }
 632
 633       while (n_left_from > 0 && n_left_to_next > 0)
 634         {
 635           u32 bi0;
 636           vlib_buffer_t * b0;
 637           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 638           u32 sw_if_index0;
 639           ip4_header_t * ip0;
 640           ip_csum_t sum0;
 641           u32 new_addr0, old_addr0;
 642           u16 new_port0, old_port0;
 643           udp_header_t * udp0;
 644           tcp_header_t * tcp0;
 645           icmp46_header_t * icmp0;
 646           snat_session_key_t key0, sm0;
 647           u32 rx_fib_index0;
 648           u32 proto0;
 649           snat_session_t * s0 = 0;
 650           clib_bihash_kv_8_8_t kv0, value0;
 651
 652           /* speculatively enqueue b0 to the current next frame */
 653           bi0 = from[0];
 654           to_next[0] = bi0;
 655           from += 1;
 656           to_next += 1;
 657           n_left_from -= 1;
 658           n_left_to_next -= 1;
 659
 660           b0 = vlib_get_buffer (vm, bi0);
 661
 662           ip0 = vlib_buffer_get_current (b0);
 663           udp0 = ip4_next_header (ip0);
 664           tcp0 = (tcp_header_t *) udp0;
 665           icmp0 = (icmp46_header_t *) udp0;
 666
 667           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 668           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 669                                    sw_if_index0);
 670
 671           proto0 = ~0;
 672           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 673             ? SNAT_PROTOCOL_UDP : proto0;
 674           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 675             ? SNAT_PROTOCOL_TCP : proto0;
 676           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 677             ? SNAT_PROTOCOL_ICMP : proto0;
 678
 679           if (PREDICT_FALSE (proto0 == ~0))
 680               goto trace00;
 681
 682           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 683             {
 684               next0 = icmp_out2in_slow_path
 685                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 686                  next0, now, cpu_index);
 687               goto trace00;
 688             }
 689
 690           key0.addr = ip0->dst_address;
 691           key0.port = udp0->dst_port;
 692           key0.protocol = proto0;
 693           key0.fib_index = rx_fib_index0;
 694
 695           kv0.key = key0.as_u64;
 696
 697           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 698             {
 699               /* Try to match static mapping by external address and port,
 700                  destination address and port in packet */
 701               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 702                 {
 703                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 704                   goto trace00;
 705                 }
 706
 707               /* Create session initiated by host from external network */
 708               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 709                                                      cpu_index);
 710               if (!s0)
 711                 goto trace00;
 712             }
 713           else
 714             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 715                                     value0.value);
 716
 717           old_addr0 = ip0->dst_address.as_u32;
 718           ip0->dst_address = s0->in2out.addr;
 719           new_addr0 = ip0->dst_address.as_u32;
 720           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 721
 722           sum0 = ip0->checksum;
 723           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 724                                  ip4_header_t,
 725                                  dst_address /* changed member */);
 726           ip0->checksum = ip_csum_fold (sum0);
 727
 728           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 729             {
 730               old_port0 = tcp0->ports.dst;
 731               tcp0->ports.dst = s0->in2out.port;
 732               new_port0 = tcp0->ports.dst;
 733
 734               sum0 = tcp0->checksum;
 735               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 736                                      ip4_header_t,
 737                                      dst_address /* changed member */);
 738
 739               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 740                                      ip4_header_t /* cheat */,
 741                                      length /* changed member */);
 742               tcp0->checksum = ip_csum_fold(sum0);
 743             }
 744           else
 745             {
 746               old_port0 = udp0->dst_port;
 747               udp0->dst_port = s0->in2out.port;
 748               udp0->checksum = 0;
 749             }
 750
 751           /* Accounting */
 752           s0->last_heard = now;
 753           s0->total_pkts++;
 754           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 755           /* Per-user LRU list maintenance for dynamic translation */
 756           if (!snat_is_session_static (s0))
 757             {
 758               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 759                                  s0->per_user_index);
 760               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 761                                   s0->per_user_list_head_index,
 762                                   s0->per_user_index);
 763             }
 764         trace00:
 765
 766           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 767                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 768             {
 769               snat_out2in_trace_t *t =
 770                  vlib_add_trace (vm, node, b0, sizeof (*t));
 771               t->sw_if_index = sw_if_index0;
 772               t->next_index = next0;
 773               t->session_index = ~0;
 774               if (s0)
 775                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 776             }
 777
 778           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 779
 780           /* verify speculative enqueue, maybe switch current next frame */
 781           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 782                                            to_next, n_left_to_next,
 783                                            bi0, next0);
 784         }
 785
 786       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 787     }
 788
 789   vlib_node_increment_counter (vm, snat_out2in_node.index,
 790                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 791                                pkts_processed);
 792   return frame->n_vectors;
 793 }
 794
 795 VLIB_REGISTER_NODE (snat_out2in_node) = {
 796   .function = snat_out2in_node_fn,
 797   .name = "snat-out2in",
 798   .vector_size = sizeof (u32),
 799   .format_trace = format_snat_out2in_trace,
 800   .type = VLIB_NODE_TYPE_INTERNAL,
 801
 802   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 803   .error_strings = snat_out2in_error_strings,
 804
 805   .runtime_data_bytes = sizeof (snat_runtime_t),
 806
 807   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 808
 809   /* edit / add dispositions here */
 810   .next_nodes = {
 811     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 812     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 813   },
 814 };
 815 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 816
 817 static uword
 818 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
 819                                vlib_node_runtime_t * node,
 820                                vlib_frame_t * frame)
 821 {
 822   snat_main_t *sm = &snat_main;
 823   vlib_thread_main_t *tm = vlib_get_thread_main ();
 824   u32 n_left_from, *from, *to_next = 0;
 825   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
 826   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
 827     = 0;
 828   vlib_frame_queue_elt_t *hf = 0;
 829   vlib_frame_t *f = 0;
 830   int i;
 831   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
 832   u32 next_worker_index = 0;
 833   u32 current_worker_index = ~0;
 834   u32 cpu_index = os_get_cpu_number ();
 835
 836   ASSERT (vec_len (sm->workers));
 837
 838   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
 839     {
 840       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
 841
 842       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
 843                                sm->first_worker_index + sm->num_workers - 1,
 844                                (vlib_frame_queue_t *) (~0));
 845     }
 846
 847   from = vlib_frame_vector_args (frame);
 848   n_left_from = frame->n_vectors;
 849
 850   while (n_left_from > 0)
 851     {
 852       u32 bi0;
 853       vlib_buffer_t *b0;
 854       u32 sw_if_index0;
 855       u32 rx_fib_index0;
 856       ip4_header_t * ip0;
 857       udp_header_t * udp0;
 858       snat_static_mapping_key_t key0;
 859       clib_bihash_kv_8_8_t kv0, value0;
 860       u8 do_handoff;
 861
 862       bi0 = from[0];
 863       from += 1;
 864       n_left_from -= 1;
 865
 866       b0 = vlib_get_buffer (vm, bi0);
 867
 868       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 869       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
 870
 871       ip0 = vlib_buffer_get_current (b0);
 872       udp0 = ip4_next_header (ip0);
 873
 874       key0.addr = ip0->dst_address;
 875       key0.port = udp0->dst_port;
 876       key0.fib_index = rx_fib_index0;
 877
 878       if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
 879         {
 880           icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
 881           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 882           key0.port = echo0->identifier;
 883         }
 884
 885       kv0.key = key0.as_u64;
 886
 887       /* Ever heard of of the "user" before? */
 888       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 889         {
 890           key0.port = 0;
 891           kv0.key = key0.as_u64;
 892
 893           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 894             {
 895               /* No, assign next available worker (RR) */
 896               next_worker_index = sm->first_worker_index +
 897                 sm->workers[sm->next_worker++ % vec_len (sm->workers)];
 898             }
 899           else
 900             {
 901               /* Static mapping without port */
 902               next_worker_index = value0.value;
 903             }
 904
 905           /* Add to translated packets worker lookup */
 906           kv0.value = next_worker_index;
 907           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 908         }
 909       else
 910         next_worker_index = value0.value;
 911
 912       if (PREDICT_FALSE (next_worker_index != cpu_index))
 913         {
 914           do_handoff = 1;
 915
 916           if (next_worker_index != current_worker_index)
 917             {
 918               if (hf)
 919                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 920
 921               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
 922                                                       next_worker_index,
 923                                                       handoff_queue_elt_by_worker_index);
 924
 925               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
 926               to_next_worker = &hf->buffer_index[hf->n_vectors];
 927               current_worker_index = next_worker_index;
 928             }
 929
 930           /* enqueue to correct worker thread */
 931           to_next_worker[0] = bi0;
 932           to_next_worker++;
 933           n_left_to_next_worker--;
 934
 935           if (n_left_to_next_worker == 0)
 936             {
 937               hf->n_vectors = VLIB_FRAME_SIZE;
 938               vlib_put_frame_queue_elt (hf);
 939               current_worker_index = ~0;
 940               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
 941               hf = 0;
 942             }
 943         }
 944       else
 945         {
 946           do_handoff = 0;
 947           /* if this is 1st frame */
 948           if (!f)
 949             {
 950               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
 951               to_next = vlib_frame_vector_args (f);
 952             }
 953
 954           to_next[0] = bi0;
 955           to_next += 1;
 956           f->n_vectors++;
 957         }
 958
 959       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 960                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 961         {
 962           snat_out2in_worker_handoff_trace_t *t =
 963             vlib_add_trace (vm, node, b0, sizeof (*t));
 964           t->next_worker_index = next_worker_index;
 965           t->do_handoff = do_handoff;
 966         }
 967     }
 968
 969   if (f)
 970     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
 971
 972   if (hf)
 973     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 974
 975   /* Ship frames to the worker nodes */
 976   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
 977     {
 978       if (handoff_queue_elt_by_worker_index[i])
 979         {
 980           hf = handoff_queue_elt_by_worker_index[i];
 981           /*
 982            * It works better to let the handoff node
 983            * rate-adapt, always ship the handoff queue element.
 984            */
 985           if (1 || hf->n_vectors == hf->last_n_vectors)
 986             {
 987               vlib_put_frame_queue_elt (hf);
 988               handoff_queue_elt_by_worker_index[i] = 0;
 989             }
 990           else
 991             hf->last_n_vectors = hf->n_vectors;
 992         }
 993       congested_handoff_queue_by_worker_index[i] =
 994         (vlib_frame_queue_t *) (~0);
 995     }
 996   hf = 0;
 997   current_worker_index = ~0;
 998   return frame->n_vectors;
 999 }
1000
1001 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1002   .function = snat_out2in_worker_handoff_fn,
1003   .name = "snat-out2in-worker-handoff",
1004   .vector_size = sizeof (u32),
1005   .format_trace = format_snat_out2in_worker_handoff_trace,
1006   .type = VLIB_NODE_TYPE_INTERNAL,
1007
1008   .n_next_nodes = 1,
1009
1010   .next_nodes = {
1011     [0] = "error-drop",
1012   },
1013 };
1014
1015 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1016
1017 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1018                                     vlib_buffer_t * b0,
1019                                     ip4_header_t * ip0,
1020                                     icmp46_header_t * icmp0,
1021                                     u32 sw_if_index0,
1022                                     vlib_node_runtime_t * node,
1023                                     u32 next0,
1024                                     u32 rx_fib_index0)
1025 {
1026   snat_session_key_t key0, sm0;
1027   icmp_echo_header_t *echo0;
1028   u32 new_addr0, old_addr0;
1029   u16 old_id0, new_id0;
1030   ip_csum_t sum0;
1031   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1032
1033   echo0 = (icmp_echo_header_t *)(icmp0+1);
1034
1035   key0.addr = ip0->dst_address;
1036   key0.port = echo0->identifier;
1037   key0.fib_index = rx_fib_index0;
1038
1039   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1040     {
1041       ip4_address_t * first_int_addr;
1042
1043       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1044         {
1045           first_int_addr =
1046             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1047                                          0 /* just want the address */);
1048           rt->cached_sw_if_index = sw_if_index0;
1049           if (first_int_addr)
1050             rt->cached_ip4_address = first_int_addr->as_u32;
1051           else
1052             rt->cached_ip4_address = 0;
1053         }
1054
1055       /* Don't NAT packet aimed at the intfc address */
1056       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1057                         rt->cached_ip4_address))
1058         return next0;
1059
1060       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1061       return SNAT_OUT2IN_NEXT_DROP;
1062     }
1063
1064   new_addr0 = sm0.addr.as_u32;
1065   new_id0 = sm0.port;
1066   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1067
1068   old_addr0 = ip0->dst_address.as_u32;
1069   ip0->dst_address.as_u32 = new_addr0;
1070
1071   sum0 = ip0->checksum;
1072   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1073                          ip4_header_t,
1074                          dst_address /* changed member */);
1075   ip0->checksum = ip_csum_fold (sum0);
1076
1077   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1078     {
1079       old_id0 = echo0->identifier;
1080       echo0->identifier = new_id0;
1081
1082       sum0 = icmp0->checksum;
1083       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1084                              identifier);
1085       icmp0->checksum = ip_csum_fold (sum0);
1086     }
1087
1088   return next0;
1089 }
1090
1091 static uword
1092 snat_out2in_fast_node_fn (vlib_main_t * vm,
1093                           vlib_node_runtime_t * node,
1094                           vlib_frame_t * frame)
1095 {
1096   u32 n_left_from, * from, * to_next;
1097   snat_out2in_next_t next_index;
1098   u32 pkts_processed = 0;
1099   snat_main_t * sm = &snat_main;
1100
1101   from = vlib_frame_vector_args (frame);
1102   n_left_from = frame->n_vectors;
1103   next_index = node->cached_next_index;
1104
1105   while (n_left_from > 0)
1106     {
1107       u32 n_left_to_next;
1108
1109       vlib_get_next_frame (vm, node, next_index,
1110                            to_next, n_left_to_next);
1111
1112       while (n_left_from > 0 && n_left_to_next > 0)
1113         {
1114           u32 bi0;
1115           vlib_buffer_t * b0;
1116           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1117           u32 sw_if_index0;
1118           ip4_header_t * ip0;
1119           ip_csum_t sum0;
1120           u32 new_addr0, old_addr0;
1121           u16 new_port0, old_port0;
1122           udp_header_t * udp0;
1123           tcp_header_t * tcp0;
1124           icmp46_header_t * icmp0;
1125           snat_session_key_t key0, sm0;
1126           u32 proto0;
1127           u32 rx_fib_index0;
1128
1129           /* speculatively enqueue b0 to the current next frame */
1130           bi0 = from[0];
1131           to_next[0] = bi0;
1132           from += 1;
1133           to_next += 1;
1134           n_left_from -= 1;
1135           n_left_to_next -= 1;
1136
1137           b0 = vlib_get_buffer (vm, bi0);
1138
1139           ip0 = vlib_buffer_get_current (b0);
1140           udp0 = ip4_next_header (ip0);
1141           tcp0 = (tcp_header_t *) udp0;
1142           icmp0 = (icmp46_header_t *) udp0;
1143
1144           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1145           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1146
1147           vnet_feature_next (sw_if_index0, &next0, b0);
1148
1149           proto0 = ~0;
1150           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1151             ? SNAT_PROTOCOL_UDP : proto0;
1152           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1153             ? SNAT_PROTOCOL_TCP : proto0;
1154           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1155             ? SNAT_PROTOCOL_ICMP : proto0;
1156
1157           if (PREDICT_FALSE (proto0 == ~0))
1158               goto trace00;
1159
1160           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1161             {
1162               next0 = icmp_out2in_fast
1163                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1164               goto trace00;
1165             }
1166
1167           key0.addr = ip0->dst_address;
1168           key0.port = udp0->dst_port;
1169           key0.fib_index = rx_fib_index0;
1170
1171           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1172             {
1173               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1174               goto trace00;
1175             }
1176
1177           new_addr0 = sm0.addr.as_u32;
1178           new_port0 = sm0.port;
1179           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1180           old_addr0 = ip0->dst_address.as_u32;
1181           ip0->dst_address.as_u32 = new_addr0;
1182
1183           sum0 = ip0->checksum;
1184           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1185                                  ip4_header_t,
1186                                  dst_address /* changed member */);
1187           ip0->checksum = ip_csum_fold (sum0);
1188
1189           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1190             {
1191                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1192                 {
1193                   old_port0 = tcp0->ports.dst;
1194                   tcp0->ports.dst = new_port0;
1195
1196                   sum0 = tcp0->checksum;
1197                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1198                                          ip4_header_t,
1199                                          dst_address /* changed member */);
1200
1201                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1202                                          ip4_header_t /* cheat */,
1203                                          length /* changed member */);
1204                   tcp0->checksum = ip_csum_fold(sum0);
1205                 }
1206               else
1207                 {
1208                   old_port0 = udp0->dst_port;
1209                   udp0->dst_port = new_port0;
1210                   udp0->checksum = 0;
1211                 }
1212             }
1213           else
1214             {
1215               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1216                 {
1217                   sum0 = tcp0->checksum;
1218                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1219                                          ip4_header_t,
1220                                          dst_address /* changed member */);
1221
1222                   tcp0->checksum = ip_csum_fold(sum0);
1223                 }
1224             }
1225
1226         trace00:
1227
1228           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1229                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1230             {
1231               snat_out2in_trace_t *t =
1232                  vlib_add_trace (vm, node, b0, sizeof (*t));
1233               t->sw_if_index = sw_if_index0;
1234               t->next_index = next0;
1235             }
1236
1237           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1238
1239           /* verify speculative enqueue, maybe switch current next frame */
1240           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1241                                            to_next, n_left_to_next,
1242                                            bi0, next0);
1243         }
1244
1245       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1246     }
1247
1248   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1249                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1250                                pkts_processed);
1251   return frame->n_vectors;
1252 }
1253
1254 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1255   .function = snat_out2in_fast_node_fn,
1256   .name = "snat-out2in-fast",
1257   .vector_size = sizeof (u32),
1258   .format_trace = format_snat_out2in_fast_trace,
1259   .type = VLIB_NODE_TYPE_INTERNAL,
1260
1261   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1262   .error_strings = snat_out2in_error_strings,
1263
1264   .runtime_data_bytes = sizeof (snat_runtime_t),
1265
1266   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1267
1268   /* edit / add dispositions here */
1269   .next_nodes = {
1270     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1271     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1272   },
1273 };
1274 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);