src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25
  26 #include <vppinfra/hash.h>
  27 #include <vppinfra/error.h>
  28 #include <vppinfra/elog.h>
  29
  30 typedef struct {
  31   u32 sw_if_index;
  32   u32 next_index;
  33   u32 session_index;
  34 } snat_out2in_trace_t;
  35
  36 typedef struct {
  37   u32 next_worker_index;
  38   u8 do_handoff;
  39 } snat_out2in_worker_handoff_trace_t;
  40
  41 /* packet trace format function */
  42 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  43 {
  44   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  45   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  46   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  47
  48   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  49               t->sw_if_index, t->next_index, t->session_index);
  50   return s;
  51 }
  52
  53 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  54 {
  55   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  56   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  57   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  58
  59   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  60               t->sw_if_index, t->next_index);
  61   return s;
  62 }
  63
  64 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  65 {
  66   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  67   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  68   snat_out2in_worker_handoff_trace_t * t =
  69     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  70   char * m;
  71
  72   m = t->do_handoff ? "next worker" : "same worker";
  73   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  74
  75   return s;
  76 }
  77
  78 vlib_node_registration_t snat_out2in_node;
  79 vlib_node_registration_t snat_out2in_fast_node;
  80 vlib_node_registration_t snat_out2in_worker_handoff_node;
  81
  82 #define foreach_snat_out2in_error                       \
  83 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  84 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  85 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  86 _(NO_TRANSLATION, "No translation")
  87
  88 typedef enum {
  89 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  90   foreach_snat_out2in_error
  91 #undef _
  92   SNAT_OUT2IN_N_ERROR,
  93 } snat_out2in_error_t;
  94
  95 static char * snat_out2in_error_strings[] = {
  96 #define _(sym,string) string,
  97   foreach_snat_out2in_error
  98 #undef _
  99 };
 100
 101 typedef enum {
 102   SNAT_OUT2IN_NEXT_DROP,
 103   SNAT_OUT2IN_NEXT_LOOKUP,
 104   SNAT_OUT2IN_N_NEXT,
 105 } snat_out2in_next_t;
 106
 107 /**
 108  * @brief Create session for static mapping.
 109  *
 110  * Create NAT session initiated by host from external network with static
 111  * mapping.
 112  *
 113  * @param sm     SNAT main.
 114  * @param b0     Vlib buffer.
 115  * @param in2out In2out SNAT session key.
 116  * @param out2in Out2in SNAT session key.
 117  * @param node   Vlib node.
 118  *
 119  * @returns SNAT session if successfully created otherwise 0.
 120  */
 121 static inline snat_session_t *
 122 create_session_for_static_mapping (snat_main_t *sm,
 123                                    vlib_buffer_t *b0,
 124                                    snat_session_key_t in2out,
 125                                    snat_session_key_t out2in,
 126                                    vlib_node_runtime_t * node,
 127                                    u32 cpu_index)
 128 {
 129   snat_user_t *u;
 130   snat_user_key_t user_key;
 131   snat_session_t *s;
 132   clib_bihash_kv_8_8_t kv0, value0;
 133   dlist_elt_t * per_user_translation_list_elt;
 134   dlist_elt_t * per_user_list_head_elt;
 135
 136   user_key.addr = in2out.addr;
 137   user_key.fib_index = in2out.fib_index;
 138   kv0.key = user_key.as_u64;
 139
 140   /* Ever heard of the "user" = inside ip4 address before? */
 141   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 142     {
 143       /* no, make a new one */
 144       pool_get (sm->per_thread_data[cpu_index].users, u);
 145       memset (u, 0, sizeof (*u));
 146       u->addr = in2out.addr;
 147
 148       pool_get (sm->per_thread_data[cpu_index].list_pool,
 149                 per_user_list_head_elt);
 150
 151       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 152         sm->per_thread_data[cpu_index].list_pool;
 153
 154       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 155                        u->sessions_per_user_list_head_index);
 156
 157       kv0.value = u - sm->per_thread_data[cpu_index].users;
 158
 159       /* add user */
 160       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 161
 162       /* add non-traslated packets worker lookup */
 163       kv0.value = cpu_index;
 164       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 165     }
 166   else
 167     {
 168       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 169                              value0.value);
 170     }
 171
 172   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 173   memset (s, 0, sizeof (*s));
 174
 175   s->outside_address_index = ~0;
 176   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 177   u->nstaticsessions++;
 178
 179   /* Create list elts */
 180   pool_get (sm->per_thread_data[cpu_index].list_pool,
 181             per_user_translation_list_elt);
 182   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 183                    per_user_translation_list_elt -
 184                    sm->per_thread_data[cpu_index].list_pool);
 185
 186   per_user_translation_list_elt->value =
 187     s - sm->per_thread_data[cpu_index].sessions;
 188   s->per_user_index =
 189     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 190   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 191
 192   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 193                       s->per_user_list_head_index,
 194                       per_user_translation_list_elt -
 195                       sm->per_thread_data[cpu_index].list_pool);
 196
 197   s->in2out = in2out;
 198   s->out2in = out2in;
 199   s->in2out.protocol = out2in.protocol;
 200
 201   /* Add to translation hashes */
 202   kv0.key = s->in2out.as_u64;
 203   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 204   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 205       clib_warning ("in2out key add failed");
 206
 207   kv0.key = s->out2in.as_u64;
 208   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 209
 210   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 211       clib_warning ("out2in key add failed");
 212
 213   return s;
 214 }
 215
 216 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 217                                          vlib_buffer_t * b0,
 218                                          ip4_header_t * ip0,
 219                                          icmp46_header_t * icmp0,
 220                                          u32 sw_if_index0,
 221                                          u32 rx_fib_index0,
 222                                          vlib_node_runtime_t * node,
 223                                          u32 next0, f64 now,
 224                                          u32 cpu_index)
 225 {
 226   snat_session_key_t key0, sm0;
 227   icmp_echo_header_t *echo0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229   snat_session_t * s0;
 230   u32 new_addr0, old_addr0;
 231   u16 old_id0, new_id0;
 232   ip_csum_t sum0;
 233   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 234
 235   echo0 = (icmp_echo_header_t *)(icmp0+1);
 236
 237   key0.addr = ip0->dst_address;
 238   key0.port = echo0->identifier;
 239   key0.protocol = SNAT_PROTOCOL_ICMP;
 240   key0.fib_index = rx_fib_index0;
 241
 242   kv0.key = key0.as_u64;
 243
 244   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 245     {
 246       /* Try to match static mapping by external address and port,
 247          destination address and port in packet */
 248       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 249         {
 250            ip4_address_t * first_int_addr;
 251
 252           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 253             {
 254               first_int_addr =
 255                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 256                                              0 /* just want the address */);
 257               rt->cached_sw_if_index = sw_if_index0;
 258               rt->cached_ip4_address = first_int_addr->as_u32;
 259             }
 260
 261           /* Don't NAT packet aimed at the intfc address */
 262           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 263                             rt->cached_ip4_address))
 264             return next0;
 265
 266           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 267           return SNAT_OUT2IN_NEXT_DROP;
 268         }
 269
 270       /* Create session initiated by host from external network */
 271       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 272                                              node, cpu_index);
 273       if (!s0)
 274         return SNAT_OUT2IN_NEXT_DROP;
 275     }
 276   else
 277     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 278                             value0.value);
 279
 280   old_addr0 = ip0->dst_address.as_u32;
 281   ip0->dst_address = s0->in2out.addr;
 282   new_addr0 = ip0->dst_address.as_u32;
 283   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 284
 285   sum0 = ip0->checksum;
 286   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 287                          ip4_header_t,
 288                          dst_address /* changed member */);
 289   ip0->checksum = ip_csum_fold (sum0);
 290
 291   old_id0 = echo0->identifier;
 292   new_id0 = s0->in2out.port;
 293   echo0->identifier = new_id0;
 294
 295   sum0 = icmp0->checksum;
 296   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 297                          identifier);
 298   icmp0->checksum = ip_csum_fold (sum0);
 299
 300   /* Accounting */
 301   s0->last_heard = now;
 302   s0->total_pkts++;
 303   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 304   /* Per-user LRU list maintenance for dynamic translation */
 305   if (!snat_is_session_static (s0))
 306     {
 307       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 308                          s0->per_user_index);
 309       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 310                           s0->per_user_list_head_index,
 311                           s0->per_user_index);
 312     }
 313
 314   return next0;
 315 }
 316
 317 static uword
 318 snat_out2in_node_fn (vlib_main_t * vm,
 319                   vlib_node_runtime_t * node,
 320                   vlib_frame_t * frame)
 321 {
 322   u32 n_left_from, * from, * to_next;
 323   snat_out2in_next_t next_index;
 324   u32 pkts_processed = 0;
 325   snat_main_t * sm = &snat_main;
 326   f64 now = vlib_time_now (vm);
 327   u32 cpu_index = os_get_cpu_number ();
 328
 329   from = vlib_frame_vector_args (frame);
 330   n_left_from = frame->n_vectors;
 331   next_index = node->cached_next_index;
 332
 333   while (n_left_from > 0)
 334     {
 335       u32 n_left_to_next;
 336
 337       vlib_get_next_frame (vm, node, next_index,
 338                            to_next, n_left_to_next);
 339
 340       while (n_left_from >= 4 && n_left_to_next >= 2)
 341         {
 342           u32 bi0, bi1;
 343           vlib_buffer_t * b0, * b1;
 344           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 345           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 346           u32 sw_if_index0, sw_if_index1;
 347           ip4_header_t * ip0, *ip1;
 348           ip_csum_t sum0, sum1;
 349           u32 new_addr0, old_addr0;
 350           u16 new_port0, old_port0;
 351           u32 new_addr1, old_addr1;
 352           u16 new_port1, old_port1;
 353           udp_header_t * udp0, * udp1;
 354           tcp_header_t * tcp0, * tcp1;
 355           icmp46_header_t * icmp0, * icmp1;
 356           snat_session_key_t key0, key1, sm0, sm1;
 357           u32 rx_fib_index0, rx_fib_index1;
 358           u32 proto0, proto1;
 359           snat_session_t * s0 = 0, * s1 = 0;
 360           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 361
 362           /* Prefetch next iteration. */
 363           {
 364             vlib_buffer_t * p2, * p3;
 365
 366             p2 = vlib_get_buffer (vm, from[2]);
 367             p3 = vlib_get_buffer (vm, from[3]);
 368
 369             vlib_prefetch_buffer_header (p2, LOAD);
 370             vlib_prefetch_buffer_header (p3, LOAD);
 371
 372             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 373             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 374           }
 375
 376           /* speculatively enqueue b0 and b1 to the current next frame */
 377           to_next[0] = bi0 = from[0];
 378           to_next[1] = bi1 = from[1];
 379           from += 2;
 380           to_next += 2;
 381           n_left_from -= 2;
 382           n_left_to_next -= 2;
 383
 384           b0 = vlib_get_buffer (vm, bi0);
 385           b1 = vlib_get_buffer (vm, bi1);
 386
 387           ip0 = vlib_buffer_get_current (b0);
 388           udp0 = ip4_next_header (ip0);
 389           tcp0 = (tcp_header_t *) udp0;
 390           icmp0 = (icmp46_header_t *) udp0;
 391
 392           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 393           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 394                                    sw_if_index0);
 395
 396           proto0 = ~0;
 397           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 398             ? SNAT_PROTOCOL_UDP : proto0;
 399           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 400             ? SNAT_PROTOCOL_TCP : proto0;
 401           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 402             ? SNAT_PROTOCOL_ICMP : proto0;
 403
 404           if (PREDICT_FALSE (proto0 == ~0))
 405               goto trace0;
 406
 407           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 408             {
 409               next0 = icmp_out2in_slow_path
 410                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 411                  next0, now, cpu_index);
 412               goto trace0;
 413             }
 414
 415           key0.addr = ip0->dst_address;
 416           key0.port = udp0->dst_port;
 417           key0.protocol = proto0;
 418           key0.fib_index = rx_fib_index0;
 419
 420           kv0.key = key0.as_u64;
 421
 422           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 423             {
 424               /* Try to match static mapping by external address and port,
 425                  destination address and port in packet */
 426               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 427                 {
 428                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 429                   goto trace0;
 430                 }
 431
 432               /* Create session initiated by host from external network */
 433               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 434                                                      cpu_index);
 435               if (!s0)
 436                 goto trace0;
 437             }
 438           else
 439             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 440                                     value0.value);
 441
 442           old_addr0 = ip0->dst_address.as_u32;
 443           ip0->dst_address = s0->in2out.addr;
 444           new_addr0 = ip0->dst_address.as_u32;
 445           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 446
 447           sum0 = ip0->checksum;
 448           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 449                                  ip4_header_t,
 450                                  dst_address /* changed member */);
 451           ip0->checksum = ip_csum_fold (sum0);
 452
 453           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 454             {
 455               old_port0 = tcp0->ports.dst;
 456               tcp0->ports.dst = s0->in2out.port;
 457               new_port0 = tcp0->ports.dst;
 458
 459               sum0 = tcp0->checksum;
 460               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 461                                      ip4_header_t,
 462                                      dst_address /* changed member */);
 463
 464               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 465                                      ip4_header_t /* cheat */,
 466                                      length /* changed member */);
 467               tcp0->checksum = ip_csum_fold(sum0);
 468             }
 469           else
 470             {
 471               old_port0 = udp0->dst_port;
 472               udp0->dst_port = s0->in2out.port;
 473               udp0->checksum = 0;
 474             }
 475
 476           /* Accounting */
 477           s0->last_heard = now;
 478           s0->total_pkts++;
 479           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 480           /* Per-user LRU list maintenance for dynamic translation */
 481           if (!snat_is_session_static (s0))
 482             {
 483               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 484                                  s0->per_user_index);
 485               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 486                                   s0->per_user_list_head_index,
 487                                   s0->per_user_index);
 488             }
 489         trace0:
 490
 491           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 492                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 493             {
 494               snat_out2in_trace_t *t =
 495                  vlib_add_trace (vm, node, b0, sizeof (*t));
 496               t->sw_if_index = sw_if_index0;
 497               t->next_index = next0;
 498               t->session_index = ~0;
 499               if (s0)
 500                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 501             }
 502
 503           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 504
 505
 506           ip1 = vlib_buffer_get_current (b1);
 507           udp1 = ip4_next_header (ip1);
 508           tcp1 = (tcp_header_t *) udp1;
 509           icmp1 = (icmp46_header_t *) udp1;
 510
 511           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 512           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 513                                    sw_if_index1);
 514
 515           proto1 = ~0;
 516           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 517             ? SNAT_PROTOCOL_UDP : proto1;
 518           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 519             ? SNAT_PROTOCOL_TCP : proto1;
 520           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 521             ? SNAT_PROTOCOL_ICMP : proto1;
 522
 523           if (PREDICT_FALSE (proto1 == ~0))
 524               goto trace1;
 525
 526           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 527             {
 528               next1 = icmp_out2in_slow_path
 529                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 530                  next1, now, cpu_index);
 531               goto trace1;
 532             }
 533
 534           key1.addr = ip1->dst_address;
 535           key1.port = udp1->dst_port;
 536           key1.protocol = proto1;
 537           key1.fib_index = rx_fib_index1;
 538
 539           kv1.key = key1.as_u64;
 540
 541           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 542             {
 543               /* Try to match static mapping by external address and port,
 544                  destination address and port in packet */
 545               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 546                 {
 547                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 548                   goto trace1;
 549                 }
 550
 551               /* Create session initiated by host from external network */
 552               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 553                                                      cpu_index);
 554               if (!s1)
 555                 goto trace1;
 556             }
 557           else
 558             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 559                                     value1.value);
 560
 561           old_addr1 = ip1->dst_address.as_u32;
 562           ip1->dst_address = s1->in2out.addr;
 563           new_addr1 = ip1->dst_address.as_u32;
 564           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 565
 566           sum1 = ip1->checksum;
 567           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 568                                  ip4_header_t,
 569                                  dst_address /* changed member */);
 570           ip1->checksum = ip_csum_fold (sum1);
 571
 572           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 573             {
 574               old_port1 = tcp1->ports.dst;
 575               tcp1->ports.dst = s1->in2out.port;
 576               new_port1 = tcp1->ports.dst;
 577
 578               sum1 = tcp1->checksum;
 579               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 580                                      ip4_header_t,
 581                                      dst_address /* changed member */);
 582
 583               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 584                                      ip4_header_t /* cheat */,
 585                                      length /* changed member */);
 586               tcp1->checksum = ip_csum_fold(sum1);
 587             }
 588           else
 589             {
 590               old_port1 = udp1->dst_port;
 591               udp1->dst_port = s1->in2out.port;
 592               udp1->checksum = 0;
 593             }
 594
 595           /* Accounting */
 596           s1->last_heard = now;
 597           s1->total_pkts++;
 598           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 599           /* Per-user LRU list maintenance for dynamic translation */
 600           if (!snat_is_session_static (s1))
 601             {
 602               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 603                                  s1->per_user_index);
 604               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 605                                   s1->per_user_list_head_index,
 606                                   s1->per_user_index);
 607             }
 608         trace1:
 609
 610           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 611                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 612             {
 613               snat_out2in_trace_t *t =
 614                  vlib_add_trace (vm, node, b1, sizeof (*t));
 615               t->sw_if_index = sw_if_index1;
 616               t->next_index = next1;
 617               t->session_index = ~0;
 618               if (s1)
 619                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 620             }
 621
 622           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 623
 624           /* verify speculative enqueues, maybe switch current next frame */
 625           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 626                                            to_next, n_left_to_next,
 627                                            bi0, bi1, next0, next1);
 628         }
 629
 630       while (n_left_from > 0 && n_left_to_next > 0)
 631         {
 632           u32 bi0;
 633           vlib_buffer_t * b0;
 634           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 635           u32 sw_if_index0;
 636           ip4_header_t * ip0;
 637           ip_csum_t sum0;
 638           u32 new_addr0, old_addr0;
 639           u16 new_port0, old_port0;
 640           udp_header_t * udp0;
 641           tcp_header_t * tcp0;
 642           icmp46_header_t * icmp0;
 643           snat_session_key_t key0, sm0;
 644           u32 rx_fib_index0;
 645           u32 proto0;
 646           snat_session_t * s0 = 0;
 647           clib_bihash_kv_8_8_t kv0, value0;
 648
 649           /* speculatively enqueue b0 to the current next frame */
 650           bi0 = from[0];
 651           to_next[0] = bi0;
 652           from += 1;
 653           to_next += 1;
 654           n_left_from -= 1;
 655           n_left_to_next -= 1;
 656
 657           b0 = vlib_get_buffer (vm, bi0);
 658
 659           ip0 = vlib_buffer_get_current (b0);
 660           udp0 = ip4_next_header (ip0);
 661           tcp0 = (tcp_header_t *) udp0;
 662           icmp0 = (icmp46_header_t *) udp0;
 663
 664           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 665           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 666                                    sw_if_index0);
 667
 668           proto0 = ~0;
 669           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 670             ? SNAT_PROTOCOL_UDP : proto0;
 671           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 672             ? SNAT_PROTOCOL_TCP : proto0;
 673           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 674             ? SNAT_PROTOCOL_ICMP : proto0;
 675
 676           if (PREDICT_FALSE (proto0 == ~0))
 677               goto trace00;
 678
 679           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 680             {
 681               next0 = icmp_out2in_slow_path
 682                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 683                  next0, now, cpu_index);
 684               goto trace00;
 685             }
 686
 687           key0.addr = ip0->dst_address;
 688           key0.port = udp0->dst_port;
 689           key0.protocol = proto0;
 690           key0.fib_index = rx_fib_index0;
 691
 692           kv0.key = key0.as_u64;
 693
 694           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 695             {
 696               /* Try to match static mapping by external address and port,
 697                  destination address and port in packet */
 698               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 699                 {
 700                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 701                   goto trace00;
 702                 }
 703
 704               /* Create session initiated by host from external network */
 705               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 706                                                      cpu_index);
 707               if (!s0)
 708                 goto trace00;
 709             }
 710           else
 711             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 712                                     value0.value);
 713
 714           old_addr0 = ip0->dst_address.as_u32;
 715           ip0->dst_address = s0->in2out.addr;
 716           new_addr0 = ip0->dst_address.as_u32;
 717           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 718
 719           sum0 = ip0->checksum;
 720           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 721                                  ip4_header_t,
 722                                  dst_address /* changed member */);
 723           ip0->checksum = ip_csum_fold (sum0);
 724
 725           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 726             {
 727               old_port0 = tcp0->ports.dst;
 728               tcp0->ports.dst = s0->in2out.port;
 729               new_port0 = tcp0->ports.dst;
 730
 731               sum0 = tcp0->checksum;
 732               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 733                                      ip4_header_t,
 734                                      dst_address /* changed member */);
 735
 736               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 737                                      ip4_header_t /* cheat */,
 738                                      length /* changed member */);
 739               tcp0->checksum = ip_csum_fold(sum0);
 740             }
 741           else
 742             {
 743               old_port0 = udp0->dst_port;
 744               udp0->dst_port = s0->in2out.port;
 745               udp0->checksum = 0;
 746             }
 747
 748           /* Accounting */
 749           s0->last_heard = now;
 750           s0->total_pkts++;
 751           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 752           /* Per-user LRU list maintenance for dynamic translation */
 753           if (!snat_is_session_static (s0))
 754             {
 755               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 756                                  s0->per_user_index);
 757               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 758                                   s0->per_user_list_head_index,
 759                                   s0->per_user_index);
 760             }
 761         trace00:
 762
 763           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 764                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 765             {
 766               snat_out2in_trace_t *t =
 767                  vlib_add_trace (vm, node, b0, sizeof (*t));
 768               t->sw_if_index = sw_if_index0;
 769               t->next_index = next0;
 770               t->session_index = ~0;
 771               if (s0)
 772                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 773             }
 774
 775           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 776
 777           /* verify speculative enqueue, maybe switch current next frame */
 778           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 779                                            to_next, n_left_to_next,
 780                                            bi0, next0);
 781         }
 782
 783       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 784     }
 785
 786   vlib_node_increment_counter (vm, snat_out2in_node.index,
 787                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 788                                pkts_processed);
 789   return frame->n_vectors;
 790 }
 791
 792 VLIB_REGISTER_NODE (snat_out2in_node) = {
 793   .function = snat_out2in_node_fn,
 794   .name = "snat-out2in",
 795   .vector_size = sizeof (u32),
 796   .format_trace = format_snat_out2in_trace,
 797   .type = VLIB_NODE_TYPE_INTERNAL,
 798
 799   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 800   .error_strings = snat_out2in_error_strings,
 801
 802   .runtime_data_bytes = sizeof (snat_runtime_t),
 803
 804   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 805
 806   /* edit / add dispositions here */
 807   .next_nodes = {
 808     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 809     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 810   },
 811 };
 812 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 813
 814 static uword
 815 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
 816                                vlib_node_runtime_t * node,
 817                                vlib_frame_t * frame)
 818 {
 819   snat_main_t *sm = &snat_main;
 820   vlib_thread_main_t *tm = vlib_get_thread_main ();
 821   u32 n_left_from, *from, *to_next = 0;
 822   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
 823   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
 824     = 0;
 825   vlib_frame_queue_elt_t *hf = 0;
 826   vlib_frame_t *f = 0;
 827   int i;
 828   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
 829   u32 next_worker_index = 0;
 830   u32 current_worker_index = ~0;
 831   u32 cpu_index = os_get_cpu_number ();
 832
 833   ASSERT (vec_len (sm->workers));
 834
 835   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
 836     {
 837       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
 838
 839       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
 840                                sm->first_worker_index + sm->num_workers - 1,
 841                                (vlib_frame_queue_t *) (~0));
 842     }
 843
 844   from = vlib_frame_vector_args (frame);
 845   n_left_from = frame->n_vectors;
 846
 847   while (n_left_from > 0)
 848     {
 849       u32 bi0;
 850       vlib_buffer_t *b0;
 851       u32 sw_if_index0;
 852       u32 rx_fib_index0;
 853       ip4_header_t * ip0;
 854       udp_header_t * udp0;
 855       snat_static_mapping_key_t key0;
 856       clib_bihash_kv_8_8_t kv0, value0;
 857       u8 do_handoff;
 858
 859       bi0 = from[0];
 860       from += 1;
 861       n_left_from -= 1;
 862
 863       b0 = vlib_get_buffer (vm, bi0);
 864
 865       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 866       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
 867
 868       ip0 = vlib_buffer_get_current (b0);
 869       udp0 = ip4_next_header (ip0);
 870
 871       key0.addr = ip0->dst_address;
 872       key0.port = udp0->dst_port;
 873       key0.fib_index = rx_fib_index0;
 874
 875       kv0.key = key0.as_u64;
 876
 877       /* Ever heard of of the "user" before? */
 878       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 879         {
 880           key0.port = 0;
 881           kv0.key = key0.as_u64;
 882
 883           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 884             {
 885               /* No, assign next available worker (RR) */
 886               next_worker_index = sm->first_worker_index +
 887                 sm->workers[sm->next_worker++ % vec_len (sm->workers)];
 888             }
 889           else
 890             {
 891               /* Static mapping without port */
 892               next_worker_index = value0.value;
 893             }
 894
 895           /* Add to translated packets worker lookup */
 896           kv0.value = next_worker_index;
 897           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 898         }
 899       else
 900         next_worker_index = value0.value;
 901
 902       if (PREDICT_FALSE (next_worker_index != cpu_index))
 903         {
 904           do_handoff = 1;
 905
 906           if (next_worker_index != current_worker_index)
 907             {
 908               if (hf)
 909                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 910
 911               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
 912                                                       next_worker_index,
 913                                                       handoff_queue_elt_by_worker_index);
 914
 915               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
 916               to_next_worker = &hf->buffer_index[hf->n_vectors];
 917               current_worker_index = next_worker_index;
 918             }
 919
 920           /* enqueue to correct worker thread */
 921           to_next_worker[0] = bi0;
 922           to_next_worker++;
 923           n_left_to_next_worker--;
 924
 925           if (n_left_to_next_worker == 0)
 926             {
 927               hf->n_vectors = VLIB_FRAME_SIZE;
 928               vlib_put_frame_queue_elt (hf);
 929               current_worker_index = ~0;
 930               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
 931               hf = 0;
 932             }
 933         }
 934       else
 935         {
 936           do_handoff = 0;
 937           /* if this is 1st frame */
 938           if (!f)
 939             {
 940               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
 941               to_next = vlib_frame_vector_args (f);
 942             }
 943
 944           to_next[0] = bi0;
 945           to_next += 1;
 946           f->n_vectors++;
 947         }
 948
 949       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 950                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 951         {
 952           snat_out2in_worker_handoff_trace_t *t =
 953             vlib_add_trace (vm, node, b0, sizeof (*t));
 954           t->next_worker_index = next_worker_index;
 955           t->do_handoff = do_handoff;
 956         }
 957     }
 958
 959   if (f)
 960     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
 961
 962   if (hf)
 963     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 964
 965   /* Ship frames to the worker nodes */
 966   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
 967     {
 968       if (handoff_queue_elt_by_worker_index[i])
 969         {
 970           hf = handoff_queue_elt_by_worker_index[i];
 971           /*
 972            * It works better to let the handoff node
 973            * rate-adapt, always ship the handoff queue element.
 974            */
 975           if (1 || hf->n_vectors == hf->last_n_vectors)
 976             {
 977               vlib_put_frame_queue_elt (hf);
 978               handoff_queue_elt_by_worker_index[i] = 0;
 979             }
 980           else
 981             hf->last_n_vectors = hf->n_vectors;
 982         }
 983       congested_handoff_queue_by_worker_index[i] =
 984         (vlib_frame_queue_t *) (~0);
 985     }
 986   hf = 0;
 987   current_worker_index = ~0;
 988   return frame->n_vectors;
 989 }
 990
 991 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
 992   .function = snat_out2in_worker_handoff_fn,
 993   .name = "snat-out2in-worker-handoff",
 994   .vector_size = sizeof (u32),
 995   .format_trace = format_snat_out2in_worker_handoff_trace,
 996   .type = VLIB_NODE_TYPE_INTERNAL,
 997
 998   .n_next_nodes = 1,
 999
1000   .next_nodes = {
1001     [0] = "error-drop",
1002   },
1003 };
1004
1005 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1006
1007 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1008                                     vlib_buffer_t * b0,
1009                                     ip4_header_t * ip0,
1010                                     icmp46_header_t * icmp0,
1011                                     u32 sw_if_index0,
1012                                     vlib_node_runtime_t * node,
1013                                     u32 next0,
1014                                     u32 rx_fib_index0)
1015 {
1016   snat_session_key_t key0, sm0;
1017   icmp_echo_header_t *echo0;
1018   u32 new_addr0, old_addr0;
1019   u16 old_id0, new_id0;
1020   ip_csum_t sum0;
1021   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1022
1023   echo0 = (icmp_echo_header_t *)(icmp0+1);
1024
1025   key0.addr = ip0->dst_address;
1026   key0.port = echo0->identifier;
1027   key0.fib_index = rx_fib_index0;
1028
1029   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1030     {
1031       ip4_address_t * first_int_addr;
1032
1033       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1034         {
1035           first_int_addr =
1036             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1037                                          0 /* just want the address */);
1038           rt->cached_sw_if_index = sw_if_index0;
1039           rt->cached_ip4_address = first_int_addr->as_u32;
1040         }
1041
1042       /* Don't NAT packet aimed at the intfc address */
1043       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1044                         rt->cached_ip4_address))
1045         return next0;
1046
1047       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1048       return SNAT_OUT2IN_NEXT_DROP;
1049     }
1050
1051   new_addr0 = sm0.addr.as_u32;
1052   new_id0 = sm0.port;
1053   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1054
1055   old_addr0 = ip0->dst_address.as_u32;
1056   ip0->dst_address.as_u32 = new_addr0;
1057
1058   sum0 = ip0->checksum;
1059   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1060                          ip4_header_t,
1061                          dst_address /* changed member */);
1062   ip0->checksum = ip_csum_fold (sum0);
1063
1064   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1065     {
1066       old_id0 = echo0->identifier;
1067       echo0->identifier = new_id0;
1068
1069       sum0 = icmp0->checksum;
1070       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1071                              identifier);
1072       icmp0->checksum = ip_csum_fold (sum0);
1073     }
1074
1075   return next0;
1076 }
1077
1078 static uword
1079 snat_out2in_fast_node_fn (vlib_main_t * vm,
1080                           vlib_node_runtime_t * node,
1081                           vlib_frame_t * frame)
1082 {
1083   u32 n_left_from, * from, * to_next;
1084   snat_out2in_next_t next_index;
1085   u32 pkts_processed = 0;
1086   snat_main_t * sm = &snat_main;
1087
1088   from = vlib_frame_vector_args (frame);
1089   n_left_from = frame->n_vectors;
1090   next_index = node->cached_next_index;
1091
1092   while (n_left_from > 0)
1093     {
1094       u32 n_left_to_next;
1095
1096       vlib_get_next_frame (vm, node, next_index,
1097                            to_next, n_left_to_next);
1098
1099       while (n_left_from > 0 && n_left_to_next > 0)
1100         {
1101           u32 bi0;
1102           vlib_buffer_t * b0;
1103           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1104           u32 sw_if_index0;
1105           ip4_header_t * ip0;
1106           ip_csum_t sum0;
1107           u32 new_addr0, old_addr0;
1108           u16 new_port0, old_port0;
1109           udp_header_t * udp0;
1110           tcp_header_t * tcp0;
1111           icmp46_header_t * icmp0;
1112           snat_session_key_t key0, sm0;
1113           u32 proto0;
1114           u32 rx_fib_index0;
1115
1116           /* speculatively enqueue b0 to the current next frame */
1117           bi0 = from[0];
1118           to_next[0] = bi0;
1119           from += 1;
1120           to_next += 1;
1121           n_left_from -= 1;
1122           n_left_to_next -= 1;
1123
1124           b0 = vlib_get_buffer (vm, bi0);
1125
1126           ip0 = vlib_buffer_get_current (b0);
1127           udp0 = ip4_next_header (ip0);
1128           tcp0 = (tcp_header_t *) udp0;
1129           icmp0 = (icmp46_header_t *) udp0;
1130
1131           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1132           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1133
1134           vnet_feature_next (sw_if_index0, &next0, b0);
1135
1136           proto0 = ~0;
1137           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1138             ? SNAT_PROTOCOL_UDP : proto0;
1139           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1140             ? SNAT_PROTOCOL_TCP : proto0;
1141           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1142             ? SNAT_PROTOCOL_ICMP : proto0;
1143
1144           if (PREDICT_FALSE (proto0 == ~0))
1145               goto trace00;
1146
1147           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1148             {
1149               next0 = icmp_out2in_fast
1150                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1151               goto trace00;
1152             }
1153
1154           key0.addr = ip0->dst_address;
1155           key0.port = udp0->dst_port;
1156           key0.fib_index = rx_fib_index0;
1157
1158           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1159             {
1160               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1161               goto trace00;
1162             }
1163
1164           new_addr0 = sm0.addr.as_u32;
1165           new_port0 = sm0.port;
1166           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1167           old_addr0 = ip0->dst_address.as_u32;
1168           ip0->dst_address.as_u32 = new_addr0;
1169
1170           sum0 = ip0->checksum;
1171           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1172                                  ip4_header_t,
1173                                  dst_address /* changed member */);
1174           ip0->checksum = ip_csum_fold (sum0);
1175
1176           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1177             {
1178                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1179                 {
1180                   old_port0 = tcp0->ports.dst;
1181                   tcp0->ports.dst = new_port0;
1182
1183                   sum0 = tcp0->checksum;
1184                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1185                                          ip4_header_t,
1186                                          dst_address /* changed member */);
1187
1188                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1189                                          ip4_header_t /* cheat */,
1190                                          length /* changed member */);
1191                   tcp0->checksum = ip_csum_fold(sum0);
1192                 }
1193               else
1194                 {
1195                   old_port0 = udp0->dst_port;
1196                   udp0->dst_port = new_port0;
1197                   udp0->checksum = 0;
1198                 }
1199             }
1200           else
1201             {
1202               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1203                 {
1204                   sum0 = tcp0->checksum;
1205                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1206                                          ip4_header_t,
1207                                          dst_address /* changed member */);
1208
1209                   tcp0->checksum = ip_csum_fold(sum0);
1210                 }
1211             }
1212
1213         trace00:
1214
1215           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1216                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1217             {
1218               snat_out2in_trace_t *t =
1219                  vlib_add_trace (vm, node, b0, sizeof (*t));
1220               t->sw_if_index = sw_if_index0;
1221               t->next_index = next0;
1222             }
1223
1224           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1225
1226           /* verify speculative enqueue, maybe switch current next frame */
1227           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1228                                            to_next, n_left_to_next,
1229                                            bi0, next0);
1230         }
1231
1232       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1233     }
1234
1235   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1236                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1237                                pkts_processed);
1238   return frame->n_vectors;
1239 }
1240
1241 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1242   .function = snat_out2in_fast_node_fn,
1243   .name = "snat-out2in-fast",
1244   .vector_size = sizeof (u32),
1245   .format_trace = format_snat_out2in_fast_trace,
1246   .type = VLIB_NODE_TYPE_INTERNAL,
1247
1248   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1249   .error_strings = snat_out2in_error_strings,
1250
1251   .runtime_data_bytes = sizeof (snat_runtime_t),
1252
1253   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1254
1255   /* edit / add dispositions here */
1256   .next_nodes = {
1257     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1258     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1259   },
1260 };
1261 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);