src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25 #include <snat/snat_ipfix_logging.h>
  26
  27 #include <vppinfra/hash.h>
  28 #include <vppinfra/error.h>
  29 #include <vppinfra/elog.h>
  30
  31 typedef struct {
  32   u32 sw_if_index;
  33   u32 next_index;
  34   u32 session_index;
  35 } snat_out2in_trace_t;
  36
  37 typedef struct {
  38   u32 next_worker_index;
  39   u8 do_handoff;
  40 } snat_out2in_worker_handoff_trace_t;
  41
  42 /* packet trace format function */
  43 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  44 {
  45   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  46   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  47   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  48
  49   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  50               t->sw_if_index, t->next_index, t->session_index);
  51   return s;
  52 }
  53
  54 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  55 {
  56   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  57   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  58   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  59
  60   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  61               t->sw_if_index, t->next_index);
  62   return s;
  63 }
  64
  65 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  66 {
  67   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  68   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  69   snat_out2in_worker_handoff_trace_t * t =
  70     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  71   char * m;
  72
  73   m = t->do_handoff ? "next worker" : "same worker";
  74   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  75
  76   return s;
  77 }
  78
  79 vlib_node_registration_t snat_out2in_node;
  80 vlib_node_registration_t snat_out2in_fast_node;
  81 vlib_node_registration_t snat_out2in_worker_handoff_node;
  82
  83 #define foreach_snat_out2in_error                       \
  84 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  85 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  86 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  87 _(NO_TRANSLATION, "No translation")
  88
  89 typedef enum {
  90 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  91   foreach_snat_out2in_error
  92 #undef _
  93   SNAT_OUT2IN_N_ERROR,
  94 } snat_out2in_error_t;
  95
  96 static char * snat_out2in_error_strings[] = {
  97 #define _(sym,string) string,
  98   foreach_snat_out2in_error
  99 #undef _
 100 };
 101
 102 typedef enum {
 103   SNAT_OUT2IN_NEXT_DROP,
 104   SNAT_OUT2IN_NEXT_LOOKUP,
 105   SNAT_OUT2IN_N_NEXT,
 106 } snat_out2in_next_t;
 107
 108 /**
 109  * @brief Create session for static mapping.
 110  *
 111  * Create NAT session initiated by host from external network with static
 112  * mapping.
 113  *
 114  * @param sm     SNAT main.
 115  * @param b0     Vlib buffer.
 116  * @param in2out In2out SNAT session key.
 117  * @param out2in Out2in SNAT session key.
 118  * @param node   Vlib node.
 119  *
 120  * @returns SNAT session if successfully created otherwise 0.
 121  */
 122 static inline snat_session_t *
 123 create_session_for_static_mapping (snat_main_t *sm,
 124                                    vlib_buffer_t *b0,
 125                                    snat_session_key_t in2out,
 126                                    snat_session_key_t out2in,
 127                                    vlib_node_runtime_t * node,
 128                                    u32 cpu_index)
 129 {
 130   snat_user_t *u;
 131   snat_user_key_t user_key;
 132   snat_session_t *s;
 133   clib_bihash_kv_8_8_t kv0, value0;
 134   dlist_elt_t * per_user_translation_list_elt;
 135   dlist_elt_t * per_user_list_head_elt;
 136
 137   user_key.addr = in2out.addr;
 138   user_key.fib_index = in2out.fib_index;
 139   kv0.key = user_key.as_u64;
 140
 141   /* Ever heard of the "user" = inside ip4 address before? */
 142   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 143     {
 144       /* no, make a new one */
 145       pool_get (sm->per_thread_data[cpu_index].users, u);
 146       memset (u, 0, sizeof (*u));
 147       u->addr = in2out.addr;
 148
 149       pool_get (sm->per_thread_data[cpu_index].list_pool,
 150                 per_user_list_head_elt);
 151
 152       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 153         sm->per_thread_data[cpu_index].list_pool;
 154
 155       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 156                        u->sessions_per_user_list_head_index);
 157
 158       kv0.value = u - sm->per_thread_data[cpu_index].users;
 159
 160       /* add user */
 161       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 162
 163       /* add non-traslated packets worker lookup */
 164       kv0.value = cpu_index;
 165       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 166     }
 167   else
 168     {
 169       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 170                              value0.value);
 171     }
 172
 173   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 174   memset (s, 0, sizeof (*s));
 175
 176   s->outside_address_index = ~0;
 177   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 178   u->nstaticsessions++;
 179
 180   /* Create list elts */
 181   pool_get (sm->per_thread_data[cpu_index].list_pool,
 182             per_user_translation_list_elt);
 183   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 184                    per_user_translation_list_elt -
 185                    sm->per_thread_data[cpu_index].list_pool);
 186
 187   per_user_translation_list_elt->value =
 188     s - sm->per_thread_data[cpu_index].sessions;
 189   s->per_user_index =
 190     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 191   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 192
 193   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 194                       s->per_user_list_head_index,
 195                       per_user_translation_list_elt -
 196                       sm->per_thread_data[cpu_index].list_pool);
 197
 198   s->in2out = in2out;
 199   s->out2in = out2in;
 200   s->in2out.protocol = out2in.protocol;
 201
 202   /* Add to translation hashes */
 203   kv0.key = s->in2out.as_u64;
 204   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 205   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 206       clib_warning ("in2out key add failed");
 207
 208   kv0.key = s->out2in.as_u64;
 209   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 210
 211   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 212       clib_warning ("out2in key add failed");
 213
 214   /* log NAT event */
 215   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 216                                       s->out2in.addr.as_u32,
 217                                       s->in2out.protocol,
 218                                       s->in2out.port,
 219                                       s->out2in.port,
 220                                       s->in2out.fib_index);
 221    return s;
 222 }
 223
 224 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 225                                          vlib_buffer_t * b0,
 226                                          ip4_header_t * ip0,
 227                                          icmp46_header_t * icmp0,
 228                                          u32 sw_if_index0,
 229                                          u32 rx_fib_index0,
 230                                          vlib_node_runtime_t * node,
 231                                          u32 next0, f64 now,
 232                                          u32 cpu_index)
 233 {
 234   snat_session_key_t key0, sm0;
 235   icmp_echo_header_t *echo0;
 236   clib_bihash_kv_8_8_t kv0, value0;
 237   snat_session_t * s0;
 238   u32 new_addr0, old_addr0;
 239   u16 old_id0, new_id0;
 240   ip_csum_t sum0;
 241   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 242
 243   echo0 = (icmp_echo_header_t *)(icmp0+1);
 244
 245   key0.addr = ip0->dst_address;
 246   key0.port = echo0->identifier;
 247   key0.protocol = SNAT_PROTOCOL_ICMP;
 248   key0.fib_index = rx_fib_index0;
 249
 250   kv0.key = key0.as_u64;
 251
 252   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 253     {
 254       /* Try to match static mapping by external address and port,
 255          destination address and port in packet */
 256       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 257         {
 258            ip4_address_t * first_int_addr;
 259
 260           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 261             {
 262               first_int_addr =
 263                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 264                                              0 /* just want the address */);
 265               rt->cached_sw_if_index = sw_if_index0;
 266               if (first_int_addr)
 267                 rt->cached_ip4_address = first_int_addr->as_u32;
 268               else
 269                 rt->cached_ip4_address = 0;
 270             }
 271
 272           /* Don't NAT packet aimed at the intfc address */
 273           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 274                             rt->cached_ip4_address))
 275             return next0;
 276
 277           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 278           return SNAT_OUT2IN_NEXT_DROP;
 279         }
 280
 281       /* Create session initiated by host from external network */
 282       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 283                                              node, cpu_index);
 284       if (!s0)
 285         return SNAT_OUT2IN_NEXT_DROP;
 286     }
 287   else
 288     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 289                             value0.value);
 290
 291   old_addr0 = ip0->dst_address.as_u32;
 292   ip0->dst_address = s0->in2out.addr;
 293   new_addr0 = ip0->dst_address.as_u32;
 294   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 295
 296   sum0 = ip0->checksum;
 297   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 298                          ip4_header_t,
 299                          dst_address /* changed member */);
 300   ip0->checksum = ip_csum_fold (sum0);
 301
 302   old_id0 = echo0->identifier;
 303   new_id0 = s0->in2out.port;
 304   echo0->identifier = new_id0;
 305
 306   sum0 = icmp0->checksum;
 307   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 308                          identifier);
 309   icmp0->checksum = ip_csum_fold (sum0);
 310
 311   /* Accounting */
 312   s0->last_heard = now;
 313   s0->total_pkts++;
 314   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 315   /* Per-user LRU list maintenance for dynamic translation */
 316   if (!snat_is_session_static (s0))
 317     {
 318       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 319                          s0->per_user_index);
 320       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 321                           s0->per_user_list_head_index,
 322                           s0->per_user_index);
 323     }
 324
 325   return next0;
 326 }
 327
 328 static uword
 329 snat_out2in_node_fn (vlib_main_t * vm,
 330                   vlib_node_runtime_t * node,
 331                   vlib_frame_t * frame)
 332 {
 333   u32 n_left_from, * from, * to_next;
 334   snat_out2in_next_t next_index;
 335   u32 pkts_processed = 0;
 336   snat_main_t * sm = &snat_main;
 337   f64 now = vlib_time_now (vm);
 338   u32 cpu_index = os_get_cpu_number ();
 339
 340   from = vlib_frame_vector_args (frame);
 341   n_left_from = frame->n_vectors;
 342   next_index = node->cached_next_index;
 343
 344   while (n_left_from > 0)
 345     {
 346       u32 n_left_to_next;
 347
 348       vlib_get_next_frame (vm, node, next_index,
 349                            to_next, n_left_to_next);
 350
 351       while (n_left_from >= 4 && n_left_to_next >= 2)
 352         {
 353           u32 bi0, bi1;
 354           vlib_buffer_t * b0, * b1;
 355           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 356           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 357           u32 sw_if_index0, sw_if_index1;
 358           ip4_header_t * ip0, *ip1;
 359           ip_csum_t sum0, sum1;
 360           u32 new_addr0, old_addr0;
 361           u16 new_port0, old_port0;
 362           u32 new_addr1, old_addr1;
 363           u16 new_port1, old_port1;
 364           udp_header_t * udp0, * udp1;
 365           tcp_header_t * tcp0, * tcp1;
 366           icmp46_header_t * icmp0, * icmp1;
 367           snat_session_key_t key0, key1, sm0, sm1;
 368           u32 rx_fib_index0, rx_fib_index1;
 369           u32 proto0, proto1;
 370           snat_session_t * s0 = 0, * s1 = 0;
 371           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 372
 373           /* Prefetch next iteration. */
 374           {
 375             vlib_buffer_t * p2, * p3;
 376
 377             p2 = vlib_get_buffer (vm, from[2]);
 378             p3 = vlib_get_buffer (vm, from[3]);
 379
 380             vlib_prefetch_buffer_header (p2, LOAD);
 381             vlib_prefetch_buffer_header (p3, LOAD);
 382
 383             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 384             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 385           }
 386
 387           /* speculatively enqueue b0 and b1 to the current next frame */
 388           to_next[0] = bi0 = from[0];
 389           to_next[1] = bi1 = from[1];
 390           from += 2;
 391           to_next += 2;
 392           n_left_from -= 2;
 393           n_left_to_next -= 2;
 394
 395           b0 = vlib_get_buffer (vm, bi0);
 396           b1 = vlib_get_buffer (vm, bi1);
 397
 398           ip0 = vlib_buffer_get_current (b0);
 399           udp0 = ip4_next_header (ip0);
 400           tcp0 = (tcp_header_t *) udp0;
 401           icmp0 = (icmp46_header_t *) udp0;
 402
 403           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 404           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 405                                    sw_if_index0);
 406
 407           proto0 = ~0;
 408           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 409             ? SNAT_PROTOCOL_UDP : proto0;
 410           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 411             ? SNAT_PROTOCOL_TCP : proto0;
 412           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 413             ? SNAT_PROTOCOL_ICMP : proto0;
 414
 415           if (PREDICT_FALSE (proto0 == ~0))
 416               goto trace0;
 417
 418           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 419             {
 420               next0 = icmp_out2in_slow_path
 421                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 422                  next0, now, cpu_index);
 423               goto trace0;
 424             }
 425
 426           key0.addr = ip0->dst_address;
 427           key0.port = udp0->dst_port;
 428           key0.protocol = proto0;
 429           key0.fib_index = rx_fib_index0;
 430
 431           kv0.key = key0.as_u64;
 432
 433           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 434             {
 435               /* Try to match static mapping by external address and port,
 436                  destination address and port in packet */
 437               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 438                 {
 439                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 440                   goto trace0;
 441                 }
 442
 443               /* Create session initiated by host from external network */
 444               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 445                                                      cpu_index);
 446               if (!s0)
 447                 goto trace0;
 448             }
 449           else
 450             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 451                                     value0.value);
 452
 453           old_addr0 = ip0->dst_address.as_u32;
 454           ip0->dst_address = s0->in2out.addr;
 455           new_addr0 = ip0->dst_address.as_u32;
 456           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 457
 458           sum0 = ip0->checksum;
 459           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 460                                  ip4_header_t,
 461                                  dst_address /* changed member */);
 462           ip0->checksum = ip_csum_fold (sum0);
 463
 464           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 465             {
 466               old_port0 = tcp0->ports.dst;
 467               tcp0->ports.dst = s0->in2out.port;
 468               new_port0 = tcp0->ports.dst;
 469
 470               sum0 = tcp0->checksum;
 471               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 472                                      ip4_header_t,
 473                                      dst_address /* changed member */);
 474
 475               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 476                                      ip4_header_t /* cheat */,
 477                                      length /* changed member */);
 478               tcp0->checksum = ip_csum_fold(sum0);
 479             }
 480           else
 481             {
 482               old_port0 = udp0->dst_port;
 483               udp0->dst_port = s0->in2out.port;
 484               udp0->checksum = 0;
 485             }
 486
 487           /* Accounting */
 488           s0->last_heard = now;
 489           s0->total_pkts++;
 490           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 491           /* Per-user LRU list maintenance for dynamic translation */
 492           if (!snat_is_session_static (s0))
 493             {
 494               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 495                                  s0->per_user_index);
 496               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 497                                   s0->per_user_list_head_index,
 498                                   s0->per_user_index);
 499             }
 500         trace0:
 501
 502           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 503                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 504             {
 505               snat_out2in_trace_t *t =
 506                  vlib_add_trace (vm, node, b0, sizeof (*t));
 507               t->sw_if_index = sw_if_index0;
 508               t->next_index = next0;
 509               t->session_index = ~0;
 510               if (s0)
 511                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 512             }
 513
 514           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 515
 516
 517           ip1 = vlib_buffer_get_current (b1);
 518           udp1 = ip4_next_header (ip1);
 519           tcp1 = (tcp_header_t *) udp1;
 520           icmp1 = (icmp46_header_t *) udp1;
 521
 522           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 523           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 524                                    sw_if_index1);
 525
 526           proto1 = ~0;
 527           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 528             ? SNAT_PROTOCOL_UDP : proto1;
 529           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 530             ? SNAT_PROTOCOL_TCP : proto1;
 531           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 532             ? SNAT_PROTOCOL_ICMP : proto1;
 533
 534           if (PREDICT_FALSE (proto1 == ~0))
 535               goto trace1;
 536
 537           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 538             {
 539               next1 = icmp_out2in_slow_path
 540                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 541                  next1, now, cpu_index);
 542               goto trace1;
 543             }
 544
 545           key1.addr = ip1->dst_address;
 546           key1.port = udp1->dst_port;
 547           key1.protocol = proto1;
 548           key1.fib_index = rx_fib_index1;
 549
 550           kv1.key = key1.as_u64;
 551
 552           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 553             {
 554               /* Try to match static mapping by external address and port,
 555                  destination address and port in packet */
 556               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 557                 {
 558                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 559                   goto trace1;
 560                 }
 561
 562               /* Create session initiated by host from external network */
 563               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 564                                                      cpu_index);
 565               if (!s1)
 566                 goto trace1;
 567             }
 568           else
 569             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 570                                     value1.value);
 571
 572           old_addr1 = ip1->dst_address.as_u32;
 573           ip1->dst_address = s1->in2out.addr;
 574           new_addr1 = ip1->dst_address.as_u32;
 575           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 576
 577           sum1 = ip1->checksum;
 578           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 579                                  ip4_header_t,
 580                                  dst_address /* changed member */);
 581           ip1->checksum = ip_csum_fold (sum1);
 582
 583           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 584             {
 585               old_port1 = tcp1->ports.dst;
 586               tcp1->ports.dst = s1->in2out.port;
 587               new_port1 = tcp1->ports.dst;
 588
 589               sum1 = tcp1->checksum;
 590               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 591                                      ip4_header_t,
 592                                      dst_address /* changed member */);
 593
 594               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 595                                      ip4_header_t /* cheat */,
 596                                      length /* changed member */);
 597               tcp1->checksum = ip_csum_fold(sum1);
 598             }
 599           else
 600             {
 601               old_port1 = udp1->dst_port;
 602               udp1->dst_port = s1->in2out.port;
 603               udp1->checksum = 0;
 604             }
 605
 606           /* Accounting */
 607           s1->last_heard = now;
 608           s1->total_pkts++;
 609           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 610           /* Per-user LRU list maintenance for dynamic translation */
 611           if (!snat_is_session_static (s1))
 612             {
 613               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 614                                  s1->per_user_index);
 615               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 616                                   s1->per_user_list_head_index,
 617                                   s1->per_user_index);
 618             }
 619         trace1:
 620
 621           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 622                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 623             {
 624               snat_out2in_trace_t *t =
 625                  vlib_add_trace (vm, node, b1, sizeof (*t));
 626               t->sw_if_index = sw_if_index1;
 627               t->next_index = next1;
 628               t->session_index = ~0;
 629               if (s1)
 630                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 631             }
 632
 633           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 634
 635           /* verify speculative enqueues, maybe switch current next frame */
 636           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 637                                            to_next, n_left_to_next,
 638                                            bi0, bi1, next0, next1);
 639         }
 640
 641       while (n_left_from > 0 && n_left_to_next > 0)
 642         {
 643           u32 bi0;
 644           vlib_buffer_t * b0;
 645           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 646           u32 sw_if_index0;
 647           ip4_header_t * ip0;
 648           ip_csum_t sum0;
 649           u32 new_addr0, old_addr0;
 650           u16 new_port0, old_port0;
 651           udp_header_t * udp0;
 652           tcp_header_t * tcp0;
 653           icmp46_header_t * icmp0;
 654           snat_session_key_t key0, sm0;
 655           u32 rx_fib_index0;
 656           u32 proto0;
 657           snat_session_t * s0 = 0;
 658           clib_bihash_kv_8_8_t kv0, value0;
 659
 660           /* speculatively enqueue b0 to the current next frame */
 661           bi0 = from[0];
 662           to_next[0] = bi0;
 663           from += 1;
 664           to_next += 1;
 665           n_left_from -= 1;
 666           n_left_to_next -= 1;
 667
 668           b0 = vlib_get_buffer (vm, bi0);
 669
 670           ip0 = vlib_buffer_get_current (b0);
 671           udp0 = ip4_next_header (ip0);
 672           tcp0 = (tcp_header_t *) udp0;
 673           icmp0 = (icmp46_header_t *) udp0;
 674
 675           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 676           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 677                                    sw_if_index0);
 678
 679           proto0 = ~0;
 680           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 681             ? SNAT_PROTOCOL_UDP : proto0;
 682           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 683             ? SNAT_PROTOCOL_TCP : proto0;
 684           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 685             ? SNAT_PROTOCOL_ICMP : proto0;
 686
 687           if (PREDICT_FALSE (proto0 == ~0))
 688               goto trace00;
 689
 690           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 691             {
 692               next0 = icmp_out2in_slow_path
 693                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 694                  next0, now, cpu_index);
 695               goto trace00;
 696             }
 697
 698           key0.addr = ip0->dst_address;
 699           key0.port = udp0->dst_port;
 700           key0.protocol = proto0;
 701           key0.fib_index = rx_fib_index0;
 702
 703           kv0.key = key0.as_u64;
 704
 705           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 706             {
 707               /* Try to match static mapping by external address and port,
 708                  destination address and port in packet */
 709               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 710                 {
 711                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 712                   goto trace00;
 713                 }
 714
 715               /* Create session initiated by host from external network */
 716               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 717                                                      cpu_index);
 718               if (!s0)
 719                 goto trace00;
 720             }
 721           else
 722             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 723                                     value0.value);
 724
 725           old_addr0 = ip0->dst_address.as_u32;
 726           ip0->dst_address = s0->in2out.addr;
 727           new_addr0 = ip0->dst_address.as_u32;
 728           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 729
 730           sum0 = ip0->checksum;
 731           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 732                                  ip4_header_t,
 733                                  dst_address /* changed member */);
 734           ip0->checksum = ip_csum_fold (sum0);
 735
 736           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 737             {
 738               old_port0 = tcp0->ports.dst;
 739               tcp0->ports.dst = s0->in2out.port;
 740               new_port0 = tcp0->ports.dst;
 741
 742               sum0 = tcp0->checksum;
 743               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 744                                      ip4_header_t,
 745                                      dst_address /* changed member */);
 746
 747               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 748                                      ip4_header_t /* cheat */,
 749                                      length /* changed member */);
 750               tcp0->checksum = ip_csum_fold(sum0);
 751             }
 752           else
 753             {
 754               old_port0 = udp0->dst_port;
 755               udp0->dst_port = s0->in2out.port;
 756               udp0->checksum = 0;
 757             }
 758
 759           /* Accounting */
 760           s0->last_heard = now;
 761           s0->total_pkts++;
 762           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 763           /* Per-user LRU list maintenance for dynamic translation */
 764           if (!snat_is_session_static (s0))
 765             {
 766               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 767                                  s0->per_user_index);
 768               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 769                                   s0->per_user_list_head_index,
 770                                   s0->per_user_index);
 771             }
 772         trace00:
 773
 774           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 775                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 776             {
 777               snat_out2in_trace_t *t =
 778                  vlib_add_trace (vm, node, b0, sizeof (*t));
 779               t->sw_if_index = sw_if_index0;
 780               t->next_index = next0;
 781               t->session_index = ~0;
 782               if (s0)
 783                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 784             }
 785
 786           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 787
 788           /* verify speculative enqueue, maybe switch current next frame */
 789           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 790                                            to_next, n_left_to_next,
 791                                            bi0, next0);
 792         }
 793
 794       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 795     }
 796
 797   vlib_node_increment_counter (vm, snat_out2in_node.index,
 798                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 799                                pkts_processed);
 800   return frame->n_vectors;
 801 }
 802
 803 VLIB_REGISTER_NODE (snat_out2in_node) = {
 804   .function = snat_out2in_node_fn,
 805   .name = "snat-out2in",
 806   .vector_size = sizeof (u32),
 807   .format_trace = format_snat_out2in_trace,
 808   .type = VLIB_NODE_TYPE_INTERNAL,
 809
 810   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 811   .error_strings = snat_out2in_error_strings,
 812
 813   .runtime_data_bytes = sizeof (snat_runtime_t),
 814
 815   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 816
 817   /* edit / add dispositions here */
 818   .next_nodes = {
 819     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 820     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 821   },
 822 };
 823 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 824
 825 static uword
 826 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
 827                                vlib_node_runtime_t * node,
 828                                vlib_frame_t * frame)
 829 {
 830   snat_main_t *sm = &snat_main;
 831   vlib_thread_main_t *tm = vlib_get_thread_main ();
 832   u32 n_left_from, *from, *to_next = 0;
 833   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
 834   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
 835     = 0;
 836   vlib_frame_queue_elt_t *hf = 0;
 837   vlib_frame_t *f = 0;
 838   int i;
 839   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
 840   u32 next_worker_index = 0;
 841   u32 current_worker_index = ~0;
 842   u32 cpu_index = os_get_cpu_number ();
 843
 844   ASSERT (vec_len (sm->workers));
 845
 846   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
 847     {
 848       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
 849
 850       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
 851                                sm->first_worker_index + sm->num_workers - 1,
 852                                (vlib_frame_queue_t *) (~0));
 853     }
 854
 855   from = vlib_frame_vector_args (frame);
 856   n_left_from = frame->n_vectors;
 857
 858   while (n_left_from > 0)
 859     {
 860       u32 bi0;
 861       vlib_buffer_t *b0;
 862       u32 sw_if_index0;
 863       u32 rx_fib_index0;
 864       ip4_header_t * ip0;
 865       udp_header_t * udp0;
 866       snat_static_mapping_key_t key0;
 867       clib_bihash_kv_8_8_t kv0, value0;
 868       u8 do_handoff;
 869
 870       bi0 = from[0];
 871       from += 1;
 872       n_left_from -= 1;
 873
 874       b0 = vlib_get_buffer (vm, bi0);
 875
 876       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 877       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
 878
 879       ip0 = vlib_buffer_get_current (b0);
 880       udp0 = ip4_next_header (ip0);
 881
 882       key0.addr = ip0->dst_address;
 883       key0.port = udp0->dst_port;
 884       key0.fib_index = rx_fib_index0;
 885
 886       if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
 887         {
 888           icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
 889           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 890           key0.port = echo0->identifier;
 891         }
 892
 893       kv0.key = key0.as_u64;
 894
 895       /* Ever heard of of the "user" before? */
 896       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 897         {
 898           key0.port = 0;
 899           kv0.key = key0.as_u64;
 900
 901           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 902             {
 903               /* No, assign next available worker (RR) */
 904               next_worker_index = sm->first_worker_index;
 905               if (vec_len (sm->workers))
 906                 {
 907                   next_worker_index +=
 908                     sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
 909                 }
 910             }
 911           else
 912             {
 913               /* Static mapping without port */
 914               next_worker_index = value0.value;
 915             }
 916
 917           /* Add to translated packets worker lookup */
 918           kv0.value = next_worker_index;
 919           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 920         }
 921       else
 922         next_worker_index = value0.value;
 923
 924       if (PREDICT_FALSE (next_worker_index != cpu_index))
 925         {
 926           do_handoff = 1;
 927
 928           if (next_worker_index != current_worker_index)
 929             {
 930               if (hf)
 931                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 932
 933               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
 934                                                       next_worker_index,
 935                                                       handoff_queue_elt_by_worker_index);
 936
 937               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
 938               to_next_worker = &hf->buffer_index[hf->n_vectors];
 939               current_worker_index = next_worker_index;
 940             }
 941
 942           /* enqueue to correct worker thread */
 943           to_next_worker[0] = bi0;
 944           to_next_worker++;
 945           n_left_to_next_worker--;
 946
 947           if (n_left_to_next_worker == 0)
 948             {
 949               hf->n_vectors = VLIB_FRAME_SIZE;
 950               vlib_put_frame_queue_elt (hf);
 951               current_worker_index = ~0;
 952               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
 953               hf = 0;
 954             }
 955         }
 956       else
 957         {
 958           do_handoff = 0;
 959           /* if this is 1st frame */
 960           if (!f)
 961             {
 962               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
 963               to_next = vlib_frame_vector_args (f);
 964             }
 965
 966           to_next[0] = bi0;
 967           to_next += 1;
 968           f->n_vectors++;
 969         }
 970
 971       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 972                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 973         {
 974           snat_out2in_worker_handoff_trace_t *t =
 975             vlib_add_trace (vm, node, b0, sizeof (*t));
 976           t->next_worker_index = next_worker_index;
 977           t->do_handoff = do_handoff;
 978         }
 979     }
 980
 981   if (f)
 982     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
 983
 984   if (hf)
 985     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 986
 987   /* Ship frames to the worker nodes */
 988   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
 989     {
 990       if (handoff_queue_elt_by_worker_index[i])
 991         {
 992           hf = handoff_queue_elt_by_worker_index[i];
 993           /*
 994            * It works better to let the handoff node
 995            * rate-adapt, always ship the handoff queue element.
 996            */
 997           if (1 || hf->n_vectors == hf->last_n_vectors)
 998             {
 999               vlib_put_frame_queue_elt (hf);
1000               handoff_queue_elt_by_worker_index[i] = 0;
1001             }
1002           else
1003             hf->last_n_vectors = hf->n_vectors;
1004         }
1005       congested_handoff_queue_by_worker_index[i] =
1006         (vlib_frame_queue_t *) (~0);
1007     }
1008   hf = 0;
1009   current_worker_index = ~0;
1010   return frame->n_vectors;
1011 }
1012
1013 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1014   .function = snat_out2in_worker_handoff_fn,
1015   .name = "snat-out2in-worker-handoff",
1016   .vector_size = sizeof (u32),
1017   .format_trace = format_snat_out2in_worker_handoff_trace,
1018   .type = VLIB_NODE_TYPE_INTERNAL,
1019
1020   .n_next_nodes = 1,
1021
1022   .next_nodes = {
1023     [0] = "error-drop",
1024   },
1025 };
1026
1027 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1028
1029 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1030                                     vlib_buffer_t * b0,
1031                                     ip4_header_t * ip0,
1032                                     icmp46_header_t * icmp0,
1033                                     u32 sw_if_index0,
1034                                     vlib_node_runtime_t * node,
1035                                     u32 next0,
1036                                     u32 rx_fib_index0)
1037 {
1038   snat_session_key_t key0, sm0;
1039   icmp_echo_header_t *echo0;
1040   u32 new_addr0, old_addr0;
1041   u16 old_id0, new_id0;
1042   ip_csum_t sum0;
1043   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1044
1045   echo0 = (icmp_echo_header_t *)(icmp0+1);
1046
1047   key0.addr = ip0->dst_address;
1048   key0.port = echo0->identifier;
1049   key0.fib_index = rx_fib_index0;
1050
1051   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1052     {
1053       ip4_address_t * first_int_addr;
1054
1055       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1056         {
1057           first_int_addr =
1058             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1059                                          0 /* just want the address */);
1060           rt->cached_sw_if_index = sw_if_index0;
1061           if (first_int_addr)
1062             rt->cached_ip4_address = first_int_addr->as_u32;
1063           else
1064             rt->cached_ip4_address = 0;
1065         }
1066
1067       /* Don't NAT packet aimed at the intfc address */
1068       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1069                         rt->cached_ip4_address))
1070         return next0;
1071
1072       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1073       return SNAT_OUT2IN_NEXT_DROP;
1074     }
1075
1076   new_addr0 = sm0.addr.as_u32;
1077   new_id0 = sm0.port;
1078   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1079
1080   old_addr0 = ip0->dst_address.as_u32;
1081   ip0->dst_address.as_u32 = new_addr0;
1082
1083   sum0 = ip0->checksum;
1084   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1085                          ip4_header_t,
1086                          dst_address /* changed member */);
1087   ip0->checksum = ip_csum_fold (sum0);
1088
1089   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1090     {
1091       old_id0 = echo0->identifier;
1092       echo0->identifier = new_id0;
1093
1094       sum0 = icmp0->checksum;
1095       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1096                              identifier);
1097       icmp0->checksum = ip_csum_fold (sum0);
1098     }
1099
1100   return next0;
1101 }
1102
1103 static uword
1104 snat_out2in_fast_node_fn (vlib_main_t * vm,
1105                           vlib_node_runtime_t * node,
1106                           vlib_frame_t * frame)
1107 {
1108   u32 n_left_from, * from, * to_next;
1109   snat_out2in_next_t next_index;
1110   u32 pkts_processed = 0;
1111   snat_main_t * sm = &snat_main;
1112
1113   from = vlib_frame_vector_args (frame);
1114   n_left_from = frame->n_vectors;
1115   next_index = node->cached_next_index;
1116
1117   while (n_left_from > 0)
1118     {
1119       u32 n_left_to_next;
1120
1121       vlib_get_next_frame (vm, node, next_index,
1122                            to_next, n_left_to_next);
1123
1124       while (n_left_from > 0 && n_left_to_next > 0)
1125         {
1126           u32 bi0;
1127           vlib_buffer_t * b0;
1128           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1129           u32 sw_if_index0;
1130           ip4_header_t * ip0;
1131           ip_csum_t sum0;
1132           u32 new_addr0, old_addr0;
1133           u16 new_port0, old_port0;
1134           udp_header_t * udp0;
1135           tcp_header_t * tcp0;
1136           icmp46_header_t * icmp0;
1137           snat_session_key_t key0, sm0;
1138           u32 proto0;
1139           u32 rx_fib_index0;
1140
1141           /* speculatively enqueue b0 to the current next frame */
1142           bi0 = from[0];
1143           to_next[0] = bi0;
1144           from += 1;
1145           to_next += 1;
1146           n_left_from -= 1;
1147           n_left_to_next -= 1;
1148
1149           b0 = vlib_get_buffer (vm, bi0);
1150
1151           ip0 = vlib_buffer_get_current (b0);
1152           udp0 = ip4_next_header (ip0);
1153           tcp0 = (tcp_header_t *) udp0;
1154           icmp0 = (icmp46_header_t *) udp0;
1155
1156           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1157           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1158
1159           vnet_feature_next (sw_if_index0, &next0, b0);
1160
1161           proto0 = ~0;
1162           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1163             ? SNAT_PROTOCOL_UDP : proto0;
1164           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1165             ? SNAT_PROTOCOL_TCP : proto0;
1166           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1167             ? SNAT_PROTOCOL_ICMP : proto0;
1168
1169           if (PREDICT_FALSE (proto0 == ~0))
1170               goto trace00;
1171
1172           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1173             {
1174               next0 = icmp_out2in_fast
1175                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1176               goto trace00;
1177             }
1178
1179           key0.addr = ip0->dst_address;
1180           key0.port = udp0->dst_port;
1181           key0.fib_index = rx_fib_index0;
1182
1183           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1184             {
1185               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1186               goto trace00;
1187             }
1188
1189           new_addr0 = sm0.addr.as_u32;
1190           new_port0 = sm0.port;
1191           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1192           old_addr0 = ip0->dst_address.as_u32;
1193           ip0->dst_address.as_u32 = new_addr0;
1194
1195           sum0 = ip0->checksum;
1196           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1197                                  ip4_header_t,
1198                                  dst_address /* changed member */);
1199           ip0->checksum = ip_csum_fold (sum0);
1200
1201           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1202             {
1203                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1204                 {
1205                   old_port0 = tcp0->ports.dst;
1206                   tcp0->ports.dst = new_port0;
1207
1208                   sum0 = tcp0->checksum;
1209                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1210                                          ip4_header_t,
1211                                          dst_address /* changed member */);
1212
1213                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1214                                          ip4_header_t /* cheat */,
1215                                          length /* changed member */);
1216                   tcp0->checksum = ip_csum_fold(sum0);
1217                 }
1218               else
1219                 {
1220                   old_port0 = udp0->dst_port;
1221                   udp0->dst_port = new_port0;
1222                   udp0->checksum = 0;
1223                 }
1224             }
1225           else
1226             {
1227               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1228                 {
1229                   sum0 = tcp0->checksum;
1230                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1231                                          ip4_header_t,
1232                                          dst_address /* changed member */);
1233
1234                   tcp0->checksum = ip_csum_fold(sum0);
1235                 }
1236             }
1237
1238         trace00:
1239
1240           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1241                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1242             {
1243               snat_out2in_trace_t *t =
1244                  vlib_add_trace (vm, node, b0, sizeof (*t));
1245               t->sw_if_index = sw_if_index0;
1246               t->next_index = next0;
1247             }
1248
1249           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1250
1251           /* verify speculative enqueue, maybe switch current next frame */
1252           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1253                                            to_next, n_left_to_next,
1254                                            bi0, next0);
1255         }
1256
1257       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1258     }
1259
1260   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1261                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1262                                pkts_processed);
1263   return frame->n_vectors;
1264 }
1265
1266 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1267   .function = snat_out2in_fast_node_fn,
1268   .name = "snat-out2in-fast",
1269   .vector_size = sizeof (u32),
1270   .format_trace = format_snat_out2in_fast_trace,
1271   .type = VLIB_NODE_TYPE_INTERNAL,
1272
1273   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1274   .error_strings = snat_out2in_error_strings,
1275
1276   .runtime_data_bytes = sizeof (snat_runtime_t),
1277
1278   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1279
1280   /* edit / add dispositions here */
1281   .next_nodes = {
1282     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1283     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1284   },
1285 };
1286 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);