src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27 #include <snat/snat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     SNAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out SNAT session key.
 121  * @param out2in Out2in SNAT session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 cpu_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140
 141   user_key.addr = in2out.addr;
 142   user_key.fib_index = in2out.fib_index;
 143   kv0.key = user_key.as_u64;
 144
 145   /* Ever heard of the "user" = inside ip4 address before? */
 146   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 147     {
 148       /* no, make a new one */
 149       pool_get (sm->per_thread_data[cpu_index].users, u);
 150       memset (u, 0, sizeof (*u));
 151       u->addr = in2out.addr;
 152       u->fib_index = in2out.fib_index;
 153
 154       pool_get (sm->per_thread_data[cpu_index].list_pool,
 155                 per_user_list_head_elt);
 156
 157       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 158         sm->per_thread_data[cpu_index].list_pool;
 159
 160       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 161                        u->sessions_per_user_list_head_index);
 162
 163       kv0.value = u - sm->per_thread_data[cpu_index].users;
 164
 165       /* add user */
 166       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 167
 168       /* add non-traslated packets worker lookup */
 169       kv0.value = cpu_index;
 170       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 171     }
 172   else
 173     {
 174       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 175                              value0.value);
 176     }
 177
 178   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 179   memset (s, 0, sizeof (*s));
 180
 181   s->outside_address_index = ~0;
 182   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 183   u->nstaticsessions++;
 184
 185   /* Create list elts */
 186   pool_get (sm->per_thread_data[cpu_index].list_pool,
 187             per_user_translation_list_elt);
 188   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 189                    per_user_translation_list_elt -
 190                    sm->per_thread_data[cpu_index].list_pool);
 191
 192   per_user_translation_list_elt->value =
 193     s - sm->per_thread_data[cpu_index].sessions;
 194   s->per_user_index =
 195     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 196   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 197
 198   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 199                       s->per_user_list_head_index,
 200                       per_user_translation_list_elt -
 201                       sm->per_thread_data[cpu_index].list_pool);
 202
 203   s->in2out = in2out;
 204   s->out2in = out2in;
 205   s->in2out.protocol = out2in.protocol;
 206
 207   /* Add to translation hashes */
 208   kv0.key = s->in2out.as_u64;
 209   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 210   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 211       clib_warning ("in2out key add failed");
 212
 213   kv0.key = s->out2in.as_u64;
 214   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 215
 216   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 217       clib_warning ("out2in key add failed");
 218
 219   /* log NAT event */
 220   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 221                                       s->out2in.addr.as_u32,
 222                                       s->in2out.protocol,
 223                                       s->in2out.port,
 224                                       s->out2in.port,
 225                                       s->in2out.fib_index);
 226    return s;
 227 }
 228
 229 typedef struct {
 230   u16 src_port, dst_port;
 231 } tcp_udp_header_t;
 232
 233 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 234                                          vlib_buffer_t * b0,
 235                                          ip4_header_t * ip0,
 236                                          icmp46_header_t * icmp0,
 237                                          u32 sw_if_index0,
 238                                          u32 rx_fib_index0,
 239                                          vlib_node_runtime_t * node,
 240                                          u32 next0, f64 now,
 241                                          u32 cpu_index,
 242                                          snat_session_t ** p_s0)
 243 {
 244   snat_session_key_t key0, sm0;
 245   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 246   ip4_header_t *inner_ip0 = 0;
 247   void *l4_header = 0;
 248   icmp46_header_t *inner_icmp0;
 249   clib_bihash_kv_8_8_t kv0, value0;
 250   snat_session_t * s0 = 0;
 251   u32 new_addr0, old_addr0;
 252   u16 old_id0, new_id0;
 253   ip_csum_t sum0;
 254   u16 checksum0;
 255   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 256   u8 is_error_message = 0;
 257
 258   echo0 = (icmp_echo_header_t *)(icmp0+1);
 259
 260   key0.addr = ip0->dst_address;
 261   key0.fib_index = rx_fib_index0;
 262
 263   switch(icmp0->type)
 264     {
 265     case ICMP4_destination_unreachable:
 266     case ICMP4_time_exceeded:
 267     case ICMP4_parameter_problem:
 268     case ICMP4_source_quench:
 269     case ICMP4_redirect:
 270     case ICMP4_alternate_host_address:
 271       is_error_message = 1;
 272     }
 273
 274   if (!is_error_message)
 275     {
 276       key0.protocol = SNAT_PROTOCOL_ICMP;
 277       key0.port = echo0->identifier;
 278     }
 279   else
 280     {
 281       inner_ip0 = (ip4_header_t *)(echo0+1);
 282       l4_header = ip4_next_header (inner_ip0);
 283       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 284       switch (key0.protocol)
 285         {
 286         case SNAT_PROTOCOL_ICMP:
 287           inner_icmp0 = (icmp46_header_t*)l4_header;
 288           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 289           key0.port = inner_echo0->identifier;
 290           break;
 291         case SNAT_PROTOCOL_UDP:
 292         case SNAT_PROTOCOL_TCP:
 293           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 294           break;
 295         default:
 296           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 297           next0 = SNAT_OUT2IN_NEXT_DROP;
 298           goto out;
 299         }
 300     }
 301
 302   kv0.key = key0.as_u64;
 303
 304   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 305     {
 306       /* Try to match static mapping by external address and port,
 307          destination address and port in packet */
 308       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 309         {
 310            ip4_address_t * first_int_addr;
 311
 312           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 313             {
 314               first_int_addr =
 315                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 316                                              0 /* just want the address */);
 317               rt->cached_sw_if_index = sw_if_index0;
 318               if (first_int_addr)
 319                 rt->cached_ip4_address = first_int_addr->as_u32;
 320               else
 321                 rt->cached_ip4_address = 0;
 322             }
 323
 324           /* Don't NAT packet aimed at the intfc address */
 325           if (PREDICT_FALSE(ip0->dst_address.as_u32 == rt->cached_ip4_address))
 326             goto out;
 327
 328           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 329           next0 = SNAT_OUT2IN_NEXT_DROP;
 330           goto out;
 331         }
 332
 333       if (is_error_message)
 334         {
 335           next0 = SNAT_OUT2IN_NEXT_DROP;
 336           goto out;
 337         }
 338
 339       /* Create session initiated by host from external network */
 340       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 341                                              node, cpu_index);
 342
 343       if (!s0)
 344         {
 345           next0 = SNAT_OUT2IN_NEXT_DROP;
 346           goto out;
 347         }
 348     }
 349   else
 350     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 351                             value0.value);
 352
 353   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply && !is_error_message))
 354     {
 355       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 356       next0 = SNAT_OUT2IN_NEXT_DROP;
 357       goto out;
 358     }
 359
 360   sum0 = ip_incremental_checksum (0, icmp0,
 361                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 362   checksum0 = ~ip_csum_fold (sum0);
 363   if (checksum0 != 0 && checksum0 != 0xffff)
 364     {
 365       next0 = SNAT_OUT2IN_NEXT_DROP;
 366       goto out;
 367     }
 368
 369   old_addr0 = ip0->dst_address.as_u32;
 370   ip0->dst_address = s0->in2out.addr;
 371   new_addr0 = ip0->dst_address.as_u32;
 372   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 373
 374   sum0 = ip0->checksum;
 375   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 376                          dst_address /* changed member */);
 377   ip0->checksum = ip_csum_fold (sum0);
 378
 379   if (!is_error_message)
 380     {
 381       old_id0 = echo0->identifier;
 382       new_id0 = s0->in2out.port;
 383       echo0->identifier = new_id0;
 384
 385       sum0 = icmp0->checksum;
 386       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 387                              identifier /* changed member */);
 388       icmp0->checksum = ip_csum_fold (sum0);
 389     }
 390   else
 391     {
 392       if (!ip4_header_checksum_is_valid (inner_ip0))
 393         {
 394           next0 = SNAT_OUT2IN_NEXT_DROP;
 395           goto out;
 396         }
 397
 398       old_addr0 = inner_ip0->src_address.as_u32;
 399       inner_ip0->src_address = s0->in2out.addr;
 400       new_addr0 = inner_ip0->src_address.as_u32;
 401
 402       sum0 = icmp0->checksum;
 403       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 404                              src_address /* changed member */);
 405       icmp0->checksum = ip_csum_fold (sum0);
 406
 407       switch (key0.protocol)
 408         {
 409         case SNAT_PROTOCOL_ICMP:
 410           old_id0 = inner_echo0->identifier;
 411           new_id0 = s0->in2out.port;
 412           inner_echo0->identifier = new_id0;
 413
 414           sum0 = icmp0->checksum;
 415           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 416                                  identifier);
 417           icmp0->checksum = ip_csum_fold (sum0);
 418           break;
 419         case SNAT_PROTOCOL_UDP:
 420         case SNAT_PROTOCOL_TCP:
 421           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 422           new_id0 = s0->in2out.port;
 423           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 424
 425           sum0 = icmp0->checksum;
 426           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 427                                  src_port);
 428           icmp0->checksum = ip_csum_fold (sum0);
 429           break;
 430         default:
 431           ASSERT(0);
 432         }
 433     }
 434
 435   /* Accounting */
 436   s0->last_heard = now;
 437   s0->total_pkts++;
 438   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 439   /* Per-user LRU list maintenance for dynamic translation */
 440   if (!snat_is_session_static (s0))
 441     {
 442       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 443                          s0->per_user_index);
 444       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 445                           s0->per_user_list_head_index,
 446                           s0->per_user_index);
 447     }
 448
 449 out:
 450   *p_s0 = s0;
 451   return next0;
 452 }
 453
 454 static uword
 455 snat_out2in_node_fn (vlib_main_t * vm,
 456                   vlib_node_runtime_t * node,
 457                   vlib_frame_t * frame)
 458 {
 459   u32 n_left_from, * from, * to_next;
 460   snat_out2in_next_t next_index;
 461   u32 pkts_processed = 0;
 462   snat_main_t * sm = &snat_main;
 463   f64 now = vlib_time_now (vm);
 464   u32 cpu_index = os_get_cpu_number ();
 465
 466   from = vlib_frame_vector_args (frame);
 467   n_left_from = frame->n_vectors;
 468   next_index = node->cached_next_index;
 469
 470   while (n_left_from > 0)
 471     {
 472       u32 n_left_to_next;
 473
 474       vlib_get_next_frame (vm, node, next_index,
 475                            to_next, n_left_to_next);
 476
 477       while (n_left_from >= 4 && n_left_to_next >= 2)
 478         {
 479           u32 bi0, bi1;
 480           vlib_buffer_t * b0, * b1;
 481           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 482           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 483           u32 sw_if_index0, sw_if_index1;
 484           ip4_header_t * ip0, *ip1;
 485           ip_csum_t sum0, sum1;
 486           u32 new_addr0, old_addr0;
 487           u16 new_port0, old_port0;
 488           u32 new_addr1, old_addr1;
 489           u16 new_port1, old_port1;
 490           udp_header_t * udp0, * udp1;
 491           tcp_header_t * tcp0, * tcp1;
 492           icmp46_header_t * icmp0, * icmp1;
 493           snat_session_key_t key0, key1, sm0, sm1;
 494           u32 rx_fib_index0, rx_fib_index1;
 495           u32 proto0, proto1;
 496           snat_session_t * s0 = 0, * s1 = 0;
 497           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 498
 499           /* Prefetch next iteration. */
 500           {
 501             vlib_buffer_t * p2, * p3;
 502
 503             p2 = vlib_get_buffer (vm, from[2]);
 504             p3 = vlib_get_buffer (vm, from[3]);
 505
 506             vlib_prefetch_buffer_header (p2, LOAD);
 507             vlib_prefetch_buffer_header (p3, LOAD);
 508
 509             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 510             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 511           }
 512
 513           /* speculatively enqueue b0 and b1 to the current next frame */
 514           to_next[0] = bi0 = from[0];
 515           to_next[1] = bi1 = from[1];
 516           from += 2;
 517           to_next += 2;
 518           n_left_from -= 2;
 519           n_left_to_next -= 2;
 520
 521           b0 = vlib_get_buffer (vm, bi0);
 522           b1 = vlib_get_buffer (vm, bi1);
 523
 524           ip0 = vlib_buffer_get_current (b0);
 525           udp0 = ip4_next_header (ip0);
 526           tcp0 = (tcp_header_t *) udp0;
 527           icmp0 = (icmp46_header_t *) udp0;
 528
 529           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 530           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 531                                    sw_if_index0);
 532
 533           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 534
 535           if (PREDICT_FALSE (proto0 == ~0))
 536               goto trace0;
 537
 538           if (PREDICT_FALSE(ip0->ttl == 1))
 539             {
 540               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 541               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 542                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 543                                            0);
 544               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 545               goto trace0;
 546             }
 547
 548           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 549             {
 550               next0 = icmp_out2in_slow_path
 551                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 552                  next0, now, cpu_index, &s0);
 553               goto trace0;
 554             }
 555
 556           key0.addr = ip0->dst_address;
 557           key0.port = udp0->dst_port;
 558           key0.protocol = proto0;
 559           key0.fib_index = rx_fib_index0;
 560
 561           kv0.key = key0.as_u64;
 562
 563           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 564             {
 565               /* Try to match static mapping by external address and port,
 566                  destination address and port in packet */
 567               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 568                 {
 569                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 570                   /*
 571                    * Send DHCP packets to the ipv4 stack, or we won't
 572                    * be able to use dhcp client on the outside interface
 573                    */
 574                   if (proto0 != SNAT_PROTOCOL_UDP
 575                       || (udp0->dst_port
 576                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 577                     next0 = SNAT_OUT2IN_NEXT_DROP;
 578                   goto trace0;
 579                 }
 580
 581               /* Create session initiated by host from external network */
 582               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 583                                                      cpu_index);
 584               if (!s0)
 585                 {
 586                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 587                   next0 = SNAT_OUT2IN_NEXT_DROP;
 588                   goto trace0;
 589                 }
 590             }
 591           else
 592             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 593                                     value0.value);
 594
 595           old_addr0 = ip0->dst_address.as_u32;
 596           ip0->dst_address = s0->in2out.addr;
 597           new_addr0 = ip0->dst_address.as_u32;
 598           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 599
 600           sum0 = ip0->checksum;
 601           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 602                                  ip4_header_t,
 603                                  dst_address /* changed member */);
 604           ip0->checksum = ip_csum_fold (sum0);
 605
 606           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 607             {
 608               old_port0 = tcp0->dst_port;
 609               tcp0->dst_port = s0->in2out.port;
 610               new_port0 = tcp0->dst_port;
 611
 612               sum0 = tcp0->checksum;
 613               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 614                                      ip4_header_t,
 615                                      dst_address /* changed member */);
 616
 617               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 618                                      ip4_header_t /* cheat */,
 619                                      length /* changed member */);
 620               tcp0->checksum = ip_csum_fold(sum0);
 621             }
 622           else
 623             {
 624               old_port0 = udp0->dst_port;
 625               udp0->dst_port = s0->in2out.port;
 626               udp0->checksum = 0;
 627             }
 628
 629           /* Accounting */
 630           s0->last_heard = now;
 631           s0->total_pkts++;
 632           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 633           /* Per-user LRU list maintenance for dynamic translation */
 634           if (!snat_is_session_static (s0))
 635             {
 636               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 637                                  s0->per_user_index);
 638               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 639                                   s0->per_user_list_head_index,
 640                                   s0->per_user_index);
 641             }
 642         trace0:
 643
 644           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 645                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 646             {
 647               snat_out2in_trace_t *t =
 648                  vlib_add_trace (vm, node, b0, sizeof (*t));
 649               t->sw_if_index = sw_if_index0;
 650               t->next_index = next0;
 651               t->session_index = ~0;
 652               if (s0)
 653                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 654             }
 655
 656           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 657
 658
 659           ip1 = vlib_buffer_get_current (b1);
 660           udp1 = ip4_next_header (ip1);
 661           tcp1 = (tcp_header_t *) udp1;
 662           icmp1 = (icmp46_header_t *) udp1;
 663
 664           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 665           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 666                                    sw_if_index1);
 667
 668           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 669
 670           if (PREDICT_FALSE (proto1 == ~0))
 671               goto trace1;
 672
 673           if (PREDICT_FALSE(ip0->ttl == 1))
 674             {
 675               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 676               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 677                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 678                                            0);
 679               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 680               goto trace1;
 681             }
 682
 683           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 684             {
 685               next1 = icmp_out2in_slow_path
 686                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 687                  next1, now, cpu_index, &s1);
 688               goto trace1;
 689             }
 690
 691           key1.addr = ip1->dst_address;
 692           key1.port = udp1->dst_port;
 693           key1.protocol = proto1;
 694           key1.fib_index = rx_fib_index1;
 695
 696           kv1.key = key1.as_u64;
 697
 698           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 699             {
 700               /* Try to match static mapping by external address and port,
 701                  destination address and port in packet */
 702               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 703                 {
 704                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 705                   /*
 706                    * Send DHCP packets to the ipv4 stack, or we won't
 707                    * be able to use dhcp client on the outside interface
 708                    */
 709                   if (proto1 != SNAT_PROTOCOL_UDP
 710                       || (udp1->dst_port
 711                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 712                     next1 = SNAT_OUT2IN_NEXT_DROP;
 713                   goto trace1;
 714                 }
 715
 716               /* Create session initiated by host from external network */
 717               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 718                                                      cpu_index);
 719               if (!s1)
 720                 {
 721                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 722                   next1 = SNAT_OUT2IN_NEXT_DROP;
 723                   goto trace1;
 724                 }
 725             }
 726           else
 727             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 728                                     value1.value);
 729
 730           old_addr1 = ip1->dst_address.as_u32;
 731           ip1->dst_address = s1->in2out.addr;
 732           new_addr1 = ip1->dst_address.as_u32;
 733           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 734
 735           sum1 = ip1->checksum;
 736           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 737                                  ip4_header_t,
 738                                  dst_address /* changed member */);
 739           ip1->checksum = ip_csum_fold (sum1);
 740
 741           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 742             {
 743               old_port1 = tcp1->dst_port;
 744               tcp1->dst_port = s1->in2out.port;
 745               new_port1 = tcp1->dst_port;
 746
 747               sum1 = tcp1->checksum;
 748               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 749                                      ip4_header_t,
 750                                      dst_address /* changed member */);
 751
 752               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 753                                      ip4_header_t /* cheat */,
 754                                      length /* changed member */);
 755               tcp1->checksum = ip_csum_fold(sum1);
 756             }
 757           else
 758             {
 759               old_port1 = udp1->dst_port;
 760               udp1->dst_port = s1->in2out.port;
 761               udp1->checksum = 0;
 762             }
 763
 764           /* Accounting */
 765           s1->last_heard = now;
 766           s1->total_pkts++;
 767           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 768           /* Per-user LRU list maintenance for dynamic translation */
 769           if (!snat_is_session_static (s1))
 770             {
 771               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 772                                  s1->per_user_index);
 773               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 774                                   s1->per_user_list_head_index,
 775                                   s1->per_user_index);
 776             }
 777         trace1:
 778
 779           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 780                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 781             {
 782               snat_out2in_trace_t *t =
 783                  vlib_add_trace (vm, node, b1, sizeof (*t));
 784               t->sw_if_index = sw_if_index1;
 785               t->next_index = next1;
 786               t->session_index = ~0;
 787               if (s1)
 788                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 789             }
 790
 791           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 792
 793           /* verify speculative enqueues, maybe switch current next frame */
 794           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 795                                            to_next, n_left_to_next,
 796                                            bi0, bi1, next0, next1);
 797         }
 798
 799       while (n_left_from > 0 && n_left_to_next > 0)
 800         {
 801           u32 bi0;
 802           vlib_buffer_t * b0;
 803           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 804           u32 sw_if_index0;
 805           ip4_header_t * ip0;
 806           ip_csum_t sum0;
 807           u32 new_addr0, old_addr0;
 808           u16 new_port0, old_port0;
 809           udp_header_t * udp0;
 810           tcp_header_t * tcp0;
 811           icmp46_header_t * icmp0;
 812           snat_session_key_t key0, sm0;
 813           u32 rx_fib_index0;
 814           u32 proto0;
 815           snat_session_t * s0 = 0;
 816           clib_bihash_kv_8_8_t kv0, value0;
 817
 818           /* speculatively enqueue b0 to the current next frame */
 819           bi0 = from[0];
 820           to_next[0] = bi0;
 821           from += 1;
 822           to_next += 1;
 823           n_left_from -= 1;
 824           n_left_to_next -= 1;
 825
 826           b0 = vlib_get_buffer (vm, bi0);
 827
 828           ip0 = vlib_buffer_get_current (b0);
 829           udp0 = ip4_next_header (ip0);
 830           tcp0 = (tcp_header_t *) udp0;
 831           icmp0 = (icmp46_header_t *) udp0;
 832
 833           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 834           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 835                                    sw_if_index0);
 836
 837           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 838
 839           if (PREDICT_FALSE (proto0 == ~0))
 840               goto trace00;
 841
 842           if (PREDICT_FALSE(ip0->ttl == 1))
 843             {
 844               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 845               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 846                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 847                                            0);
 848               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 849               goto trace00;
 850             }
 851
 852           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 853             {
 854               next0 = icmp_out2in_slow_path
 855                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 856                  next0, now, cpu_index, &s0);
 857               goto trace00;
 858             }
 859
 860           key0.addr = ip0->dst_address;
 861           key0.port = udp0->dst_port;
 862           key0.protocol = proto0;
 863           key0.fib_index = rx_fib_index0;
 864
 865           kv0.key = key0.as_u64;
 866
 867           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 868             {
 869               /* Try to match static mapping by external address and port,
 870                  destination address and port in packet */
 871               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 872                 {
 873                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 874                   /*
 875                    * Send DHCP packets to the ipv4 stack, or we won't
 876                    * be able to use dhcp client on the outside interface
 877                    */
 878                   if (proto0 != SNAT_PROTOCOL_UDP
 879                       || (udp0->dst_port
 880                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 881
 882                     next0 = SNAT_OUT2IN_NEXT_DROP;
 883                   goto trace00;
 884                 }
 885
 886               /* Create session initiated by host from external network */
 887               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 888                                                      cpu_index);
 889               if (!s0)
 890                 {
 891                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 892                     next0 = SNAT_OUT2IN_NEXT_DROP;
 893                   goto trace00;
 894                 }
 895             }
 896           else
 897             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 898                                     value0.value);
 899
 900           old_addr0 = ip0->dst_address.as_u32;
 901           ip0->dst_address = s0->in2out.addr;
 902           new_addr0 = ip0->dst_address.as_u32;
 903           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 904
 905           sum0 = ip0->checksum;
 906           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 907                                  ip4_header_t,
 908                                  dst_address /* changed member */);
 909           ip0->checksum = ip_csum_fold (sum0);
 910
 911           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 912             {
 913               old_port0 = tcp0->dst_port;
 914               tcp0->dst_port = s0->in2out.port;
 915               new_port0 = tcp0->dst_port;
 916
 917               sum0 = tcp0->checksum;
 918               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 919                                      ip4_header_t,
 920                                      dst_address /* changed member */);
 921
 922               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 923                                      ip4_header_t /* cheat */,
 924                                      length /* changed member */);
 925               tcp0->checksum = ip_csum_fold(sum0);
 926             }
 927           else
 928             {
 929               old_port0 = udp0->dst_port;
 930               udp0->dst_port = s0->in2out.port;
 931               udp0->checksum = 0;
 932             }
 933
 934           /* Accounting */
 935           s0->last_heard = now;
 936           s0->total_pkts++;
 937           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 938           /* Per-user LRU list maintenance for dynamic translation */
 939           if (!snat_is_session_static (s0))
 940             {
 941               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 942                                  s0->per_user_index);
 943               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 944                                   s0->per_user_list_head_index,
 945                                   s0->per_user_index);
 946             }
 947         trace00:
 948
 949           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 950                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 951             {
 952               snat_out2in_trace_t *t =
 953                  vlib_add_trace (vm, node, b0, sizeof (*t));
 954               t->sw_if_index = sw_if_index0;
 955               t->next_index = next0;
 956               t->session_index = ~0;
 957               if (s0)
 958                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 959             }
 960
 961           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 962
 963           /* verify speculative enqueue, maybe switch current next frame */
 964           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 965                                            to_next, n_left_to_next,
 966                                            bi0, next0);
 967         }
 968
 969       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 970     }
 971
 972   vlib_node_increment_counter (vm, snat_out2in_node.index,
 973                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 974                                pkts_processed);
 975   return frame->n_vectors;
 976 }
 977
 978 VLIB_REGISTER_NODE (snat_out2in_node) = {
 979   .function = snat_out2in_node_fn,
 980   .name = "snat-out2in",
 981   .vector_size = sizeof (u32),
 982   .format_trace = format_snat_out2in_trace,
 983   .type = VLIB_NODE_TYPE_INTERNAL,
 984
 985   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 986   .error_strings = snat_out2in_error_strings,
 987
 988   .runtime_data_bytes = sizeof (snat_runtime_t),
 989
 990   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 991
 992   /* edit / add dispositions here */
 993   .next_nodes = {
 994     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 995     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 996     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
 997   },
 998 };
 999 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1000
1001 /**************************/
1002 /*** deterministic mode ***/
1003 /**************************/
1004 static uword
1005 snat_det_out2in_node_fn (vlib_main_t * vm,
1006                          vlib_node_runtime_t * node,
1007                          vlib_frame_t * frame)
1008 {
1009   u32 n_left_from, * from, * to_next;
1010   snat_out2in_next_t next_index;
1011   u32 pkts_processed = 0;
1012   snat_main_t * sm = &snat_main;
1013
1014   from = vlib_frame_vector_args (frame);
1015   n_left_from = frame->n_vectors;
1016   next_index = node->cached_next_index;
1017
1018   while (n_left_from > 0)
1019     {
1020       u32 n_left_to_next;
1021
1022       vlib_get_next_frame (vm, node, next_index,
1023                            to_next, n_left_to_next);
1024
1025       while (n_left_from >= 4 && n_left_to_next >= 2)
1026         {
1027           u32 bi0, bi1;
1028           vlib_buffer_t * b0, * b1;
1029           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1030           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1031           u32 sw_if_index0, sw_if_index1;
1032           ip4_header_t * ip0, * ip1;
1033           ip_csum_t sum0, sum1;
1034           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1035           u16 new_port0, old_port0, old_port1, new_port1;
1036           udp_header_t * udp0, * udp1;
1037           tcp_header_t * tcp0, * tcp1;
1038           u32 proto0, proto1;
1039           snat_det_out_key_t key0, key1;
1040           snat_det_map_t * dm0, * dm1;
1041           snat_det_session_t * ses0 = 0, * ses1 = 0;
1042
1043           /* Prefetch next iteration. */
1044           {
1045             vlib_buffer_t * p2, * p3;
1046
1047             p2 = vlib_get_buffer (vm, from[2]);
1048             p3 = vlib_get_buffer (vm, from[3]);
1049
1050             vlib_prefetch_buffer_header (p2, LOAD);
1051             vlib_prefetch_buffer_header (p3, LOAD);
1052
1053             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1054             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1055           }
1056
1057           /* speculatively enqueue b0 and b1 to the current next frame */
1058           to_next[0] = bi0 = from[0];
1059           to_next[1] = bi1 = from[1];
1060           from += 2;
1061           to_next += 2;
1062           n_left_from -= 2;
1063           n_left_to_next -= 2;
1064
1065           b0 = vlib_get_buffer (vm, bi0);
1066           b1 = vlib_get_buffer (vm, bi1);
1067
1068           ip0 = vlib_buffer_get_current (b0);
1069           udp0 = ip4_next_header (ip0);
1070           tcp0 = (tcp_header_t *) udp0;
1071
1072           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1073
1074           key0.ext_host_addr = ip0->src_address;
1075           key0.ext_host_port = tcp0->src;
1076           key0.out_port = tcp0->dst;
1077
1078           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1079           if (PREDICT_FALSE(!dm0))
1080             {
1081               clib_warning("unknown dst address:  %U",
1082                            format_ip4_address, &ip0->dst_address);
1083               next0 = SNAT_OUT2IN_NEXT_DROP;
1084               goto trace0;
1085             }
1086
1087           snat_det_reverse(dm0, &ip0->dst_address,
1088                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1089
1090           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1091           if (PREDICT_FALSE(!ses0))
1092             {
1093               clib_warning("no match src %U:%d dst %d for user %U",
1094                            format_ip4_address, &ip0->dst_address,
1095                            clib_net_to_host_u16 (tcp0->src),
1096                            clib_net_to_host_u16 (tcp0->dst),
1097                            format_ip4_address, &new_addr0);
1098               next0 = SNAT_OUT2IN_NEXT_DROP;
1099               goto trace0;
1100             }
1101           new_port0 = ses0->in_port;
1102
1103           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1104
1105           old_addr0 = ip0->dst_address;
1106           ip0->dst_address = new_addr0;
1107           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1108
1109           sum0 = ip0->checksum;
1110           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1111                                  ip4_header_t,
1112                                  dst_address /* changed member */);
1113           ip0->checksum = ip_csum_fold (sum0);
1114
1115           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1116             {
1117               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1118                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1119               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1120                 snat_det_ses_close(dm0, ses0);
1121
1122               old_port0 = tcp0->dst;
1123               tcp0->dst = new_port0;
1124
1125               sum0 = tcp0->checksum;
1126               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1127                                      ip4_header_t,
1128                                      dst_address /* changed member */);
1129
1130               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1131                                      ip4_header_t /* cheat */,
1132                                      length /* changed member */);
1133               tcp0->checksum = ip_csum_fold(sum0);
1134             }
1135           else
1136             {
1137               old_port0 = udp0->dst_port;
1138               udp0->dst_port = new_port0;
1139               udp0->checksum = 0;
1140             }
1141
1142         trace0:
1143
1144           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1145                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1146             {
1147               snat_out2in_trace_t *t =
1148                  vlib_add_trace (vm, node, b0, sizeof (*t));
1149               t->sw_if_index = sw_if_index0;
1150               t->next_index = next0;
1151               t->session_index = ~0;
1152               if (ses0)
1153                 t->session_index = ses0 - dm0->sessions;
1154             }
1155
1156           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1157
1158           b1 = vlib_get_buffer (vm, bi1);
1159
1160           ip1 = vlib_buffer_get_current (b1);
1161           udp1 = ip4_next_header (ip1);
1162           tcp1 = (tcp_header_t *) udp1;
1163
1164           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1165
1166           key1.ext_host_addr = ip1->src_address;
1167           key1.ext_host_port = tcp1->src;
1168           key1.out_port = tcp1->dst;
1169
1170           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1171           if (PREDICT_FALSE(!dm1))
1172             {
1173               clib_warning("unknown dst address:  %U",
1174                            format_ip4_address, &ip1->dst_address);
1175               next1 = SNAT_OUT2IN_NEXT_DROP;
1176               goto trace1;
1177             }
1178
1179           snat_det_reverse(dm1, &ip1->dst_address,
1180                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1181
1182           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1183           if (PREDICT_FALSE(!ses1))
1184             {
1185               clib_warning("no match src %U:%d dst %d for user %U",
1186                            format_ip4_address, &ip1->dst_address,
1187                            clib_net_to_host_u16 (tcp1->src),
1188                            clib_net_to_host_u16 (tcp1->dst),
1189                            format_ip4_address, &new_addr1);
1190               next1 = SNAT_OUT2IN_NEXT_DROP;
1191               goto trace1;
1192             }
1193           new_port1 = ses1->in_port;
1194
1195           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1196
1197           old_addr1 = ip1->dst_address;
1198           ip1->dst_address = new_addr1;
1199           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1200
1201           sum1 = ip1->checksum;
1202           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1203                                  ip4_header_t,
1204                                  dst_address /* changed member */);
1205           ip1->checksum = ip_csum_fold (sum1);
1206
1207           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1208             {
1209               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1210                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1211               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1212                 snat_det_ses_close(dm1, ses1);
1213
1214               old_port1 = tcp1->dst;
1215               tcp1->dst = new_port1;
1216
1217               sum1 = tcp1->checksum;
1218               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1219                                      ip4_header_t,
1220                                      dst_address /* changed member */);
1221
1222               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1223                                      ip4_header_t /* cheat */,
1224                                      length /* changed member */);
1225               tcp1->checksum = ip_csum_fold(sum1);
1226             }
1227           else
1228             {
1229               old_port1 = udp1->dst_port;
1230               udp1->dst_port = new_port1;
1231               udp1->checksum = 0;
1232             }
1233
1234         trace1:
1235
1236           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1237                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1238             {
1239               snat_out2in_trace_t *t =
1240                  vlib_add_trace (vm, node, b1, sizeof (*t));
1241               t->sw_if_index = sw_if_index1;
1242               t->next_index = next1;
1243               t->session_index = ~0;
1244               if (ses1)
1245                 t->session_index = ses1 - dm1->sessions;
1246             }
1247
1248           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1249
1250           /* verify speculative enqueues, maybe switch current next frame */
1251           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1252                                            to_next, n_left_to_next,
1253                                            bi0, bi1, next0, next1);
1254          }
1255
1256       while (n_left_from > 0 && n_left_to_next > 0)
1257         {
1258           u32 bi0;
1259           vlib_buffer_t * b0;
1260           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1261           u32 sw_if_index0;
1262           ip4_header_t * ip0;
1263           ip_csum_t sum0;
1264           ip4_address_t new_addr0, old_addr0;
1265           u16 new_port0, old_port0;
1266           udp_header_t * udp0;
1267           tcp_header_t * tcp0;
1268           u32 proto0;
1269           snat_det_out_key_t key0;
1270           snat_det_map_t * dm0;
1271           snat_det_session_t * ses0 = 0;
1272
1273           /* speculatively enqueue b0 to the current next frame */
1274           bi0 = from[0];
1275           to_next[0] = bi0;
1276           from += 1;
1277           to_next += 1;
1278           n_left_from -= 1;
1279           n_left_to_next -= 1;
1280
1281           b0 = vlib_get_buffer (vm, bi0);
1282
1283           ip0 = vlib_buffer_get_current (b0);
1284           udp0 = ip4_next_header (ip0);
1285           tcp0 = (tcp_header_t *) udp0;
1286
1287           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1288
1289           key0.ext_host_addr = ip0->src_address;
1290           key0.ext_host_port = tcp0->src;
1291           key0.out_port = tcp0->dst;
1292
1293           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1294           if (PREDICT_FALSE(!dm0))
1295             {
1296               clib_warning("unknown dst address:  %U",
1297                            format_ip4_address, &ip0->dst_address);
1298               next0 = SNAT_OUT2IN_NEXT_DROP;
1299               goto trace00;
1300             }
1301
1302           snat_det_reverse(dm0, &ip0->dst_address,
1303                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1304
1305           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1306           if (PREDICT_FALSE(!ses0))
1307             {
1308               clib_warning("no match src %U:%d dst %d for user %U",
1309                            format_ip4_address, &ip0->dst_address,
1310                            clib_net_to_host_u16 (tcp0->src),
1311                            clib_net_to_host_u16 (tcp0->dst),
1312                            format_ip4_address, &new_addr0);
1313               next0 = SNAT_OUT2IN_NEXT_DROP;
1314               goto trace00;
1315             }
1316           new_port0 = ses0->in_port;
1317
1318           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1319
1320           old_addr0 = ip0->dst_address;
1321           ip0->dst_address = new_addr0;
1322           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1323
1324           sum0 = ip0->checksum;
1325           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1326                                  ip4_header_t,
1327                                  dst_address /* changed member */);
1328           ip0->checksum = ip_csum_fold (sum0);
1329
1330           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1331             {
1332               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1333                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1334               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1335                 snat_det_ses_close(dm0, ses0);
1336
1337               old_port0 = tcp0->dst;
1338               tcp0->dst = new_port0;
1339
1340               sum0 = tcp0->checksum;
1341               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1342                                      ip4_header_t,
1343                                      dst_address /* changed member */);
1344
1345               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1346                                      ip4_header_t /* cheat */,
1347                                      length /* changed member */);
1348               tcp0->checksum = ip_csum_fold(sum0);
1349             }
1350           else
1351             {
1352               old_port0 = udp0->dst_port;
1353               udp0->dst_port = new_port0;
1354               udp0->checksum = 0;
1355             }
1356
1357         trace00:
1358
1359           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1360                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1361             {
1362               snat_out2in_trace_t *t =
1363                  vlib_add_trace (vm, node, b0, sizeof (*t));
1364               t->sw_if_index = sw_if_index0;
1365               t->next_index = next0;
1366               t->session_index = ~0;
1367               if (ses0)
1368                 t->session_index = ses0 - dm0->sessions;
1369             }
1370
1371           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1372
1373           /* verify speculative enqueue, maybe switch current next frame */
1374           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1375                                            to_next, n_left_to_next,
1376                                            bi0, next0);
1377         }
1378
1379       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1380     }
1381
1382   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1383                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1384                                pkts_processed);
1385   return frame->n_vectors;
1386 }
1387
1388 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1389   .function = snat_det_out2in_node_fn,
1390   .name = "snat-det-out2in",
1391   .vector_size = sizeof (u32),
1392   .format_trace = format_snat_out2in_trace,
1393   .type = VLIB_NODE_TYPE_INTERNAL,
1394
1395   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1396   .error_strings = snat_out2in_error_strings,
1397
1398   .runtime_data_bytes = sizeof (snat_runtime_t),
1399
1400   .n_next_nodes = 2,
1401
1402   /* edit / add dispositions here */
1403   .next_nodes = {
1404     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1405     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1406   },
1407 };
1408 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1409
1410 /**********************/
1411 /*** worker handoff ***/
1412 /**********************/
1413 static uword
1414 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1415                                vlib_node_runtime_t * node,
1416                                vlib_frame_t * frame)
1417 {
1418   snat_main_t *sm = &snat_main;
1419   vlib_thread_main_t *tm = vlib_get_thread_main ();
1420   u32 n_left_from, *from, *to_next = 0;
1421   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1422   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1423     = 0;
1424   vlib_frame_queue_elt_t *hf = 0;
1425   vlib_frame_t *f = 0;
1426   int i;
1427   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1428   u32 next_worker_index = 0;
1429   u32 current_worker_index = ~0;
1430   u32 cpu_index = os_get_cpu_number ();
1431
1432   ASSERT (vec_len (sm->workers));
1433
1434   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1435     {
1436       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1437
1438       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1439                                sm->first_worker_index + sm->num_workers - 1,
1440                                (vlib_frame_queue_t *) (~0));
1441     }
1442
1443   from = vlib_frame_vector_args (frame);
1444   n_left_from = frame->n_vectors;
1445
1446   while (n_left_from > 0)
1447     {
1448       u32 bi0;
1449       vlib_buffer_t *b0;
1450       u32 sw_if_index0;
1451       u32 rx_fib_index0;
1452       ip4_header_t * ip0;
1453       u8 do_handoff;
1454
1455       bi0 = from[0];
1456       from += 1;
1457       n_left_from -= 1;
1458
1459       b0 = vlib_get_buffer (vm, bi0);
1460
1461       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1462       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1463
1464       ip0 = vlib_buffer_get_current (b0);
1465
1466       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1467
1468       if (PREDICT_FALSE (next_worker_index != cpu_index))
1469         {
1470           do_handoff = 1;
1471
1472           if (next_worker_index != current_worker_index)
1473             {
1474               if (hf)
1475                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1476
1477               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1478                                                       next_worker_index,
1479                                                       handoff_queue_elt_by_worker_index);
1480
1481               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1482               to_next_worker = &hf->buffer_index[hf->n_vectors];
1483               current_worker_index = next_worker_index;
1484             }
1485
1486           /* enqueue to correct worker thread */
1487           to_next_worker[0] = bi0;
1488           to_next_worker++;
1489           n_left_to_next_worker--;
1490
1491           if (n_left_to_next_worker == 0)
1492             {
1493               hf->n_vectors = VLIB_FRAME_SIZE;
1494               vlib_put_frame_queue_elt (hf);
1495               current_worker_index = ~0;
1496               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1497               hf = 0;
1498             }
1499         }
1500       else
1501         {
1502           do_handoff = 0;
1503           /* if this is 1st frame */
1504           if (!f)
1505             {
1506               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1507               to_next = vlib_frame_vector_args (f);
1508             }
1509
1510           to_next[0] = bi0;
1511           to_next += 1;
1512           f->n_vectors++;
1513         }
1514
1515       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1516                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1517         {
1518           snat_out2in_worker_handoff_trace_t *t =
1519             vlib_add_trace (vm, node, b0, sizeof (*t));
1520           t->next_worker_index = next_worker_index;
1521           t->do_handoff = do_handoff;
1522         }
1523     }
1524
1525   if (f)
1526     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1527
1528   if (hf)
1529     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1530
1531   /* Ship frames to the worker nodes */
1532   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1533     {
1534       if (handoff_queue_elt_by_worker_index[i])
1535         {
1536           hf = handoff_queue_elt_by_worker_index[i];
1537           /*
1538            * It works better to let the handoff node
1539            * rate-adapt, always ship the handoff queue element.
1540            */
1541           if (1 || hf->n_vectors == hf->last_n_vectors)
1542             {
1543               vlib_put_frame_queue_elt (hf);
1544               handoff_queue_elt_by_worker_index[i] = 0;
1545             }
1546           else
1547             hf->last_n_vectors = hf->n_vectors;
1548         }
1549       congested_handoff_queue_by_worker_index[i] =
1550         (vlib_frame_queue_t *) (~0);
1551     }
1552   hf = 0;
1553   current_worker_index = ~0;
1554   return frame->n_vectors;
1555 }
1556
1557 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1558   .function = snat_out2in_worker_handoff_fn,
1559   .name = "snat-out2in-worker-handoff",
1560   .vector_size = sizeof (u32),
1561   .format_trace = format_snat_out2in_worker_handoff_trace,
1562   .type = VLIB_NODE_TYPE_INTERNAL,
1563
1564   .n_next_nodes = 1,
1565
1566   .next_nodes = {
1567     [0] = "error-drop",
1568   },
1569 };
1570
1571 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1572
1573 /********************************/
1574 /*** static mapping only mode ***/
1575 /********************************/
1576 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1577                                     vlib_buffer_t * b0,
1578                                     ip4_header_t * ip0,
1579                                     icmp46_header_t * icmp0,
1580                                     u32 sw_if_index0,
1581                                     vlib_node_runtime_t * node,
1582                                     u32 next0,
1583                                     u32 rx_fib_index0)
1584 {
1585   snat_session_key_t key0, sm0;
1586   icmp_echo_header_t *echo0;
1587   u32 new_addr0, old_addr0;
1588   u16 old_id0, new_id0;
1589   ip_csum_t sum0;
1590   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1591
1592   echo0 = (icmp_echo_header_t *)(icmp0+1);
1593
1594   key0.addr = ip0->dst_address;
1595   key0.port = echo0->identifier;
1596   key0.fib_index = rx_fib_index0;
1597
1598   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1599     {
1600       ip4_address_t * first_int_addr;
1601
1602       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1603         {
1604           first_int_addr =
1605             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1606                                          0 /* just want the address */);
1607           rt->cached_sw_if_index = sw_if_index0;
1608           if (first_int_addr)
1609             rt->cached_ip4_address = first_int_addr->as_u32;
1610           else
1611             rt->cached_ip4_address = 0;
1612         }
1613
1614       /* Don't NAT packet aimed at the intfc address */
1615       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1616                         rt->cached_ip4_address))
1617         return next0;
1618
1619       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1620       return SNAT_OUT2IN_NEXT_DROP;
1621     }
1622
1623   new_addr0 = sm0.addr.as_u32;
1624   new_id0 = sm0.port;
1625   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1626
1627   old_addr0 = ip0->dst_address.as_u32;
1628   ip0->dst_address.as_u32 = new_addr0;
1629
1630   sum0 = ip0->checksum;
1631   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1632                          ip4_header_t,
1633                          dst_address /* changed member */);
1634   ip0->checksum = ip_csum_fold (sum0);
1635
1636   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1637     {
1638       old_id0 = echo0->identifier;
1639       echo0->identifier = new_id0;
1640
1641       sum0 = icmp0->checksum;
1642       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1643                              identifier);
1644       icmp0->checksum = ip_csum_fold (sum0);
1645     }
1646
1647   return next0;
1648 }
1649
1650 static uword
1651 snat_out2in_fast_node_fn (vlib_main_t * vm,
1652                           vlib_node_runtime_t * node,
1653                           vlib_frame_t * frame)
1654 {
1655   u32 n_left_from, * from, * to_next;
1656   snat_out2in_next_t next_index;
1657   u32 pkts_processed = 0;
1658   snat_main_t * sm = &snat_main;
1659
1660   from = vlib_frame_vector_args (frame);
1661   n_left_from = frame->n_vectors;
1662   next_index = node->cached_next_index;
1663
1664   while (n_left_from > 0)
1665     {
1666       u32 n_left_to_next;
1667
1668       vlib_get_next_frame (vm, node, next_index,
1669                            to_next, n_left_to_next);
1670
1671       while (n_left_from > 0 && n_left_to_next > 0)
1672         {
1673           u32 bi0;
1674           vlib_buffer_t * b0;
1675           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1676           u32 sw_if_index0;
1677           ip4_header_t * ip0;
1678           ip_csum_t sum0;
1679           u32 new_addr0, old_addr0;
1680           u16 new_port0, old_port0;
1681           udp_header_t * udp0;
1682           tcp_header_t * tcp0;
1683           icmp46_header_t * icmp0;
1684           snat_session_key_t key0, sm0;
1685           u32 proto0;
1686           u32 rx_fib_index0;
1687
1688           /* speculatively enqueue b0 to the current next frame */
1689           bi0 = from[0];
1690           to_next[0] = bi0;
1691           from += 1;
1692           to_next += 1;
1693           n_left_from -= 1;
1694           n_left_to_next -= 1;
1695
1696           b0 = vlib_get_buffer (vm, bi0);
1697
1698           ip0 = vlib_buffer_get_current (b0);
1699           udp0 = ip4_next_header (ip0);
1700           tcp0 = (tcp_header_t *) udp0;
1701           icmp0 = (icmp46_header_t *) udp0;
1702
1703           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1704           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1705
1706           vnet_feature_next (sw_if_index0, &next0, b0);
1707
1708           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1709
1710           if (PREDICT_FALSE (proto0 == ~0))
1711               goto trace00;
1712
1713           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1714             {
1715               next0 = icmp_out2in_fast
1716                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1717               goto trace00;
1718             }
1719
1720           key0.addr = ip0->dst_address;
1721           key0.port = udp0->dst_port;
1722           key0.fib_index = rx_fib_index0;
1723
1724           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1725             {
1726               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1727               goto trace00;
1728             }
1729
1730           new_addr0 = sm0.addr.as_u32;
1731           new_port0 = sm0.port;
1732           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1733           old_addr0 = ip0->dst_address.as_u32;
1734           ip0->dst_address.as_u32 = new_addr0;
1735
1736           sum0 = ip0->checksum;
1737           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1738                                  ip4_header_t,
1739                                  dst_address /* changed member */);
1740           ip0->checksum = ip_csum_fold (sum0);
1741
1742           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1743             {
1744                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1745                 {
1746                   old_port0 = tcp0->dst_port;
1747                   tcp0->dst_port = new_port0;
1748
1749                   sum0 = tcp0->checksum;
1750                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1751                                          ip4_header_t,
1752                                          dst_address /* changed member */);
1753
1754                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1755                                          ip4_header_t /* cheat */,
1756                                          length /* changed member */);
1757                   tcp0->checksum = ip_csum_fold(sum0);
1758                 }
1759               else
1760                 {
1761                   old_port0 = udp0->dst_port;
1762                   udp0->dst_port = new_port0;
1763                   udp0->checksum = 0;
1764                 }
1765             }
1766           else
1767             {
1768               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1769                 {
1770                   sum0 = tcp0->checksum;
1771                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1772                                          ip4_header_t,
1773                                          dst_address /* changed member */);
1774
1775                   tcp0->checksum = ip_csum_fold(sum0);
1776                 }
1777             }
1778
1779         trace00:
1780
1781           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1782                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1783             {
1784               snat_out2in_trace_t *t =
1785                  vlib_add_trace (vm, node, b0, sizeof (*t));
1786               t->sw_if_index = sw_if_index0;
1787               t->next_index = next0;
1788             }
1789
1790           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1791
1792           /* verify speculative enqueue, maybe switch current next frame */
1793           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1794                                            to_next, n_left_to_next,
1795                                            bi0, next0);
1796         }
1797
1798       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1799     }
1800
1801   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1802                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1803                                pkts_processed);
1804   return frame->n_vectors;
1805 }
1806
1807 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1808   .function = snat_out2in_fast_node_fn,
1809   .name = "snat-out2in-fast",
1810   .vector_size = sizeof (u32),
1811   .format_trace = format_snat_out2in_fast_trace,
1812   .type = VLIB_NODE_TYPE_INTERNAL,
1813
1814   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1815   .error_strings = snat_out2in_error_strings,
1816
1817   .runtime_data_bytes = sizeof (snat_runtime_t),
1818
1819   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1820
1821   /* edit / add dispositions here */
1822   .next_nodes = {
1823     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1824     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1825     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1826   },
1827 };
1828 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);