src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36 } snat_out2in_trace_t;
  37
  38 typedef struct {
  39   u32 next_worker_index;
  40   u8 do_handoff;
  41 } snat_out2in_worker_handoff_trace_t;
  42
  43 /* packet trace format function */
  44 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  45 {
  46   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  47   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  48   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  49
  50   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  51               t->sw_if_index, t->next_index, t->session_index);
  52   return s;
  53 }
  54
  55 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  56 {
  57   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  58   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  59   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  60
  61   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  62               t->sw_if_index, t->next_index);
  63   return s;
  64 }
  65
  66 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   snat_out2in_worker_handoff_trace_t * t =
  71     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  72   char * m;
  73
  74   m = t->do_handoff ? "next worker" : "same worker";
  75   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  76
  77   return s;
  78 }
  79
  80 vlib_node_registration_t snat_out2in_node;
  81 vlib_node_registration_t snat_out2in_fast_node;
  82 vlib_node_registration_t snat_out2in_worker_handoff_node;
  83
  84 #define foreach_snat_out2in_error                       \
  85 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  86 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  87 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  88 _(NO_TRANSLATION, "No translation")
  89
  90 typedef enum {
  91 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  92   foreach_snat_out2in_error
  93 #undef _
  94   SNAT_OUT2IN_N_ERROR,
  95 } snat_out2in_error_t;
  96
  97 static char * snat_out2in_error_strings[] = {
  98 #define _(sym,string) string,
  99   foreach_snat_out2in_error
 100 #undef _
 101 };
 102
 103 typedef enum {
 104   SNAT_OUT2IN_NEXT_DROP,
 105   SNAT_OUT2IN_NEXT_LOOKUP,
 106   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 107   SNAT_OUT2IN_N_NEXT,
 108 } snat_out2in_next_t;
 109
 110 /**
 111  * @brief Create session for static mapping.
 112  *
 113  * Create NAT session initiated by host from external network with static
 114  * mapping.
 115  *
 116  * @param sm     SNAT main.
 117  * @param b0     Vlib buffer.
 118  * @param in2out In2out SNAT session key.
 119  * @param out2in Out2in SNAT session key.
 120  * @param node   Vlib node.
 121  *
 122  * @returns SNAT session if successfully created otherwise 0.
 123  */
 124 static inline snat_session_t *
 125 create_session_for_static_mapping (snat_main_t *sm,
 126                                    vlib_buffer_t *b0,
 127                                    snat_session_key_t in2out,
 128                                    snat_session_key_t out2in,
 129                                    vlib_node_runtime_t * node,
 130                                    u32 cpu_index)
 131 {
 132   snat_user_t *u;
 133   snat_user_key_t user_key;
 134   snat_session_t *s;
 135   clib_bihash_kv_8_8_t kv0, value0;
 136   dlist_elt_t * per_user_translation_list_elt;
 137   dlist_elt_t * per_user_list_head_elt;
 138
 139   user_key.addr = in2out.addr;
 140   user_key.fib_index = in2out.fib_index;
 141   kv0.key = user_key.as_u64;
 142
 143   /* Ever heard of the "user" = inside ip4 address before? */
 144   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 145     {
 146       /* no, make a new one */
 147       pool_get (sm->per_thread_data[cpu_index].users, u);
 148       memset (u, 0, sizeof (*u));
 149       u->addr = in2out.addr;
 150       u->fib_index = in2out.fib_index;
 151
 152       pool_get (sm->per_thread_data[cpu_index].list_pool,
 153                 per_user_list_head_elt);
 154
 155       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 156         sm->per_thread_data[cpu_index].list_pool;
 157
 158       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 159                        u->sessions_per_user_list_head_index);
 160
 161       kv0.value = u - sm->per_thread_data[cpu_index].users;
 162
 163       /* add user */
 164       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 165
 166       /* add non-traslated packets worker lookup */
 167       kv0.value = cpu_index;
 168       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 169     }
 170   else
 171     {
 172       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 173                              value0.value);
 174     }
 175
 176   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 177   memset (s, 0, sizeof (*s));
 178
 179   s->outside_address_index = ~0;
 180   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 181   u->nstaticsessions++;
 182
 183   /* Create list elts */
 184   pool_get (sm->per_thread_data[cpu_index].list_pool,
 185             per_user_translation_list_elt);
 186   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 187                    per_user_translation_list_elt -
 188                    sm->per_thread_data[cpu_index].list_pool);
 189
 190   per_user_translation_list_elt->value =
 191     s - sm->per_thread_data[cpu_index].sessions;
 192   s->per_user_index =
 193     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 194   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 195
 196   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 197                       s->per_user_list_head_index,
 198                       per_user_translation_list_elt -
 199                       sm->per_thread_data[cpu_index].list_pool);
 200
 201   s->in2out = in2out;
 202   s->out2in = out2in;
 203   s->in2out.protocol = out2in.protocol;
 204
 205   /* Add to translation hashes */
 206   kv0.key = s->in2out.as_u64;
 207   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 208   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 209       clib_warning ("in2out key add failed");
 210
 211   kv0.key = s->out2in.as_u64;
 212   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 213
 214   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 215       clib_warning ("out2in key add failed");
 216
 217   /* log NAT event */
 218   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 219                                       s->out2in.addr.as_u32,
 220                                       s->in2out.protocol,
 221                                       s->in2out.port,
 222                                       s->out2in.port,
 223                                       s->in2out.fib_index);
 224    return s;
 225 }
 226
 227 typedef struct {
 228   u16 src_port, dst_port;
 229 } tcp_udp_header_t;
 230
 231 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 232                                          vlib_buffer_t * b0,
 233                                          ip4_header_t * ip0,
 234                                          icmp46_header_t * icmp0,
 235                                          u32 sw_if_index0,
 236                                          u32 rx_fib_index0,
 237                                          vlib_node_runtime_t * node,
 238                                          u32 next0, f64 now,
 239                                          u32 cpu_index,
 240                                          snat_session_t ** p_s0)
 241 {
 242   snat_session_key_t key0, sm0;
 243   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 244   ip4_header_t *inner_ip0 = 0;
 245   void *l4_header = 0;
 246   icmp46_header_t *inner_icmp0;
 247   clib_bihash_kv_8_8_t kv0, value0;
 248   snat_session_t * s0 = 0;
 249   u32 new_addr0, old_addr0;
 250   u16 old_id0, new_id0;
 251   ip_csum_t sum0;
 252   u16 checksum0;
 253   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 254   u8 is_error_message = 0;
 255
 256   echo0 = (icmp_echo_header_t *)(icmp0+1);
 257
 258   key0.addr = ip0->dst_address;
 259   key0.fib_index = rx_fib_index0;
 260
 261   switch(icmp0->type)
 262     {
 263     case ICMP4_destination_unreachable:
 264     case ICMP4_time_exceeded:
 265     case ICMP4_parameter_problem:
 266     case ICMP4_source_quench:
 267     case ICMP4_redirect:
 268     case ICMP4_alternate_host_address:
 269       is_error_message = 1;
 270     }
 271
 272   if (!is_error_message)
 273     {
 274       key0.protocol = SNAT_PROTOCOL_ICMP;
 275       key0.port = echo0->identifier;
 276     }
 277   else
 278     {
 279       inner_ip0 = (ip4_header_t *)(echo0+1);
 280       l4_header = ip4_next_header (inner_ip0);
 281       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 282       switch (key0.protocol)
 283         {
 284         case SNAT_PROTOCOL_ICMP:
 285           inner_icmp0 = (icmp46_header_t*)l4_header;
 286           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 287           key0.port = inner_echo0->identifier;
 288           break;
 289         case SNAT_PROTOCOL_UDP:
 290         case SNAT_PROTOCOL_TCP:
 291           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 292           break;
 293         default:
 294           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 295           next0 = SNAT_OUT2IN_NEXT_DROP;
 296           goto out;
 297         }
 298     }
 299
 300   kv0.key = key0.as_u64;
 301
 302   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 303     {
 304       /* Try to match static mapping by external address and port,
 305          destination address and port in packet */
 306       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 307         {
 308            ip4_address_t * first_int_addr;
 309
 310           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 311             {
 312               first_int_addr =
 313                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 314                                              0 /* just want the address */);
 315               rt->cached_sw_if_index = sw_if_index0;
 316               if (first_int_addr)
 317                 rt->cached_ip4_address = first_int_addr->as_u32;
 318               else
 319                 rt->cached_ip4_address = 0;
 320             }
 321
 322           /* Don't NAT packet aimed at the intfc address */
 323           if (PREDICT_FALSE(ip0->dst_address.as_u32 == rt->cached_ip4_address))
 324             goto out;
 325
 326           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 327           next0 = SNAT_OUT2IN_NEXT_DROP;
 328           goto out;
 329         }
 330
 331       if (is_error_message)
 332         {
 333           next0 = SNAT_OUT2IN_NEXT_DROP;
 334           goto out;
 335         }
 336
 337       /* Create session initiated by host from external network */
 338       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 339                                              node, cpu_index);
 340
 341       if (!s0)
 342         {
 343           next0 = SNAT_OUT2IN_NEXT_DROP;
 344           goto out;
 345         }
 346     }
 347   else
 348     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 349                             value0.value);
 350
 351   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply && !is_error_message))
 352     {
 353       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 354       next0 = SNAT_OUT2IN_NEXT_DROP;
 355       goto out;
 356     }
 357
 358   sum0 = ip_incremental_checksum (0, icmp0,
 359                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 360   checksum0 = ~ip_csum_fold (sum0);
 361   if (checksum0 != 0 && checksum0 != 0xffff)
 362     {
 363       next0 = SNAT_OUT2IN_NEXT_DROP;
 364       goto out;
 365     }
 366
 367   old_addr0 = ip0->dst_address.as_u32;
 368   ip0->dst_address = s0->in2out.addr;
 369   new_addr0 = ip0->dst_address.as_u32;
 370   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 371
 372   sum0 = ip0->checksum;
 373   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 374                          dst_address /* changed member */);
 375   ip0->checksum = ip_csum_fold (sum0);
 376
 377   if (!is_error_message)
 378     {
 379       old_id0 = echo0->identifier;
 380       new_id0 = s0->in2out.port;
 381       echo0->identifier = new_id0;
 382
 383       sum0 = icmp0->checksum;
 384       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 385                              identifier /* changed member */);
 386       icmp0->checksum = ip_csum_fold (sum0);
 387     }
 388   else
 389     {
 390       if (!ip4_header_checksum_is_valid (inner_ip0))
 391         {
 392           next0 = SNAT_OUT2IN_NEXT_DROP;
 393           goto out;
 394         }
 395
 396       old_addr0 = inner_ip0->src_address.as_u32;
 397       inner_ip0->src_address = s0->in2out.addr;
 398       new_addr0 = inner_ip0->src_address.as_u32;
 399
 400       sum0 = icmp0->checksum;
 401       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 402                              src_address /* changed member */);
 403       icmp0->checksum = ip_csum_fold (sum0);
 404
 405       switch (key0.protocol)
 406         {
 407         case SNAT_PROTOCOL_ICMP:
 408           old_id0 = inner_echo0->identifier;
 409           new_id0 = s0->in2out.port;
 410           inner_echo0->identifier = new_id0;
 411
 412           sum0 = icmp0->checksum;
 413           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 414                                  identifier);
 415           icmp0->checksum = ip_csum_fold (sum0);
 416           break;
 417         case SNAT_PROTOCOL_UDP:
 418         case SNAT_PROTOCOL_TCP:
 419           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 420           new_id0 = s0->in2out.port;
 421           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 422
 423           sum0 = icmp0->checksum;
 424           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 425                                  src_port);
 426           icmp0->checksum = ip_csum_fold (sum0);
 427           break;
 428         default:
 429           ASSERT(0);
 430         }
 431     }
 432
 433   /* Accounting */
 434   s0->last_heard = now;
 435   s0->total_pkts++;
 436   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 437   /* Per-user LRU list maintenance for dynamic translation */
 438   if (!snat_is_session_static (s0))
 439     {
 440       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 441                          s0->per_user_index);
 442       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 443                           s0->per_user_list_head_index,
 444                           s0->per_user_index);
 445     }
 446
 447 out:
 448   *p_s0 = s0;
 449   return next0;
 450 }
 451
 452 static uword
 453 snat_out2in_node_fn (vlib_main_t * vm,
 454                   vlib_node_runtime_t * node,
 455                   vlib_frame_t * frame)
 456 {
 457   u32 n_left_from, * from, * to_next;
 458   snat_out2in_next_t next_index;
 459   u32 pkts_processed = 0;
 460   snat_main_t * sm = &snat_main;
 461   f64 now = vlib_time_now (vm);
 462   u32 cpu_index = os_get_cpu_number ();
 463
 464   from = vlib_frame_vector_args (frame);
 465   n_left_from = frame->n_vectors;
 466   next_index = node->cached_next_index;
 467
 468   while (n_left_from > 0)
 469     {
 470       u32 n_left_to_next;
 471
 472       vlib_get_next_frame (vm, node, next_index,
 473                            to_next, n_left_to_next);
 474
 475       while (n_left_from >= 4 && n_left_to_next >= 2)
 476         {
 477           u32 bi0, bi1;
 478           vlib_buffer_t * b0, * b1;
 479           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 480           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 481           u32 sw_if_index0, sw_if_index1;
 482           ip4_header_t * ip0, *ip1;
 483           ip_csum_t sum0, sum1;
 484           u32 new_addr0, old_addr0;
 485           u16 new_port0, old_port0;
 486           u32 new_addr1, old_addr1;
 487           u16 new_port1, old_port1;
 488           udp_header_t * udp0, * udp1;
 489           tcp_header_t * tcp0, * tcp1;
 490           icmp46_header_t * icmp0, * icmp1;
 491           snat_session_key_t key0, key1, sm0, sm1;
 492           u32 rx_fib_index0, rx_fib_index1;
 493           u32 proto0, proto1;
 494           snat_session_t * s0 = 0, * s1 = 0;
 495           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 496
 497           /* Prefetch next iteration. */
 498           {
 499             vlib_buffer_t * p2, * p3;
 500
 501             p2 = vlib_get_buffer (vm, from[2]);
 502             p3 = vlib_get_buffer (vm, from[3]);
 503
 504             vlib_prefetch_buffer_header (p2, LOAD);
 505             vlib_prefetch_buffer_header (p3, LOAD);
 506
 507             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 508             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 509           }
 510
 511           /* speculatively enqueue b0 and b1 to the current next frame */
 512           to_next[0] = bi0 = from[0];
 513           to_next[1] = bi1 = from[1];
 514           from += 2;
 515           to_next += 2;
 516           n_left_from -= 2;
 517           n_left_to_next -= 2;
 518
 519           b0 = vlib_get_buffer (vm, bi0);
 520           b1 = vlib_get_buffer (vm, bi1);
 521
 522           ip0 = vlib_buffer_get_current (b0);
 523           udp0 = ip4_next_header (ip0);
 524           tcp0 = (tcp_header_t *) udp0;
 525           icmp0 = (icmp46_header_t *) udp0;
 526
 527           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 528           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 529                                    sw_if_index0);
 530
 531           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 532
 533           if (PREDICT_FALSE (proto0 == ~0))
 534               goto trace0;
 535
 536           if (PREDICT_FALSE(ip0->ttl == 1))
 537             {
 538               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 539               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 540                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 541                                            0);
 542               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 543               goto trace0;
 544             }
 545
 546           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 547             {
 548               next0 = icmp_out2in_slow_path
 549                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 550                  next0, now, cpu_index, &s0);
 551               goto trace0;
 552             }
 553
 554           key0.addr = ip0->dst_address;
 555           key0.port = udp0->dst_port;
 556           key0.protocol = proto0;
 557           key0.fib_index = rx_fib_index0;
 558
 559           kv0.key = key0.as_u64;
 560
 561           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 562             {
 563               /* Try to match static mapping by external address and port,
 564                  destination address and port in packet */
 565               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 566                 {
 567                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 568                   /*
 569                    * Send DHCP packets to the ipv4 stack, or we won't
 570                    * be able to use dhcp client on the outside interface
 571                    */
 572                   if (proto0 != SNAT_PROTOCOL_UDP
 573                       || (udp0->dst_port
 574                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 575                     next0 = SNAT_OUT2IN_NEXT_DROP;
 576                   goto trace0;
 577                 }
 578
 579               /* Create session initiated by host from external network */
 580               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 581                                                      cpu_index);
 582               if (!s0)
 583                 {
 584                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 585                   next0 = SNAT_OUT2IN_NEXT_DROP;
 586                   goto trace0;
 587                 }
 588             }
 589           else
 590             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 591                                     value0.value);
 592
 593           old_addr0 = ip0->dst_address.as_u32;
 594           ip0->dst_address = s0->in2out.addr;
 595           new_addr0 = ip0->dst_address.as_u32;
 596           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 597
 598           sum0 = ip0->checksum;
 599           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 600                                  ip4_header_t,
 601                                  dst_address /* changed member */);
 602           ip0->checksum = ip_csum_fold (sum0);
 603
 604           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 605             {
 606               old_port0 = tcp0->dst_port;
 607               tcp0->dst_port = s0->in2out.port;
 608               new_port0 = tcp0->dst_port;
 609
 610               sum0 = tcp0->checksum;
 611               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 612                                      ip4_header_t,
 613                                      dst_address /* changed member */);
 614
 615               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 616                                      ip4_header_t /* cheat */,
 617                                      length /* changed member */);
 618               tcp0->checksum = ip_csum_fold(sum0);
 619             }
 620           else
 621             {
 622               old_port0 = udp0->dst_port;
 623               udp0->dst_port = s0->in2out.port;
 624               udp0->checksum = 0;
 625             }
 626
 627           /* Accounting */
 628           s0->last_heard = now;
 629           s0->total_pkts++;
 630           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 631           /* Per-user LRU list maintenance for dynamic translation */
 632           if (!snat_is_session_static (s0))
 633             {
 634               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 635                                  s0->per_user_index);
 636               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 637                                   s0->per_user_list_head_index,
 638                                   s0->per_user_index);
 639             }
 640         trace0:
 641
 642           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 643                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 644             {
 645               snat_out2in_trace_t *t =
 646                  vlib_add_trace (vm, node, b0, sizeof (*t));
 647               t->sw_if_index = sw_if_index0;
 648               t->next_index = next0;
 649               t->session_index = ~0;
 650               if (s0)
 651                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 652             }
 653
 654           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 655
 656
 657           ip1 = vlib_buffer_get_current (b1);
 658           udp1 = ip4_next_header (ip1);
 659           tcp1 = (tcp_header_t *) udp1;
 660           icmp1 = (icmp46_header_t *) udp1;
 661
 662           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 663           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 664                                    sw_if_index1);
 665
 666           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 667
 668           if (PREDICT_FALSE (proto1 == ~0))
 669               goto trace1;
 670
 671           if (PREDICT_FALSE(ip0->ttl == 1))
 672             {
 673               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 674               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 675                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 676                                            0);
 677               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 678               goto trace1;
 679             }
 680
 681           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 682             {
 683               next1 = icmp_out2in_slow_path
 684                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 685                  next1, now, cpu_index, &s1);
 686               goto trace1;
 687             }
 688
 689           key1.addr = ip1->dst_address;
 690           key1.port = udp1->dst_port;
 691           key1.protocol = proto1;
 692           key1.fib_index = rx_fib_index1;
 693
 694           kv1.key = key1.as_u64;
 695
 696           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 697             {
 698               /* Try to match static mapping by external address and port,
 699                  destination address and port in packet */
 700               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 701                 {
 702                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 703                   /*
 704                    * Send DHCP packets to the ipv4 stack, or we won't
 705                    * be able to use dhcp client on the outside interface
 706                    */
 707                   if (proto1 != SNAT_PROTOCOL_UDP
 708                       || (udp1->dst_port
 709                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 710                     next1 = SNAT_OUT2IN_NEXT_DROP;
 711                   goto trace1;
 712                 }
 713
 714               /* Create session initiated by host from external network */
 715               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 716                                                      cpu_index);
 717               if (!s1)
 718                 {
 719                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 720                   next1 = SNAT_OUT2IN_NEXT_DROP;
 721                   goto trace1;
 722                 }
 723             }
 724           else
 725             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 726                                     value1.value);
 727
 728           old_addr1 = ip1->dst_address.as_u32;
 729           ip1->dst_address = s1->in2out.addr;
 730           new_addr1 = ip1->dst_address.as_u32;
 731           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 732
 733           sum1 = ip1->checksum;
 734           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 735                                  ip4_header_t,
 736                                  dst_address /* changed member */);
 737           ip1->checksum = ip_csum_fold (sum1);
 738
 739           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 740             {
 741               old_port1 = tcp1->dst_port;
 742               tcp1->dst_port = s1->in2out.port;
 743               new_port1 = tcp1->dst_port;
 744
 745               sum1 = tcp1->checksum;
 746               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 747                                      ip4_header_t,
 748                                      dst_address /* changed member */);
 749
 750               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 751                                      ip4_header_t /* cheat */,
 752                                      length /* changed member */);
 753               tcp1->checksum = ip_csum_fold(sum1);
 754             }
 755           else
 756             {
 757               old_port1 = udp1->dst_port;
 758               udp1->dst_port = s1->in2out.port;
 759               udp1->checksum = 0;
 760             }
 761
 762           /* Accounting */
 763           s1->last_heard = now;
 764           s1->total_pkts++;
 765           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 766           /* Per-user LRU list maintenance for dynamic translation */
 767           if (!snat_is_session_static (s1))
 768             {
 769               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 770                                  s1->per_user_index);
 771               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 772                                   s1->per_user_list_head_index,
 773                                   s1->per_user_index);
 774             }
 775         trace1:
 776
 777           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 778                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 779             {
 780               snat_out2in_trace_t *t =
 781                  vlib_add_trace (vm, node, b1, sizeof (*t));
 782               t->sw_if_index = sw_if_index1;
 783               t->next_index = next1;
 784               t->session_index = ~0;
 785               if (s1)
 786                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 787             }
 788
 789           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 790
 791           /* verify speculative enqueues, maybe switch current next frame */
 792           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 793                                            to_next, n_left_to_next,
 794                                            bi0, bi1, next0, next1);
 795         }
 796
 797       while (n_left_from > 0 && n_left_to_next > 0)
 798         {
 799           u32 bi0;
 800           vlib_buffer_t * b0;
 801           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 802           u32 sw_if_index0;
 803           ip4_header_t * ip0;
 804           ip_csum_t sum0;
 805           u32 new_addr0, old_addr0;
 806           u16 new_port0, old_port0;
 807           udp_header_t * udp0;
 808           tcp_header_t * tcp0;
 809           icmp46_header_t * icmp0;
 810           snat_session_key_t key0, sm0;
 811           u32 rx_fib_index0;
 812           u32 proto0;
 813           snat_session_t * s0 = 0;
 814           clib_bihash_kv_8_8_t kv0, value0;
 815
 816           /* speculatively enqueue b0 to the current next frame */
 817           bi0 = from[0];
 818           to_next[0] = bi0;
 819           from += 1;
 820           to_next += 1;
 821           n_left_from -= 1;
 822           n_left_to_next -= 1;
 823
 824           b0 = vlib_get_buffer (vm, bi0);
 825
 826           ip0 = vlib_buffer_get_current (b0);
 827           udp0 = ip4_next_header (ip0);
 828           tcp0 = (tcp_header_t *) udp0;
 829           icmp0 = (icmp46_header_t *) udp0;
 830
 831           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 832           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 833                                    sw_if_index0);
 834
 835           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 836
 837           if (PREDICT_FALSE (proto0 == ~0))
 838               goto trace00;
 839
 840           if (PREDICT_FALSE(ip0->ttl == 1))
 841             {
 842               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 843               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 844                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 845                                            0);
 846               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 847               goto trace00;
 848             }
 849
 850           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 851             {
 852               next0 = icmp_out2in_slow_path
 853                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 854                  next0, now, cpu_index, &s0);
 855               goto trace00;
 856             }
 857
 858           key0.addr = ip0->dst_address;
 859           key0.port = udp0->dst_port;
 860           key0.protocol = proto0;
 861           key0.fib_index = rx_fib_index0;
 862
 863           kv0.key = key0.as_u64;
 864
 865           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 866             {
 867               /* Try to match static mapping by external address and port,
 868                  destination address and port in packet */
 869               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 870                 {
 871                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 872                   /*
 873                    * Send DHCP packets to the ipv4 stack, or we won't
 874                    * be able to use dhcp client on the outside interface
 875                    */
 876                   if (proto0 != SNAT_PROTOCOL_UDP
 877                       || (udp0->dst_port
 878                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 879
 880                     next0 = SNAT_OUT2IN_NEXT_DROP;
 881                   goto trace00;
 882                 }
 883
 884               /* Create session initiated by host from external network */
 885               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 886                                                      cpu_index);
 887               if (!s0)
 888                 {
 889                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 890                     next0 = SNAT_OUT2IN_NEXT_DROP;
 891                   goto trace00;
 892                 }
 893             }
 894           else
 895             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 896                                     value0.value);
 897
 898           old_addr0 = ip0->dst_address.as_u32;
 899           ip0->dst_address = s0->in2out.addr;
 900           new_addr0 = ip0->dst_address.as_u32;
 901           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 902
 903           sum0 = ip0->checksum;
 904           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 905                                  ip4_header_t,
 906                                  dst_address /* changed member */);
 907           ip0->checksum = ip_csum_fold (sum0);
 908
 909           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 910             {
 911               old_port0 = tcp0->dst_port;
 912               tcp0->dst_port = s0->in2out.port;
 913               new_port0 = tcp0->dst_port;
 914
 915               sum0 = tcp0->checksum;
 916               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 917                                      ip4_header_t,
 918                                      dst_address /* changed member */);
 919
 920               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 921                                      ip4_header_t /* cheat */,
 922                                      length /* changed member */);
 923               tcp0->checksum = ip_csum_fold(sum0);
 924             }
 925           else
 926             {
 927               old_port0 = udp0->dst_port;
 928               udp0->dst_port = s0->in2out.port;
 929               udp0->checksum = 0;
 930             }
 931
 932           /* Accounting */
 933           s0->last_heard = now;
 934           s0->total_pkts++;
 935           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 936           /* Per-user LRU list maintenance for dynamic translation */
 937           if (!snat_is_session_static (s0))
 938             {
 939               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 940                                  s0->per_user_index);
 941               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 942                                   s0->per_user_list_head_index,
 943                                   s0->per_user_index);
 944             }
 945         trace00:
 946
 947           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 948                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 949             {
 950               snat_out2in_trace_t *t =
 951                  vlib_add_trace (vm, node, b0, sizeof (*t));
 952               t->sw_if_index = sw_if_index0;
 953               t->next_index = next0;
 954               t->session_index = ~0;
 955               if (s0)
 956                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 957             }
 958
 959           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 960
 961           /* verify speculative enqueue, maybe switch current next frame */
 962           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 963                                            to_next, n_left_to_next,
 964                                            bi0, next0);
 965         }
 966
 967       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 968     }
 969
 970   vlib_node_increment_counter (vm, snat_out2in_node.index,
 971                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 972                                pkts_processed);
 973   return frame->n_vectors;
 974 }
 975
 976 VLIB_REGISTER_NODE (snat_out2in_node) = {
 977   .function = snat_out2in_node_fn,
 978   .name = "snat-out2in",
 979   .vector_size = sizeof (u32),
 980   .format_trace = format_snat_out2in_trace,
 981   .type = VLIB_NODE_TYPE_INTERNAL,
 982
 983   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 984   .error_strings = snat_out2in_error_strings,
 985
 986   .runtime_data_bytes = sizeof (snat_runtime_t),
 987
 988   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 989
 990   /* edit / add dispositions here */
 991   .next_nodes = {
 992     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 993     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 994     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
 995   },
 996 };
 997 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 998
 999 static uword
1000 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1001                                vlib_node_runtime_t * node,
1002                                vlib_frame_t * frame)
1003 {
1004   snat_main_t *sm = &snat_main;
1005   vlib_thread_main_t *tm = vlib_get_thread_main ();
1006   u32 n_left_from, *from, *to_next = 0;
1007   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1008   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1009     = 0;
1010   vlib_frame_queue_elt_t *hf = 0;
1011   vlib_frame_t *f = 0;
1012   int i;
1013   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1014   u32 next_worker_index = 0;
1015   u32 current_worker_index = ~0;
1016   u32 cpu_index = os_get_cpu_number ();
1017
1018   ASSERT (vec_len (sm->workers));
1019
1020   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1021     {
1022       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1023
1024       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1025                                sm->first_worker_index + sm->num_workers - 1,
1026                                (vlib_frame_queue_t *) (~0));
1027     }
1028
1029   from = vlib_frame_vector_args (frame);
1030   n_left_from = frame->n_vectors;
1031
1032   while (n_left_from > 0)
1033     {
1034       u32 bi0;
1035       vlib_buffer_t *b0;
1036       u32 sw_if_index0;
1037       u32 rx_fib_index0;
1038       ip4_header_t * ip0;
1039       udp_header_t * udp0;
1040       snat_worker_key_t key0;
1041       clib_bihash_kv_8_8_t kv0, value0;
1042       u8 do_handoff;
1043
1044       bi0 = from[0];
1045       from += 1;
1046       n_left_from -= 1;
1047
1048       b0 = vlib_get_buffer (vm, bi0);
1049
1050       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1051       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1052
1053       ip0 = vlib_buffer_get_current (b0);
1054       udp0 = ip4_next_header (ip0);
1055
1056       key0.addr = ip0->dst_address;
1057       key0.port = udp0->dst_port;
1058       key0.fib_index = rx_fib_index0;
1059
1060       if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
1061         {
1062           icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
1063           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1064           key0.port = echo0->identifier;
1065         }
1066
1067       kv0.key = key0.as_u64;
1068
1069       /* Ever heard of of the "user" before? */
1070       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
1071         {
1072           key0.port = 0;
1073           kv0.key = key0.as_u64;
1074
1075           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
1076             {
1077               /* No, assign next available worker (RR) */
1078               next_worker_index = sm->first_worker_index;
1079               if (vec_len (sm->workers))
1080                 {
1081                   next_worker_index +=
1082                     sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
1083                 }
1084             }
1085           else
1086             {
1087               /* Static mapping without port */
1088               next_worker_index = value0.value;
1089             }
1090
1091           /* Add to translated packets worker lookup */
1092           kv0.value = next_worker_index;
1093           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
1094         }
1095       else
1096         next_worker_index = value0.value;
1097
1098       if (PREDICT_FALSE (next_worker_index != cpu_index))
1099         {
1100           do_handoff = 1;
1101
1102           if (next_worker_index != current_worker_index)
1103             {
1104               if (hf)
1105                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1106
1107               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1108                                                       next_worker_index,
1109                                                       handoff_queue_elt_by_worker_index);
1110
1111               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1112               to_next_worker = &hf->buffer_index[hf->n_vectors];
1113               current_worker_index = next_worker_index;
1114             }
1115
1116           /* enqueue to correct worker thread */
1117           to_next_worker[0] = bi0;
1118           to_next_worker++;
1119           n_left_to_next_worker--;
1120
1121           if (n_left_to_next_worker == 0)
1122             {
1123               hf->n_vectors = VLIB_FRAME_SIZE;
1124               vlib_put_frame_queue_elt (hf);
1125               current_worker_index = ~0;
1126               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1127               hf = 0;
1128             }
1129         }
1130       else
1131         {
1132           do_handoff = 0;
1133           /* if this is 1st frame */
1134           if (!f)
1135             {
1136               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
1137               to_next = vlib_frame_vector_args (f);
1138             }
1139
1140           to_next[0] = bi0;
1141           to_next += 1;
1142           f->n_vectors++;
1143         }
1144
1145       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1146                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1147         {
1148           snat_out2in_worker_handoff_trace_t *t =
1149             vlib_add_trace (vm, node, b0, sizeof (*t));
1150           t->next_worker_index = next_worker_index;
1151           t->do_handoff = do_handoff;
1152         }
1153     }
1154
1155   if (f)
1156     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
1157
1158   if (hf)
1159     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1160
1161   /* Ship frames to the worker nodes */
1162   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1163     {
1164       if (handoff_queue_elt_by_worker_index[i])
1165         {
1166           hf = handoff_queue_elt_by_worker_index[i];
1167           /*
1168            * It works better to let the handoff node
1169            * rate-adapt, always ship the handoff queue element.
1170            */
1171           if (1 || hf->n_vectors == hf->last_n_vectors)
1172             {
1173               vlib_put_frame_queue_elt (hf);
1174               handoff_queue_elt_by_worker_index[i] = 0;
1175             }
1176           else
1177             hf->last_n_vectors = hf->n_vectors;
1178         }
1179       congested_handoff_queue_by_worker_index[i] =
1180         (vlib_frame_queue_t *) (~0);
1181     }
1182   hf = 0;
1183   current_worker_index = ~0;
1184   return frame->n_vectors;
1185 }
1186
1187 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1188   .function = snat_out2in_worker_handoff_fn,
1189   .name = "snat-out2in-worker-handoff",
1190   .vector_size = sizeof (u32),
1191   .format_trace = format_snat_out2in_worker_handoff_trace,
1192   .type = VLIB_NODE_TYPE_INTERNAL,
1193
1194   .n_next_nodes = 1,
1195
1196   .next_nodes = {
1197     [0] = "error-drop",
1198   },
1199 };
1200
1201 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1202
1203 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1204                                     vlib_buffer_t * b0,
1205                                     ip4_header_t * ip0,
1206                                     icmp46_header_t * icmp0,
1207                                     u32 sw_if_index0,
1208                                     vlib_node_runtime_t * node,
1209                                     u32 next0,
1210                                     u32 rx_fib_index0)
1211 {
1212   snat_session_key_t key0, sm0;
1213   icmp_echo_header_t *echo0;
1214   u32 new_addr0, old_addr0;
1215   u16 old_id0, new_id0;
1216   ip_csum_t sum0;
1217   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1218
1219   echo0 = (icmp_echo_header_t *)(icmp0+1);
1220
1221   key0.addr = ip0->dst_address;
1222   key0.port = echo0->identifier;
1223   key0.fib_index = rx_fib_index0;
1224
1225   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1226     {
1227       ip4_address_t * first_int_addr;
1228
1229       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1230         {
1231           first_int_addr =
1232             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1233                                          0 /* just want the address */);
1234           rt->cached_sw_if_index = sw_if_index0;
1235           if (first_int_addr)
1236             rt->cached_ip4_address = first_int_addr->as_u32;
1237           else
1238             rt->cached_ip4_address = 0;
1239         }
1240
1241       /* Don't NAT packet aimed at the intfc address */
1242       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1243                         rt->cached_ip4_address))
1244         return next0;
1245
1246       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1247       return SNAT_OUT2IN_NEXT_DROP;
1248     }
1249
1250   new_addr0 = sm0.addr.as_u32;
1251   new_id0 = sm0.port;
1252   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1253
1254   old_addr0 = ip0->dst_address.as_u32;
1255   ip0->dst_address.as_u32 = new_addr0;
1256
1257   sum0 = ip0->checksum;
1258   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1259                          ip4_header_t,
1260                          dst_address /* changed member */);
1261   ip0->checksum = ip_csum_fold (sum0);
1262
1263   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1264     {
1265       old_id0 = echo0->identifier;
1266       echo0->identifier = new_id0;
1267
1268       sum0 = icmp0->checksum;
1269       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1270                              identifier);
1271       icmp0->checksum = ip_csum_fold (sum0);
1272     }
1273
1274   return next0;
1275 }
1276
1277 static uword
1278 snat_out2in_fast_node_fn (vlib_main_t * vm,
1279                           vlib_node_runtime_t * node,
1280                           vlib_frame_t * frame)
1281 {
1282   u32 n_left_from, * from, * to_next;
1283   snat_out2in_next_t next_index;
1284   u32 pkts_processed = 0;
1285   snat_main_t * sm = &snat_main;
1286
1287   from = vlib_frame_vector_args (frame);
1288   n_left_from = frame->n_vectors;
1289   next_index = node->cached_next_index;
1290
1291   while (n_left_from > 0)
1292     {
1293       u32 n_left_to_next;
1294
1295       vlib_get_next_frame (vm, node, next_index,
1296                            to_next, n_left_to_next);
1297
1298       while (n_left_from > 0 && n_left_to_next > 0)
1299         {
1300           u32 bi0;
1301           vlib_buffer_t * b0;
1302           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1303           u32 sw_if_index0;
1304           ip4_header_t * ip0;
1305           ip_csum_t sum0;
1306           u32 new_addr0, old_addr0;
1307           u16 new_port0, old_port0;
1308           udp_header_t * udp0;
1309           tcp_header_t * tcp0;
1310           icmp46_header_t * icmp0;
1311           snat_session_key_t key0, sm0;
1312           u32 proto0;
1313           u32 rx_fib_index0;
1314
1315           /* speculatively enqueue b0 to the current next frame */
1316           bi0 = from[0];
1317           to_next[0] = bi0;
1318           from += 1;
1319           to_next += 1;
1320           n_left_from -= 1;
1321           n_left_to_next -= 1;
1322
1323           b0 = vlib_get_buffer (vm, bi0);
1324
1325           ip0 = vlib_buffer_get_current (b0);
1326           udp0 = ip4_next_header (ip0);
1327           tcp0 = (tcp_header_t *) udp0;
1328           icmp0 = (icmp46_header_t *) udp0;
1329
1330           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1331           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1332
1333           vnet_feature_next (sw_if_index0, &next0, b0);
1334
1335           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1336
1337           if (PREDICT_FALSE (proto0 == ~0))
1338               goto trace00;
1339
1340           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1341             {
1342               next0 = icmp_out2in_fast
1343                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1344               goto trace00;
1345             }
1346
1347           key0.addr = ip0->dst_address;
1348           key0.port = udp0->dst_port;
1349           key0.fib_index = rx_fib_index0;
1350
1351           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1352             {
1353               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1354               goto trace00;
1355             }
1356
1357           new_addr0 = sm0.addr.as_u32;
1358           new_port0 = sm0.port;
1359           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1360           old_addr0 = ip0->dst_address.as_u32;
1361           ip0->dst_address.as_u32 = new_addr0;
1362
1363           sum0 = ip0->checksum;
1364           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1365                                  ip4_header_t,
1366                                  dst_address /* changed member */);
1367           ip0->checksum = ip_csum_fold (sum0);
1368
1369           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1370             {
1371                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1372                 {
1373                   old_port0 = tcp0->dst_port;
1374                   tcp0->dst_port = new_port0;
1375
1376                   sum0 = tcp0->checksum;
1377                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1378                                          ip4_header_t,
1379                                          dst_address /* changed member */);
1380
1381                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1382                                          ip4_header_t /* cheat */,
1383                                          length /* changed member */);
1384                   tcp0->checksum = ip_csum_fold(sum0);
1385                 }
1386               else
1387                 {
1388                   old_port0 = udp0->dst_port;
1389                   udp0->dst_port = new_port0;
1390                   udp0->checksum = 0;
1391                 }
1392             }
1393           else
1394             {
1395               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1396                 {
1397                   sum0 = tcp0->checksum;
1398                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1399                                          ip4_header_t,
1400                                          dst_address /* changed member */);
1401
1402                   tcp0->checksum = ip_csum_fold(sum0);
1403                 }
1404             }
1405
1406         trace00:
1407
1408           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1409                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1410             {
1411               snat_out2in_trace_t *t =
1412                  vlib_add_trace (vm, node, b0, sizeof (*t));
1413               t->sw_if_index = sw_if_index0;
1414               t->next_index = next0;
1415             }
1416
1417           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1418
1419           /* verify speculative enqueue, maybe switch current next frame */
1420           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1421                                            to_next, n_left_to_next,
1422                                            bi0, next0);
1423         }
1424
1425       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1426     }
1427
1428   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1429                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1430                                pkts_processed);
1431   return frame->n_vectors;
1432 }
1433
1434 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1435   .function = snat_out2in_fast_node_fn,
1436   .name = "snat-out2in-fast",
1437   .vector_size = sizeof (u32),
1438   .format_trace = format_snat_out2in_fast_trace,
1439   .type = VLIB_NODE_TYPE_INTERNAL,
1440
1441   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1442   .error_strings = snat_out2in_error_strings,
1443
1444   .runtime_data_bytes = sizeof (snat_runtime_t),
1445
1446   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1447
1448   /* edit / add dispositions here */
1449   .next_nodes = {
1450     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1451     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1452     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1453   },
1454 };
1455 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);