plugins/snat-plugin/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <snat/snat.h>
  25
  26 #include <vppinfra/hash.h>
  27 #include <vppinfra/error.h>
  28 #include <vppinfra/elog.h>
  29
  30 typedef struct {
  31   u32 sw_if_index;
  32   u32 next_index;
  33   u32 session_index;
  34   u32 is_slow_path;
  35 } snat_in2out_trace_t;
  36
  37 typedef struct {
  38   u32 next_worker_index;
  39   u8 do_handoff;
  40 } snat_in2out_worker_handoff_trace_t;
  41
  42 /* packet trace format function */
  43 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  44 {
  45   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  46   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  47   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  48   char * tag;
  49
  50   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  51
  52   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  53               t->sw_if_index, t->next_index, t->session_index);
  54
  55   return s;
  56 }
  57
  58 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  63
  64   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  65               t->sw_if_index, t->next_index);
  66
  67   return s;
  68 }
  69
  70 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  71 {
  72   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  73   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  74   snat_in2out_worker_handoff_trace_t * t =
  75     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  76   char * m;
  77
  78   m = t->do_handoff ? "next worker" : "same worker";
  79   s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  80
  81   return s;
  82 }
  83
  84 vlib_node_registration_t snat_in2out_node;
  85 vlib_node_registration_t snat_in2out_slowpath_node;
  86 vlib_node_registration_t snat_in2out_fast_node;
  87 vlib_node_registration_t snat_in2out_worker_handoff_node;
  88
  89 #define foreach_snat_in2out_error                       \
  90 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  91 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  92 _(OUT_OF_PORTS, "Out of ports")                         \
  93 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  94 _(BAD_ICMP_TYPE, "icmp type not echo-request")          \
  95 _(NO_TRANSLATION, "No translation")
  96
  97 typedef enum {
  98 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  99   foreach_snat_in2out_error
 100 #undef _
 101   SNAT_IN2OUT_N_ERROR,
 102 } snat_in2out_error_t;
 103
 104 static char * snat_in2out_error_strings[] = {
 105 #define _(sym,string) string,
 106   foreach_snat_in2out_error
 107 #undef _
 108 };
 109
 110 typedef enum {
 111   SNAT_IN2OUT_NEXT_LOOKUP,
 112   SNAT_IN2OUT_NEXT_DROP,
 113   SNAT_IN2OUT_NEXT_SLOW_PATH,
 114   SNAT_IN2OUT_N_NEXT,
 115 } snat_in2out_next_t;
 116
 117 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 118                       ip4_header_t * ip0,
 119                       u32 rx_fib_index0,
 120                       snat_session_key_t * key0,
 121                       snat_session_t ** sessionp,
 122                       vlib_node_runtime_t * node,
 123                       u32 next0,
 124                       u32 cpu_index)
 125 {
 126   snat_user_t *u;
 127   snat_user_key_t user_key;
 128   snat_session_t *s;
 129   clib_bihash_kv_8_8_t kv0, value0;
 130   u32 oldest_per_user_translation_list_index;
 131   dlist_elt_t * oldest_per_user_translation_list_elt;
 132   dlist_elt_t * per_user_translation_list_elt;
 133   dlist_elt_t * per_user_list_head_elt;
 134   u32 session_index;
 135   snat_session_key_t key1;
 136   u32 address_index = ~0;
 137   u32 outside_fib_index;
 138   uword * p;
 139   snat_static_mapping_key_t worker_by_out_key;
 140
 141   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 142   if (! p)
 143     {
 144       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 145       return SNAT_IN2OUT_NEXT_DROP;
 146     }
 147   outside_fib_index = p[0];
 148
 149   user_key.addr = ip0->src_address;
 150   user_key.fib_index = rx_fib_index0;
 151   kv0.key = user_key.as_u64;
 152
 153   /* Ever heard of the "user" = src ip4 address before? */
 154   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 155     {
 156       /* no, make a new one */
 157       pool_get (sm->per_thread_data[cpu_index].users, u);
 158       memset (u, 0, sizeof (*u));
 159       u->addr = ip0->src_address;
 160
 161       pool_get (sm->per_thread_data[cpu_index].list_pool, per_user_list_head_elt);
 162
 163       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 164         sm->per_thread_data[cpu_index].list_pool;
 165
 166       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 167                        u->sessions_per_user_list_head_index);
 168
 169       kv0.value = u - sm->per_thread_data[cpu_index].users;
 170
 171       /* add user */
 172       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 173     }
 174   else
 175     {
 176       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 177                              value0.value);
 178     }
 179
 180   /* Over quota? Recycle the least recently used dynamic translation */
 181   if (u->nsessions >= sm->max_translations_per_user)
 182     {
 183       /* Remove the oldest dynamic translation */
 184       do {
 185           oldest_per_user_translation_list_index =
 186             clib_dlist_remove_head (sm->per_thread_data[cpu_index].list_pool,
 187                                     u->sessions_per_user_list_head_index);
 188
 189           ASSERT (oldest_per_user_translation_list_index != ~0);
 190
 191           /* add it back to the end of the LRU list */
 192           clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 193                               u->sessions_per_user_list_head_index,
 194                               oldest_per_user_translation_list_index);
 195           /* Get the list element */
 196           oldest_per_user_translation_list_elt =
 197             pool_elt_at_index (sm->per_thread_data[cpu_index].list_pool,
 198                                oldest_per_user_translation_list_index);
 199
 200           /* Get the session index from the list element */
 201           session_index = oldest_per_user_translation_list_elt->value;
 202
 203           /* Get the session */
 204           s = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 205                                  session_index);
 206       } while (!snat_is_session_static (s));
 207
 208       /* Remove in2out, out2in keys */
 209       kv0.key = s->in2out.as_u64;
 210       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 211           clib_warning ("in2out key delete failed");
 212       kv0.key = s->out2in.as_u64;
 213       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 214           clib_warning ("out2in key delete failed");
 215
 216       snat_free_outside_address_and_port
 217         (sm, &s->out2in, s->outside_address_index);
 218       s->outside_address_index = ~0;
 219
 220       if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 221         {
 222           ASSERT(0);
 223
 224           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 225           return SNAT_IN2OUT_NEXT_DROP;
 226         }
 227       s->outside_address_index = address_index;
 228     }
 229   else
 230     {
 231       u8 static_mapping = 1;
 232
 233       /* First try to match static mapping by local address and port */
 234       if (snat_static_mapping_match (sm, *key0, &key1, 0))
 235         {
 236           static_mapping = 0;
 237           /* Try to create dynamic translation */
 238           if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 239             {
 240               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 241               return SNAT_IN2OUT_NEXT_DROP;
 242             }
 243         }
 244
 245       /* Create a new session */
 246       pool_get (sm->per_thread_data[cpu_index].sessions, s);
 247       memset (s, 0, sizeof (*s));
 248
 249       s->outside_address_index = address_index;
 250
 251       if (static_mapping)
 252         {
 253           u->nstaticsessions++;
 254           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 255         }
 256       else
 257         {
 258           u->nsessions++;
 259         }
 260
 261       /* Create list elts */
 262       pool_get (sm->per_thread_data[cpu_index].list_pool,
 263                 per_user_translation_list_elt);
 264       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 265                        per_user_translation_list_elt -
 266                        sm->per_thread_data[cpu_index].list_pool);
 267
 268       per_user_translation_list_elt->value =
 269         s - sm->per_thread_data[cpu_index].sessions;
 270       s->per_user_index = per_user_translation_list_elt -
 271                           sm->per_thread_data[cpu_index].list_pool;
 272       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 273
 274       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 275                           s->per_user_list_head_index,
 276                           per_user_translation_list_elt -
 277                           sm->per_thread_data[cpu_index].list_pool);
 278    }
 279
 280   s->in2out = *key0;
 281   s->out2in = key1;
 282   s->out2in.protocol = key0->protocol;
 283   s->out2in.fib_index = outside_fib_index;
 284   *sessionp = s;
 285
 286   /* Add to translation hashes */
 287   kv0.key = s->in2out.as_u64;
 288   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 289   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 290       clib_warning ("in2out key add failed");
 291
 292   kv0.key = s->out2in.as_u64;
 293   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 294
 295   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 296       clib_warning ("out2in key add failed");
 297
 298   /* Add to translated packets worker lookup */
 299   worker_by_out_key.addr = s->out2in.addr;
 300   worker_by_out_key.port = s->out2in.port;
 301   worker_by_out_key.fib_index = s->out2in.fib_index;
 302   kv0.key = worker_by_out_key.as_u64;
 303   kv0.value = cpu_index;
 304   clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 305   return next0;
 306 }
 307
 308 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 309                                          vlib_buffer_t * b0,
 310                                          ip4_header_t * ip0,
 311                                          icmp46_header_t * icmp0,
 312                                          u32 sw_if_index0,
 313                                          u32 rx_fib_index0,
 314                                          vlib_node_runtime_t * node,
 315                                          u32 next0,
 316                                          f64 now,
 317                                          u32 cpu_index)
 318 {
 319   snat_session_key_t key0;
 320   icmp_echo_header_t *echo0;
 321   clib_bihash_kv_8_8_t kv0, value0;
 322   snat_session_t * s0;
 323   u32 new_addr0, old_addr0;
 324   u16 old_id0, new_id0;
 325   ip_csum_t sum0;
 326   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 327
 328   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request))
 329     {
 330       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 331       return SNAT_IN2OUT_NEXT_DROP;
 332     }
 333
 334   echo0 = (icmp_echo_header_t *)(icmp0+1);
 335
 336   key0.addr = ip0->src_address;
 337   key0.port = echo0->identifier;
 338   key0.protocol = SNAT_PROTOCOL_ICMP;
 339   key0.fib_index = rx_fib_index0;
 340
 341   kv0.key = key0.as_u64;
 342
 343   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 344     {
 345       ip4_address_t * first_int_addr;
 346
 347       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 348         {
 349           first_int_addr =
 350             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 351                                          0 /* just want the address */);
 352           rt->cached_sw_if_index = sw_if_index0;
 353           rt->cached_ip4_address = first_int_addr->as_u32;
 354         }
 355
 356       /* Don't NAT packet aimed at the intfc address */
 357       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 358                                 rt->cached_ip4_address))
 359         return next0;
 360
 361       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 362                          &s0, node, next0, cpu_index);
 363
 364       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 365         return next0;
 366     }
 367   else
 368     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 369                             value0.value);
 370
 371   old_addr0 = ip0->src_address.as_u32;
 372   ip0->src_address = s0->out2in.addr;
 373   new_addr0 = ip0->src_address.as_u32;
 374   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 375
 376   sum0 = ip0->checksum;
 377   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 378                          ip4_header_t,
 379                          src_address /* changed member */);
 380   ip0->checksum = ip_csum_fold (sum0);
 381
 382   old_id0 = echo0->identifier;
 383   new_id0 = s0->out2in.port;
 384   echo0->identifier = new_id0;
 385
 386   sum0 = icmp0->checksum;
 387   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 388                          identifier);
 389   icmp0->checksum = ip_csum_fold (sum0);
 390
 391   /* Accounting */
 392   s0->last_heard = now;
 393   s0->total_pkts++;
 394   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 395   /* Per-user LRU list maintenance for dynamic translations */
 396   if (!snat_is_session_static (s0))
 397     {
 398       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 399                          s0->per_user_index);
 400       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 401                           s0->per_user_list_head_index,
 402                           s0->per_user_index);
 403     }
 404
 405   return next0;
 406 }
 407
 408 static inline uword
 409 snat_in2out_node_fn_inline (vlib_main_t * vm,
 410                             vlib_node_runtime_t * node,
 411                             vlib_frame_t * frame, int is_slow_path)
 412 {
 413   u32 n_left_from, * from, * to_next;
 414   snat_in2out_next_t next_index;
 415   u32 pkts_processed = 0;
 416   snat_main_t * sm = &snat_main;
 417   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 418   f64 now = vlib_time_now (vm);
 419   u32 stats_node_index;
 420   u32 cpu_index = os_get_cpu_number ();
 421
 422   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 423     snat_in2out_node.index;
 424
 425   from = vlib_frame_vector_args (frame);
 426   n_left_from = frame->n_vectors;
 427   next_index = node->cached_next_index;
 428
 429   while (n_left_from > 0)
 430     {
 431       u32 n_left_to_next;
 432
 433       vlib_get_next_frame (vm, node, next_index,
 434                            to_next, n_left_to_next);
 435
 436       while (n_left_from >= 4 && n_left_to_next >= 2)
 437         {
 438           u32 bi0, bi1;
 439           vlib_buffer_t * b0, * b1;
 440           u32 next0, next1;
 441           u32 sw_if_index0, sw_if_index1;
 442           ip4_header_t * ip0, * ip1;
 443           ip_csum_t sum0, sum1;
 444           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 445           u16 old_port0, new_port0, old_port1, new_port1;
 446           udp_header_t * udp0, * udp1;
 447           tcp_header_t * tcp0, * tcp1;
 448           icmp46_header_t * icmp0, * icmp1;
 449           snat_session_key_t key0, key1;
 450           u32 rx_fib_index0, rx_fib_index1;
 451           u32 proto0, proto1;
 452           snat_session_t * s0 = 0, * s1 = 0;
 453           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 454
 455           /* Prefetch next iteration. */
 456           {
 457             vlib_buffer_t * p2, * p3;
 458
 459             p2 = vlib_get_buffer (vm, from[2]);
 460             p3 = vlib_get_buffer (vm, from[3]);
 461
 462             vlib_prefetch_buffer_header (p2, LOAD);
 463             vlib_prefetch_buffer_header (p3, LOAD);
 464
 465             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 466             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 467           }
 468
 469           /* speculatively enqueue b0 and b1 to the current next frame */
 470           to_next[0] = bi0 = from[0];
 471           to_next[1] = bi1 = from[1];
 472           from += 2;
 473           to_next += 2;
 474           n_left_from -= 2;
 475           n_left_to_next -= 2;
 476
 477           b0 = vlib_get_buffer (vm, bi0);
 478           b1 = vlib_get_buffer (vm, bi1);
 479
 480           ip0 = vlib_buffer_get_current (b0);
 481           udp0 = ip4_next_header (ip0);
 482           tcp0 = (tcp_header_t *) udp0;
 483           icmp0 = (icmp46_header_t *) udp0;
 484
 485           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 486           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 487                                    sw_if_index0);
 488
 489           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 490
 491           proto0 = ~0;
 492           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 493             ? SNAT_PROTOCOL_UDP : proto0;
 494           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 495             ? SNAT_PROTOCOL_TCP : proto0;
 496           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 497             ? SNAT_PROTOCOL_ICMP : proto0;
 498
 499           /* Next configured feature, probably ip4-lookup */
 500           if (is_slow_path)
 501             {
 502               if (PREDICT_FALSE (proto0 == ~0))
 503                 goto trace00;
 504
 505               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 506                 {
 507                   next0 = icmp_in2out_slow_path
 508                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 509                      node, next0, now, cpu_index);
 510                   goto trace00;
 511                 }
 512             }
 513           else
 514             {
 515               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 516                 {
 517                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 518                   goto trace00;
 519                 }
 520             }
 521
 522           key0.addr = ip0->src_address;
 523           key0.port = udp0->src_port;
 524           key0.protocol = proto0;
 525           key0.fib_index = rx_fib_index0;
 526
 527           kv0.key = key0.as_u64;
 528
 529           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
 530             {
 531               if (is_slow_path)
 532                 {
 533                   ip4_address_t * first_int_addr;
 534
 535                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 536                     {
 537                       first_int_addr =
 538                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 539                                                      0 /* just want the address */);
 540                       rt->cached_sw_if_index = sw_if_index0;
 541                       rt->cached_ip4_address = first_int_addr->as_u32;
 542                     }
 543
 544                   /* Don't NAT packet aimed at the intfc address */
 545                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 546                                     rt->cached_ip4_address))
 547                     goto trace00;
 548
 549                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 550                                      &s0, node, next0, cpu_index);
 551                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 552                     goto trace00;
 553                 }
 554               else
 555                 {
 556                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 557                   goto trace00;
 558                 }
 559             }
 560           else
 561             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 562                                     value0.value);
 563
 564           old_addr0 = ip0->src_address.as_u32;
 565           ip0->src_address = s0->out2in.addr;
 566           new_addr0 = ip0->src_address.as_u32;
 567           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 568
 569           sum0 = ip0->checksum;
 570           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 571                                  ip4_header_t,
 572                                  src_address /* changed member */);
 573           ip0->checksum = ip_csum_fold (sum0);
 574
 575           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 576             {
 577               old_port0 = tcp0->ports.src;
 578               tcp0->ports.src = s0->out2in.port;
 579               new_port0 = tcp0->ports.src;
 580
 581               sum0 = tcp0->checksum;
 582               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 583                                      ip4_header_t,
 584                                      dst_address /* changed member */);
 585               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 586                                      ip4_header_t /* cheat */,
 587                                      length /* changed member */);
 588               tcp0->checksum = ip_csum_fold(sum0);
 589             }
 590           else
 591             {
 592               old_port0 = udp0->src_port;
 593               udp0->src_port = s0->out2in.port;
 594               udp0->checksum = 0;
 595             }
 596
 597           /* Accounting */
 598           s0->last_heard = now;
 599           s0->total_pkts++;
 600           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 601           /* Per-user LRU list maintenance for dynamic translation */
 602           if (!snat_is_session_static (s0))
 603             {
 604               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 605                                  s0->per_user_index);
 606               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 607                                   s0->per_user_list_head_index,
 608                                   s0->per_user_index);
 609             }
 610         trace00:
 611
 612           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 613                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 614             {
 615               snat_in2out_trace_t *t =
 616                  vlib_add_trace (vm, node, b0, sizeof (*t));
 617               t->is_slow_path = is_slow_path;
 618               t->sw_if_index = sw_if_index0;
 619               t->next_index = next0;
 620                   t->session_index = ~0;
 621               if (s0)
 622                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 623             }
 624
 625           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 626
 627           ip1 = vlib_buffer_get_current (b1);
 628           udp1 = ip4_next_header (ip1);
 629           tcp1 = (tcp_header_t *) udp1;
 630           icmp1 = (icmp46_header_t *) udp1;
 631
 632           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 633           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 634                                    sw_if_index1);
 635
 636           proto1 = ~0;
 637           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 638             ? SNAT_PROTOCOL_UDP : proto1;
 639           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 640             ? SNAT_PROTOCOL_TCP : proto1;
 641           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 642             ? SNAT_PROTOCOL_ICMP : proto1;
 643
 644           /* Next configured feature, probably ip4-lookup */
 645           if (is_slow_path)
 646             {
 647               if (PREDICT_FALSE (proto1 == ~0))
 648                 goto trace01;
 649
 650               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 651                 {
 652                   next1 = icmp_in2out_slow_path
 653                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 654                      next1, now, cpu_index);
 655                   goto trace01;
 656                 }
 657             }
 658           else
 659             {
 660               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
 661                 {
 662                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 663                   goto trace01;
 664                 }
 665             }
 666
 667           key1.addr = ip1->src_address;
 668           key1.port = udp1->src_port;
 669           key1.protocol = proto1;
 670           key1.fib_index = rx_fib_index1;
 671
 672           kv1.key = key1.as_u64;
 673
 674             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
 675             {
 676               if (is_slow_path)
 677                 {
 678                   ip4_address_t * first_int_addr;
 679
 680                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index1))
 681                     {
 682                       first_int_addr =
 683                         ip4_interface_first_address (sm->ip4_main, sw_if_index1,
 684                                                      0 /* just want the address */);
 685                       rt->cached_sw_if_index = sw_if_index1;
 686                       rt->cached_ip4_address = first_int_addr->as_u32;
 687                     }
 688
 689                   /* Don't NAT packet aimed at the intfc address */
 690                   if (PREDICT_FALSE(ip1->dst_address.as_u32 ==
 691                                     rt->cached_ip4_address))
 692                     goto trace01;
 693
 694                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
 695                                      &s1, node, next1, cpu_index);
 696                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
 697                     goto trace01;
 698                 }
 699               else
 700                 {
 701                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 702                   goto trace01;
 703                 }
 704             }
 705           else
 706             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 707                                     value1.value);
 708
 709           old_addr1 = ip1->src_address.as_u32;
 710           ip1->src_address = s1->out2in.addr;
 711           new_addr1 = ip1->src_address.as_u32;
 712           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
 713
 714           sum1 = ip1->checksum;
 715           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 716                                  ip4_header_t,
 717                                  src_address /* changed member */);
 718           ip1->checksum = ip_csum_fold (sum1);
 719
 720           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 721             {
 722               old_port1 = tcp1->ports.src;
 723               tcp1->ports.src = s1->out2in.port;
 724               new_port1 = tcp1->ports.src;
 725
 726               sum1 = tcp1->checksum;
 727               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 728                                      ip4_header_t,
 729                                      dst_address /* changed member */);
 730               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 731                                      ip4_header_t /* cheat */,
 732                                      length /* changed member */);
 733               tcp1->checksum = ip_csum_fold(sum1);
 734             }
 735           else
 736             {
 737               old_port1 = udp1->src_port;
 738               udp1->src_port = s1->out2in.port;
 739               udp1->checksum = 0;
 740             }
 741
 742           /* Accounting */
 743           s1->last_heard = now;
 744           s1->total_pkts++;
 745           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 746           /* Per-user LRU list maintenance for dynamic translation */
 747           if (!snat_is_session_static (s1))
 748             {
 749               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 750                                  s1->per_user_index);
 751               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 752                                   s1->per_user_list_head_index,
 753                                   s1->per_user_index);
 754             }
 755         trace01:
 756
 757           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 758                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 759             {
 760               snat_in2out_trace_t *t =
 761                  vlib_add_trace (vm, node, b1, sizeof (*t));
 762               t->sw_if_index = sw_if_index1;
 763               t->next_index = next1;
 764               t->session_index = ~0;
 765               if (s1)
 766                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 767             }
 768
 769           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
 770
 771           /* verify speculative enqueues, maybe switch current next frame */
 772           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 773                                            to_next, n_left_to_next,
 774                                            bi0, bi1, next0, next1);
 775         }
 776
 777       while (n_left_from > 0 && n_left_to_next > 0)
 778         {
 779           u32 bi0;
 780           vlib_buffer_t * b0;
 781           u32 next0;
 782           u32 sw_if_index0;
 783           ip4_header_t * ip0;
 784           ip_csum_t sum0;
 785           u32 new_addr0, old_addr0;
 786           u16 old_port0, new_port0;
 787           udp_header_t * udp0;
 788           tcp_header_t * tcp0;
 789           icmp46_header_t * icmp0;
 790           snat_session_key_t key0;
 791           u32 rx_fib_index0;
 792           u32 proto0;
 793           snat_session_t * s0 = 0;
 794           clib_bihash_kv_8_8_t kv0, value0;
 795
 796           /* speculatively enqueue b0 to the current next frame */
 797           bi0 = from[0];
 798           to_next[0] = bi0;
 799           from += 1;
 800           to_next += 1;
 801           n_left_from -= 1;
 802           n_left_to_next -= 1;
 803
 804           b0 = vlib_get_buffer (vm, bi0);
 805           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
 806
 807           ip0 = vlib_buffer_get_current (b0);
 808           udp0 = ip4_next_header (ip0);
 809           tcp0 = (tcp_header_t *) udp0;
 810           icmp0 = (icmp46_header_t *) udp0;
 811
 812           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 813           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 814                                    sw_if_index0);
 815
 816           proto0 = ~0;
 817           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 818             ? SNAT_PROTOCOL_UDP : proto0;
 819           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 820             ? SNAT_PROTOCOL_TCP : proto0;
 821           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 822             ? SNAT_PROTOCOL_ICMP : proto0;
 823
 824           /* Next configured feature, probably ip4-lookup */
 825           if (is_slow_path)
 826             {
 827               if (PREDICT_FALSE (proto0 == ~0))
 828                 goto trace0;
 829
 830               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 831                 {
 832                   next0 = icmp_in2out_slow_path
 833                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 834                      next0, now, cpu_index);
 835                   goto trace0;
 836                 }
 837             }
 838           else
 839             {
 840               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 841                 {
 842                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 843                   goto trace0;
 844                 }
 845             }
 846
 847           key0.addr = ip0->src_address;
 848           key0.port = udp0->src_port;
 849           key0.protocol = proto0;
 850           key0.fib_index = rx_fib_index0;
 851
 852           kv0.key = key0.as_u64;
 853
 854           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 855             {
 856               if (is_slow_path)
 857                 {
 858                   ip4_address_t * first_int_addr;
 859
 860                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 861                     {
 862                       first_int_addr =
 863                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 864                                                      0 /* just want the address */);
 865                       rt->cached_sw_if_index = sw_if_index0;
 866                       rt->cached_ip4_address = first_int_addr->as_u32;
 867                     }
 868
 869                   /* Don't NAT packet aimed at the intfc address */
 870                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 871                                     rt->cached_ip4_address))
 872                     goto trace0;
 873
 874                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 875                                      &s0, node, next0, cpu_index);
 876                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 877                     goto trace0;
 878                 }
 879               else
 880                 {
 881                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 882                   goto trace0;
 883                 }
 884             }
 885           else
 886             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 887                                     value0.value);
 888
 889           old_addr0 = ip0->src_address.as_u32;
 890           ip0->src_address = s0->out2in.addr;
 891           new_addr0 = ip0->src_address.as_u32;
 892           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 893
 894           sum0 = ip0->checksum;
 895           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 896                                  ip4_header_t,
 897                                  src_address /* changed member */);
 898           ip0->checksum = ip_csum_fold (sum0);
 899
 900           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 901             {
 902               old_port0 = tcp0->ports.src;
 903               tcp0->ports.src = s0->out2in.port;
 904               new_port0 = tcp0->ports.src;
 905
 906               sum0 = tcp0->checksum;
 907               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 908                                      ip4_header_t,
 909                                      dst_address /* changed member */);
 910               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 911                                      ip4_header_t /* cheat */,
 912                                      length /* changed member */);
 913               tcp0->checksum = ip_csum_fold(sum0);
 914             }
 915           else
 916             {
 917               old_port0 = udp0->src_port;
 918               udp0->src_port = s0->out2in.port;
 919               udp0->checksum = 0;
 920             }
 921
 922           /* Accounting */
 923           s0->last_heard = now;
 924           s0->total_pkts++;
 925           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 926           /* Per-user LRU list maintenance for dynamic translation */
 927           if (!snat_is_session_static (s0))
 928             {
 929               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 930                                  s0->per_user_index);
 931               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 932                                   s0->per_user_list_head_index,
 933                                   s0->per_user_index);
 934             }
 935
 936         trace0:
 937           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 938                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 939             {
 940               snat_in2out_trace_t *t =
 941                  vlib_add_trace (vm, node, b0, sizeof (*t));
 942               t->is_slow_path = is_slow_path;
 943               t->sw_if_index = sw_if_index0;
 944               t->next_index = next0;
 945                   t->session_index = ~0;
 946               if (s0)
 947                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 948             }
 949
 950           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 951
 952           /* verify speculative enqueue, maybe switch current next frame */
 953           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 954                                            to_next, n_left_to_next,
 955                                            bi0, next0);
 956         }
 957
 958       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 959     }
 960
 961   vlib_node_increment_counter (vm, stats_node_index,
 962                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
 963                                pkts_processed);
 964   return frame->n_vectors;
 965 }
 966
 967 static uword
 968 snat_in2out_fast_path_fn (vlib_main_t * vm,
 969                           vlib_node_runtime_t * node,
 970                           vlib_frame_t * frame)
 971 {
 972   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
 973 }
 974
 975 VLIB_REGISTER_NODE (snat_in2out_node) = {
 976   .function = snat_in2out_fast_path_fn,
 977   .name = "snat-in2out",
 978   .vector_size = sizeof (u32),
 979   .format_trace = format_snat_in2out_trace,
 980   .type = VLIB_NODE_TYPE_INTERNAL,
 981
 982   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 983   .error_strings = snat_in2out_error_strings,
 984
 985   .runtime_data_bytes = sizeof (snat_runtime_t),
 986
 987   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 988
 989   /* edit / add dispositions here */
 990   .next_nodes = {
 991     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 992     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 993     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 994   },
 995 };
 996
 997 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
 998
 999 static uword
1000 snat_in2out_slow_path_fn (vlib_main_t * vm,
1001                           vlib_node_runtime_t * node,
1002                           vlib_frame_t * frame)
1003 {
1004   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
1005 }
1006
1007 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1008   .function = snat_in2out_slow_path_fn,
1009   .name = "snat-in2out-slowpath",
1010   .vector_size = sizeof (u32),
1011   .format_trace = format_snat_in2out_trace,
1012   .type = VLIB_NODE_TYPE_INTERNAL,
1013
1014   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1015   .error_strings = snat_in2out_error_strings,
1016
1017   .runtime_data_bytes = sizeof (snat_runtime_t),
1018
1019   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1020
1021   /* edit / add dispositions here */
1022   .next_nodes = {
1023     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1024     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1025     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1026   },
1027 };
1028
1029 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1030
1031 static uword
1032 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
1033                                vlib_node_runtime_t * node,
1034                                vlib_frame_t * frame)
1035 {
1036   snat_main_t *sm = &snat_main;
1037   vlib_thread_main_t *tm = vlib_get_thread_main ();
1038   u32 n_left_from, *from, *to_next = 0;
1039   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1040   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1041     = 0;
1042   vlib_frame_queue_elt_t *hf = 0;
1043   vlib_frame_t *f = 0;
1044   int i;
1045   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1046   u32 next_worker_index = 0;
1047   u32 current_worker_index = ~0;
1048   u32 cpu_index = os_get_cpu_number ();
1049
1050   ASSERT (vec_len (sm->workers));
1051
1052   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1053     {
1054       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1055
1056       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1057                                sm->first_worker_index + sm->num_workers - 1,
1058                                (vlib_frame_queue_t *) (~0));
1059     }
1060
1061   from = vlib_frame_vector_args (frame);
1062   n_left_from = frame->n_vectors;
1063
1064   while (n_left_from > 0)
1065     {
1066       u32 bi0;
1067       vlib_buffer_t *b0;
1068       u32 sw_if_index0;
1069       u32 rx_fib_index0;
1070       ip4_header_t * ip0;
1071       snat_user_key_t key0;
1072       clib_bihash_kv_8_8_t kv0, value0;
1073       u8 do_handoff;
1074
1075       bi0 = from[0];
1076       from += 1;
1077       n_left_from -= 1;
1078
1079       b0 = vlib_get_buffer (vm, bi0);
1080
1081       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1082       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1083
1084       ip0 = vlib_buffer_get_current (b0);
1085
1086       key0.addr = ip0->src_address;
1087       key0.fib_index = rx_fib_index0;
1088
1089       kv0.key = key0.as_u64;
1090
1091       /* Ever heard of of the "user" before? */
1092       if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv0, &value0))
1093         {
1094           /* No, assign next available worker (RR) */
1095           next_worker_index = sm->first_worker_index +
1096             sm->workers[sm->next_worker++ % vec_len (sm->workers)];
1097
1098           /* add non-traslated packets worker lookup */
1099           kv0.value = next_worker_index;
1100           clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
1101         }
1102       else
1103         next_worker_index = value0.value;
1104
1105       if (PREDICT_FALSE (next_worker_index != cpu_index))
1106         {
1107           do_handoff = 1;
1108
1109           if (next_worker_index != current_worker_index)
1110             {
1111               if (hf)
1112                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1113
1114               hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index,
1115                                                       next_worker_index,
1116                                                       handoff_queue_elt_by_worker_index);
1117
1118               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1119               to_next_worker = &hf->buffer_index[hf->n_vectors];
1120               current_worker_index = next_worker_index;
1121             }
1122
1123           /* enqueue to correct worker thread */
1124           to_next_worker[0] = bi0;
1125           to_next_worker++;
1126           n_left_to_next_worker--;
1127
1128           if (n_left_to_next_worker == 0)
1129             {
1130               hf->n_vectors = VLIB_FRAME_SIZE;
1131               vlib_put_frame_queue_elt (hf);
1132               current_worker_index = ~0;
1133               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1134               hf = 0;
1135             }
1136         }
1137       else
1138         {
1139           do_handoff = 0;
1140           /* if this is 1st frame */
1141           if (!f)
1142             {
1143               f = vlib_get_frame_to_node (vm, snat_in2out_node.index);
1144               to_next = vlib_frame_vector_args (f);
1145             }
1146
1147           to_next[0] = bi0;
1148           to_next += 1;
1149           f->n_vectors++;
1150         }
1151
1152       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1153                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1154         {
1155           snat_in2out_worker_handoff_trace_t *t =
1156             vlib_add_trace (vm, node, b0, sizeof (*t));
1157           t->next_worker_index = next_worker_index;
1158           t->do_handoff = do_handoff;
1159         }
1160     }
1161
1162   if (f)
1163     vlib_put_frame_to_node (vm, snat_in2out_node.index, f);
1164
1165   if (hf)
1166     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1167
1168   /* Ship frames to the worker nodes */
1169   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1170     {
1171       if (handoff_queue_elt_by_worker_index[i])
1172         {
1173           hf = handoff_queue_elt_by_worker_index[i];
1174           /*
1175            * It works better to let the handoff node
1176            * rate-adapt, always ship the handoff queue element.
1177            */
1178           if (1 || hf->n_vectors == hf->last_n_vectors)
1179             {
1180               vlib_put_frame_queue_elt (hf);
1181               handoff_queue_elt_by_worker_index[i] = 0;
1182             }
1183           else
1184             hf->last_n_vectors = hf->n_vectors;
1185         }
1186       congested_handoff_queue_by_worker_index[i] =
1187         (vlib_frame_queue_t *) (~0);
1188     }
1189   hf = 0;
1190   current_worker_index = ~0;
1191   return frame->n_vectors;
1192 }
1193
1194 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
1195   .function = snat_in2out_worker_handoff_fn,
1196   .name = "snat-in2out-worker-handoff",
1197   .vector_size = sizeof (u32),
1198   .format_trace = format_snat_in2out_worker_handoff_trace,
1199   .type = VLIB_NODE_TYPE_INTERNAL,
1200
1201   .n_next_nodes = 1,
1202
1203   .next_nodes = {
1204     [0] = "error-drop",
1205   },
1206 };
1207
1208 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn);
1209
1210 static inline u32 icmp_in2out_static_map (snat_main_t *sm,
1211                                           vlib_buffer_t * b0,
1212                                           ip4_header_t * ip0,
1213                                           icmp46_header_t * icmp0,
1214                                           u32 sw_if_index0,
1215                                           vlib_node_runtime_t * node,
1216                                           u32 next0,
1217                                           u32 rx_fib_index0)
1218 {
1219   snat_session_key_t key0, sm0;
1220   icmp_echo_header_t *echo0;
1221   u32 new_addr0, old_addr0;
1222   u16 old_id0, new_id0;
1223   ip_csum_t sum0;
1224   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1225
1226   echo0 = (icmp_echo_header_t *)(icmp0+1);
1227
1228   key0.addr = ip0->src_address;
1229   key0.port = echo0->identifier;
1230   key0.fib_index = rx_fib_index0;
1231
1232   if (snat_static_mapping_match(sm, key0, &sm0, 0))
1233     {
1234       ip4_address_t * first_int_addr;
1235
1236       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1237         {
1238           first_int_addr =
1239             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1240                                          0 /* just want the address */);
1241           rt->cached_sw_if_index = sw_if_index0;
1242           rt->cached_ip4_address = first_int_addr->as_u32;
1243         }
1244
1245       /* Don't NAT packet aimed at the intfc address */
1246       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1247                                 rt->cached_ip4_address))
1248         return next0;
1249
1250       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1251       return SNAT_IN2OUT_NEXT_DROP;
1252     }
1253
1254   new_addr0 = sm0.addr.as_u32;
1255   new_id0 = sm0.port;
1256   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1257   old_addr0 = ip0->src_address.as_u32;
1258   ip0->src_address.as_u32 = new_addr0;
1259
1260   sum0 = ip0->checksum;
1261   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1262                          ip4_header_t,
1263                          src_address /* changed member */);
1264   ip0->checksum = ip_csum_fold (sum0);
1265
1266   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1267     {
1268       old_id0 = echo0->identifier;
1269       echo0->identifier = new_id0;
1270
1271       sum0 = icmp0->checksum;
1272       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1273                              identifier);
1274       icmp0->checksum = ip_csum_fold (sum0);
1275     }
1276
1277   return next0;
1278 }
1279
1280 static uword
1281 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1282                                 vlib_node_runtime_t * node,
1283                                 vlib_frame_t * frame)
1284 {
1285   u32 n_left_from, * from, * to_next;
1286   snat_in2out_next_t next_index;
1287   u32 pkts_processed = 0;
1288   snat_main_t * sm = &snat_main;
1289   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1290   u32 stats_node_index;
1291
1292   stats_node_index = snat_in2out_fast_node.index;
1293
1294   from = vlib_frame_vector_args (frame);
1295   n_left_from = frame->n_vectors;
1296   next_index = node->cached_next_index;
1297
1298   while (n_left_from > 0)
1299     {
1300       u32 n_left_to_next;
1301
1302       vlib_get_next_frame (vm, node, next_index,
1303                            to_next, n_left_to_next);
1304
1305       while (n_left_from > 0 && n_left_to_next > 0)
1306         {
1307           u32 bi0;
1308           vlib_buffer_t * b0;
1309           u32 next0;
1310           u32 sw_if_index0;
1311           ip4_header_t * ip0;
1312           ip_csum_t sum0;
1313           u32 new_addr0, old_addr0;
1314           u16 old_port0, new_port0;
1315           udp_header_t * udp0;
1316           tcp_header_t * tcp0;
1317           icmp46_header_t * icmp0;
1318           snat_session_key_t key0, sm0;
1319           u32 proto0;
1320           u32 rx_fib_index0;
1321
1322           /* speculatively enqueue b0 to the current next frame */
1323           bi0 = from[0];
1324           to_next[0] = bi0;
1325           from += 1;
1326           to_next += 1;
1327           n_left_from -= 1;
1328           n_left_to_next -= 1;
1329
1330           b0 = vlib_get_buffer (vm, bi0);
1331           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1332
1333           ip0 = vlib_buffer_get_current (b0);
1334           udp0 = ip4_next_header (ip0);
1335           tcp0 = (tcp_header_t *) udp0;
1336           icmp0 = (icmp46_header_t *) udp0;
1337
1338           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1339           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1340
1341           proto0 = ~0;
1342           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1343             ? SNAT_PROTOCOL_UDP : proto0;
1344           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1345             ? SNAT_PROTOCOL_TCP : proto0;
1346           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1347             ? SNAT_PROTOCOL_ICMP : proto0;
1348
1349           if (PREDICT_FALSE (proto0 == ~0))
1350               goto trace0;
1351
1352           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1353             {
1354               ip4_address_t * first_int_addr;
1355
1356               if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1357                 {
1358                   first_int_addr =
1359                     ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1360                                                  0 /* just want the address */);
1361                   rt->cached_sw_if_index = sw_if_index0;
1362                   rt->cached_ip4_address = first_int_addr->as_u32;
1363                 }
1364
1365               /* Don't NAT packet aimed at the intfc address */
1366               if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1367                                 rt->cached_ip4_address))
1368                 goto trace0;
1369
1370               next0 = icmp_in2out_static_map
1371                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1372               goto trace0;
1373             }
1374
1375           key0.addr = ip0->src_address;
1376           key0.port = udp0->src_port;
1377           key0.fib_index = rx_fib_index0;
1378
1379           if (snat_static_mapping_match(sm, key0, &sm0, 0))
1380             {
1381               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1382               next0= SNAT_IN2OUT_NEXT_DROP;
1383               goto trace0;
1384             }
1385
1386           new_addr0 = sm0.addr.as_u32;
1387           new_port0 = sm0.port;
1388           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1389           old_addr0 = ip0->src_address.as_u32;
1390           ip0->src_address.as_u32 = new_addr0;
1391
1392           sum0 = ip0->checksum;
1393           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1394                                  ip4_header_t,
1395                                  src_address /* changed member */);
1396           ip0->checksum = ip_csum_fold (sum0);
1397
1398           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1399             {
1400               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1401                 {
1402                   old_port0 = tcp0->ports.src;
1403                   tcp0->ports.src = new_port0;
1404
1405                   sum0 = tcp0->checksum;
1406                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1407                                          ip4_header_t,
1408                                          dst_address /* changed member */);
1409                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1410                                          ip4_header_t /* cheat */,
1411                                          length /* changed member */);
1412                   tcp0->checksum = ip_csum_fold(sum0);
1413                 }
1414               else
1415                 {
1416                   old_port0 = udp0->src_port;
1417                   udp0->src_port = new_port0;
1418                   udp0->checksum = 0;
1419                 }
1420             }
1421           else
1422             {
1423               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1424                 {
1425                   sum0 = tcp0->checksum;
1426                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1427                                          ip4_header_t,
1428                                          dst_address /* changed member */);
1429                   tcp0->checksum = ip_csum_fold(sum0);
1430                 }
1431             }
1432
1433         trace0:
1434           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1435                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1436             {
1437               snat_in2out_trace_t *t =
1438                  vlib_add_trace (vm, node, b0, sizeof (*t));
1439               t->sw_if_index = sw_if_index0;
1440               t->next_index = next0;
1441             }
1442
1443           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1444
1445           /* verify speculative enqueue, maybe switch current next frame */
1446           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1447                                            to_next, n_left_to_next,
1448                                            bi0, next0);
1449         }
1450
1451       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1452     }
1453
1454   vlib_node_increment_counter (vm, stats_node_index,
1455                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1456                                pkts_processed);
1457   return frame->n_vectors;
1458 }
1459
1460
1461 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
1462   .function = snat_in2out_fast_static_map_fn,
1463   .name = "snat-in2out-fast",
1464   .vector_size = sizeof (u32),
1465   .format_trace = format_snat_in2out_fast_trace,
1466   .type = VLIB_NODE_TYPE_INTERNAL,
1467
1468   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1469   .error_strings = snat_in2out_error_strings,
1470
1471   .runtime_data_bytes = sizeof (snat_runtime_t),
1472
1473   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1474
1475   /* edit / add dispositions here */
1476   .next_nodes = {
1477     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1478     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1479     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1480   },
1481 };
1482
1483 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);