src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36 } snat_out2in_trace_t;
  37
  38 typedef struct {
  39   u32 next_worker_index;
  40   u8 do_handoff;
  41 } snat_out2in_worker_handoff_trace_t;
  42
  43 /* packet trace format function */
  44 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  45 {
  46   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  47   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  48   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  49
  50   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  51               t->sw_if_index, t->next_index, t->session_index);
  52   return s;
  53 }
  54
  55 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  56 {
  57   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  58   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  59   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  60
  61   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  62               t->sw_if_index, t->next_index);
  63   return s;
  64 }
  65
  66 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   snat_out2in_worker_handoff_trace_t * t =
  71     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  72   char * m;
  73
  74   m = t->do_handoff ? "next worker" : "same worker";
  75   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  76
  77   return s;
  78 }
  79
  80 vlib_node_registration_t snat_out2in_node;
  81 vlib_node_registration_t snat_out2in_fast_node;
  82 vlib_node_registration_t snat_out2in_worker_handoff_node;
  83
  84 #define foreach_snat_out2in_error                       \
  85 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  86 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  87 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  88 _(NO_TRANSLATION, "No translation")
  89
  90 typedef enum {
  91 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  92   foreach_snat_out2in_error
  93 #undef _
  94   SNAT_OUT2IN_N_ERROR,
  95 } snat_out2in_error_t;
  96
  97 static char * snat_out2in_error_strings[] = {
  98 #define _(sym,string) string,
  99   foreach_snat_out2in_error
 100 #undef _
 101 };
 102
 103 typedef enum {
 104   SNAT_OUT2IN_NEXT_DROP,
 105   SNAT_OUT2IN_NEXT_LOOKUP,
 106   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 107   SNAT_OUT2IN_N_NEXT,
 108 } snat_out2in_next_t;
 109
 110 /**
 111  * @brief Create session for static mapping.
 112  *
 113  * Create NAT session initiated by host from external network with static
 114  * mapping.
 115  *
 116  * @param sm     SNAT main.
 117  * @param b0     Vlib buffer.
 118  * @param in2out In2out SNAT session key.
 119  * @param out2in Out2in SNAT session key.
 120  * @param node   Vlib node.
 121  *
 122  * @returns SNAT session if successfully created otherwise 0.
 123  */
 124 static inline snat_session_t *
 125 create_session_for_static_mapping (snat_main_t *sm,
 126                                    vlib_buffer_t *b0,
 127                                    snat_session_key_t in2out,
 128                                    snat_session_key_t out2in,
 129                                    vlib_node_runtime_t * node,
 130                                    u32 cpu_index)
 131 {
 132   snat_user_t *u;
 133   snat_user_key_t user_key;
 134   snat_session_t *s;
 135   clib_bihash_kv_8_8_t kv0, value0;
 136   dlist_elt_t * per_user_translation_list_elt;
 137   dlist_elt_t * per_user_list_head_elt;
 138
 139   user_key.addr = in2out.addr;
 140   user_key.fib_index = in2out.fib_index;
 141   kv0.key = user_key.as_u64;
 142
 143   /* Ever heard of the "user" = inside ip4 address before? */
 144   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 145     {
 146       /* no, make a new one */
 147       pool_get (sm->per_thread_data[cpu_index].users, u);
 148       memset (u, 0, sizeof (*u));
 149       u->addr = in2out.addr;
 150
 151       pool_get (sm->per_thread_data[cpu_index].list_pool,
 152                 per_user_list_head_elt);
 153
 154       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 155         sm->per_thread_data[cpu_index].list_pool;
 156
 157       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 158                        u->sessions_per_user_list_head_index);
 159
 160       kv0.value = u - sm->per_thread_data[cpu_index].users;
 161
 162       /* add user */
 163       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 164
 165       /* add non-traslated packets worker lookup */
 166       kv0.value = cpu_index;
 167       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 168     }
 169   else
 170     {
 171       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 172                              value0.value);
 173     }
 174
 175   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 176   memset (s, 0, sizeof (*s));
 177
 178   s->outside_address_index = ~0;
 179   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 180   u->nstaticsessions++;
 181
 182   /* Create list elts */
 183   pool_get (sm->per_thread_data[cpu_index].list_pool,
 184             per_user_translation_list_elt);
 185   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 186                    per_user_translation_list_elt -
 187                    sm->per_thread_data[cpu_index].list_pool);
 188
 189   per_user_translation_list_elt->value =
 190     s - sm->per_thread_data[cpu_index].sessions;
 191   s->per_user_index =
 192     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 193   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 194
 195   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 196                       s->per_user_list_head_index,
 197                       per_user_translation_list_elt -
 198                       sm->per_thread_data[cpu_index].list_pool);
 199
 200   s->in2out = in2out;
 201   s->out2in = out2in;
 202   s->in2out.protocol = out2in.protocol;
 203
 204   /* Add to translation hashes */
 205   kv0.key = s->in2out.as_u64;
 206   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 207   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 208       clib_warning ("in2out key add failed");
 209
 210   kv0.key = s->out2in.as_u64;
 211   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 212
 213   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 214       clib_warning ("out2in key add failed");
 215
 216   /* log NAT event */
 217   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 218                                       s->out2in.addr.as_u32,
 219                                       s->in2out.protocol,
 220                                       s->in2out.port,
 221                                       s->out2in.port,
 222                                       s->in2out.fib_index);
 223    return s;
 224 }
 225
 226 typedef struct {
 227   u16 src_port, dst_port;
 228 } tcp_udp_header_t;
 229
 230 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 231                                          vlib_buffer_t * b0,
 232                                          ip4_header_t * ip0,
 233                                          icmp46_header_t * icmp0,
 234                                          u32 sw_if_index0,
 235                                          u32 rx_fib_index0,
 236                                          vlib_node_runtime_t * node,
 237                                          u32 next0, f64 now,
 238                                          u32 cpu_index,
 239                                          snat_session_t ** p_s0)
 240 {
 241   snat_session_key_t key0, sm0;
 242   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 243   ip4_header_t *inner_ip0 = 0;
 244   void *l4_header = 0;
 245   icmp46_header_t *inner_icmp0;
 246   clib_bihash_kv_8_8_t kv0, value0;
 247   snat_session_t * s0 = 0;
 248   u32 new_addr0, old_addr0;
 249   u16 old_id0, new_id0;
 250   ip_csum_t sum0;
 251   u16 checksum0;
 252   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 253   u8 is_error_message = 0;
 254
 255   echo0 = (icmp_echo_header_t *)(icmp0+1);
 256
 257   key0.addr = ip0->dst_address;
 258   key0.fib_index = rx_fib_index0;
 259
 260   switch(icmp0->type)
 261     {
 262     case ICMP4_destination_unreachable:
 263     case ICMP4_time_exceeded:
 264     case ICMP4_parameter_problem:
 265     case ICMP4_source_quench:
 266     case ICMP4_redirect:
 267     case ICMP4_alternate_host_address:
 268       is_error_message = 1;
 269     }
 270
 271   if (!is_error_message)
 272     {
 273       key0.protocol = SNAT_PROTOCOL_ICMP;
 274       key0.port = echo0->identifier;
 275     }
 276   else
 277     {
 278       inner_ip0 = (ip4_header_t *)(echo0+1);
 279       l4_header = ip4_next_header (inner_ip0);
 280       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 281       switch (key0.protocol)
 282         {
 283         case SNAT_PROTOCOL_ICMP:
 284           inner_icmp0 = (icmp46_header_t*)l4_header;
 285           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 286           key0.port = inner_echo0->identifier;
 287           break;
 288         case SNAT_PROTOCOL_UDP:
 289         case SNAT_PROTOCOL_TCP:
 290           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 291           break;
 292         default:
 293           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 294           next0 = SNAT_OUT2IN_NEXT_DROP;
 295           goto out;
 296         }
 297     }
 298
 299   kv0.key = key0.as_u64;
 300
 301   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 302     {
 303       /* Try to match static mapping by external address and port,
 304          destination address and port in packet */
 305       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 306         {
 307            ip4_address_t * first_int_addr;
 308
 309           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 310             {
 311               first_int_addr =
 312                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 313                                              0 /* just want the address */);
 314               rt->cached_sw_if_index = sw_if_index0;
 315               if (first_int_addr)
 316                 rt->cached_ip4_address = first_int_addr->as_u32;
 317               else
 318                 rt->cached_ip4_address = 0;
 319             }
 320
 321           /* Don't NAT packet aimed at the intfc address */
 322           if (PREDICT_FALSE(ip0->dst_address.as_u32 == rt->cached_ip4_address))
 323             goto out;
 324
 325           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 326           next0 = SNAT_OUT2IN_NEXT_DROP;
 327           goto out;
 328         }
 329
 330       if (is_error_message)
 331         {
 332           next0 = SNAT_OUT2IN_NEXT_DROP;
 333           goto out;
 334         }
 335
 336       /* Create session initiated by host from external network */
 337       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 338                                              node, cpu_index);
 339
 340       if (!s0)
 341         {
 342           next0 = SNAT_OUT2IN_NEXT_DROP;
 343           goto out;
 344         }
 345     }
 346   else
 347     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 348                             value0.value);
 349
 350   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply && !is_error_message))
 351     {
 352       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 353       next0 = SNAT_OUT2IN_NEXT_DROP;
 354       goto out;
 355     }
 356
 357   sum0 = ip_incremental_checksum (0, icmp0,
 358                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 359   checksum0 = ~ip_csum_fold (sum0);
 360   if (checksum0 != 0 && checksum0 != 0xffff)
 361     {
 362       next0 = SNAT_OUT2IN_NEXT_DROP;
 363       goto out;
 364     }
 365
 366   old_addr0 = ip0->dst_address.as_u32;
 367   ip0->dst_address = s0->in2out.addr;
 368   new_addr0 = ip0->dst_address.as_u32;
 369   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 370
 371   sum0 = ip0->checksum;
 372   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 373                          dst_address /* changed member */);
 374   ip0->checksum = ip_csum_fold (sum0);
 375
 376   if (!is_error_message)
 377     {
 378       old_id0 = echo0->identifier;
 379       new_id0 = s0->in2out.port;
 380       echo0->identifier = new_id0;
 381
 382       sum0 = icmp0->checksum;
 383       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 384                              identifier /* changed member */);
 385       icmp0->checksum = ip_csum_fold (sum0);
 386     }
 387   else
 388     {
 389       if (!ip4_header_checksum_is_valid (inner_ip0))
 390         {
 391           next0 = SNAT_OUT2IN_NEXT_DROP;
 392           goto out;
 393         }
 394
 395       old_addr0 = inner_ip0->src_address.as_u32;
 396       inner_ip0->src_address = s0->in2out.addr;
 397       new_addr0 = inner_ip0->src_address.as_u32;
 398
 399       sum0 = icmp0->checksum;
 400       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 401                              src_address /* changed member */);
 402       icmp0->checksum = ip_csum_fold (sum0);
 403
 404       switch (key0.protocol)
 405         {
 406         case SNAT_PROTOCOL_ICMP:
 407           old_id0 = inner_echo0->identifier;
 408           new_id0 = s0->in2out.port;
 409           inner_echo0->identifier = new_id0;
 410
 411           sum0 = icmp0->checksum;
 412           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 413                                  identifier);
 414           icmp0->checksum = ip_csum_fold (sum0);
 415           break;
 416         case SNAT_PROTOCOL_UDP:
 417         case SNAT_PROTOCOL_TCP:
 418           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 419           new_id0 = s0->in2out.port;
 420           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 421
 422           sum0 = icmp0->checksum;
 423           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 424                                  src_port);
 425           icmp0->checksum = ip_csum_fold (sum0);
 426           break;
 427         default:
 428           ASSERT(0);
 429         }
 430     }
 431
 432   /* Accounting */
 433   s0->last_heard = now;
 434   s0->total_pkts++;
 435   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 436   /* Per-user LRU list maintenance for dynamic translation */
 437   if (!snat_is_session_static (s0))
 438     {
 439       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 440                          s0->per_user_index);
 441       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 442                           s0->per_user_list_head_index,
 443                           s0->per_user_index);
 444     }
 445
 446 out:
 447   *p_s0 = s0;
 448   return next0;
 449 }
 450
 451 static uword
 452 snat_out2in_node_fn (vlib_main_t * vm,
 453                   vlib_node_runtime_t * node,
 454                   vlib_frame_t * frame)
 455 {
 456   u32 n_left_from, * from, * to_next;
 457   snat_out2in_next_t next_index;
 458   u32 pkts_processed = 0;
 459   snat_main_t * sm = &snat_main;
 460   f64 now = vlib_time_now (vm);
 461   u32 cpu_index = os_get_cpu_number ();
 462
 463   from = vlib_frame_vector_args (frame);
 464   n_left_from = frame->n_vectors;
 465   next_index = node->cached_next_index;
 466
 467   while (n_left_from > 0)
 468     {
 469       u32 n_left_to_next;
 470
 471       vlib_get_next_frame (vm, node, next_index,
 472                            to_next, n_left_to_next);
 473
 474       while (n_left_from >= 4 && n_left_to_next >= 2)
 475         {
 476           u32 bi0, bi1;
 477           vlib_buffer_t * b0, * b1;
 478           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 479           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 480           u32 sw_if_index0, sw_if_index1;
 481           ip4_header_t * ip0, *ip1;
 482           ip_csum_t sum0, sum1;
 483           u32 new_addr0, old_addr0;
 484           u16 new_port0, old_port0;
 485           u32 new_addr1, old_addr1;
 486           u16 new_port1, old_port1;
 487           udp_header_t * udp0, * udp1;
 488           tcp_header_t * tcp0, * tcp1;
 489           icmp46_header_t * icmp0, * icmp1;
 490           snat_session_key_t key0, key1, sm0, sm1;
 491           u32 rx_fib_index0, rx_fib_index1;
 492           u32 proto0, proto1;
 493           snat_session_t * s0 = 0, * s1 = 0;
 494           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 495
 496           /* Prefetch next iteration. */
 497           {
 498             vlib_buffer_t * p2, * p3;
 499
 500             p2 = vlib_get_buffer (vm, from[2]);
 501             p3 = vlib_get_buffer (vm, from[3]);
 502
 503             vlib_prefetch_buffer_header (p2, LOAD);
 504             vlib_prefetch_buffer_header (p3, LOAD);
 505
 506             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 507             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 508           }
 509
 510           /* speculatively enqueue b0 and b1 to the current next frame */
 511           to_next[0] = bi0 = from[0];
 512           to_next[1] = bi1 = from[1];
 513           from += 2;
 514           to_next += 2;
 515           n_left_from -= 2;
 516           n_left_to_next -= 2;
 517
 518           b0 = vlib_get_buffer (vm, bi0);
 519           b1 = vlib_get_buffer (vm, bi1);
 520
 521           ip0 = vlib_buffer_get_current (b0);
 522           udp0 = ip4_next_header (ip0);
 523           tcp0 = (tcp_header_t *) udp0;
 524           icmp0 = (icmp46_header_t *) udp0;
 525
 526           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 527           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 528                                    sw_if_index0);
 529
 530           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 531
 532           if (PREDICT_FALSE (proto0 == ~0))
 533               goto trace0;
 534
 535           if (PREDICT_FALSE(ip0->ttl == 1))
 536             {
 537               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 538               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 539                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 540                                            0);
 541               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 542               goto trace0;
 543             }
 544
 545           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 546             {
 547               next0 = icmp_out2in_slow_path
 548                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 549                  next0, now, cpu_index, &s0);
 550               goto trace0;
 551             }
 552
 553           key0.addr = ip0->dst_address;
 554           key0.port = udp0->dst_port;
 555           key0.protocol = proto0;
 556           key0.fib_index = rx_fib_index0;
 557
 558           kv0.key = key0.as_u64;
 559
 560           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 561             {
 562               /* Try to match static mapping by external address and port,
 563                  destination address and port in packet */
 564               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 565                 {
 566                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 567                   /*
 568                    * Send DHCP packets to the ipv4 stack, or we won't
 569                    * be able to use dhcp client on the outside interface
 570                    */
 571                   if (proto0 != SNAT_PROTOCOL_UDP
 572                       || (udp0->dst_port
 573                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 574                     next0 = SNAT_OUT2IN_NEXT_DROP;
 575                   goto trace0;
 576                 }
 577
 578               /* Create session initiated by host from external network */
 579               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 580                                                      cpu_index);
 581               if (!s0)
 582                 {
 583                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 584                   next0 = SNAT_OUT2IN_NEXT_DROP;
 585                   goto trace0;
 586                 }
 587             }
 588           else
 589             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 590                                     value0.value);
 591
 592           old_addr0 = ip0->dst_address.as_u32;
 593           ip0->dst_address = s0->in2out.addr;
 594           new_addr0 = ip0->dst_address.as_u32;
 595           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 596
 597           sum0 = ip0->checksum;
 598           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 599                                  ip4_header_t,
 600                                  dst_address /* changed member */);
 601           ip0->checksum = ip_csum_fold (sum0);
 602
 603           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 604             {
 605               old_port0 = tcp0->ports.dst;
 606               tcp0->ports.dst = s0->in2out.port;
 607               new_port0 = tcp0->ports.dst;
 608
 609               sum0 = tcp0->checksum;
 610               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 611                                      ip4_header_t,
 612                                      dst_address /* changed member */);
 613
 614               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 615                                      ip4_header_t /* cheat */,
 616                                      length /* changed member */);
 617               tcp0->checksum = ip_csum_fold(sum0);
 618             }
 619           else
 620             {
 621               old_port0 = udp0->dst_port;
 622               udp0->dst_port = s0->in2out.port;
 623               udp0->checksum = 0;
 624             }
 625
 626           /* Accounting */
 627           s0->last_heard = now;
 628           s0->total_pkts++;
 629           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 630           /* Per-user LRU list maintenance for dynamic translation */
 631           if (!snat_is_session_static (s0))
 632             {
 633               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 634                                  s0->per_user_index);
 635               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 636                                   s0->per_user_list_head_index,
 637                                   s0->per_user_index);
 638             }
 639         trace0:
 640
 641           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 642                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 643             {
 644               snat_out2in_trace_t *t =
 645                  vlib_add_trace (vm, node, b0, sizeof (*t));
 646               t->sw_if_index = sw_if_index0;
 647               t->next_index = next0;
 648               t->session_index = ~0;
 649               if (s0)
 650                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 651             }
 652
 653           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 654
 655
 656           ip1 = vlib_buffer_get_current (b1);
 657           udp1 = ip4_next_header (ip1);
 658           tcp1 = (tcp_header_t *) udp1;
 659           icmp1 = (icmp46_header_t *) udp1;
 660
 661           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 662           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 663                                    sw_if_index1);
 664
 665           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 666
 667           if (PREDICT_FALSE (proto1 == ~0))
 668               goto trace1;
 669
 670           if (PREDICT_FALSE(ip0->ttl == 1))
 671             {
 672               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 673               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 674                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 675                                            0);
 676               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 677               goto trace1;
 678             }
 679
 680           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 681             {
 682               next1 = icmp_out2in_slow_path
 683                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 684                  next1, now, cpu_index, &s1);
 685               goto trace1;
 686             }
 687
 688           key1.addr = ip1->dst_address;
 689           key1.port = udp1->dst_port;
 690           key1.protocol = proto1;
 691           key1.fib_index = rx_fib_index1;
 692
 693           kv1.key = key1.as_u64;
 694
 695           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 696             {
 697               /* Try to match static mapping by external address and port,
 698                  destination address and port in packet */
 699               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 700                 {
 701                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 702                   /*
 703                    * Send DHCP packets to the ipv4 stack, or we won't
 704                    * be able to use dhcp client on the outside interface
 705                    */
 706                   if (proto1 != SNAT_PROTOCOL_UDP
 707                       || (udp1->dst_port
 708                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 709                     next1 = SNAT_OUT2IN_NEXT_DROP;
 710                   goto trace1;
 711                 }
 712
 713               /* Create session initiated by host from external network */
 714               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 715                                                      cpu_index);
 716               if (!s1)
 717                 {
 718                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 719                   next1 = SNAT_OUT2IN_NEXT_DROP;
 720                   goto trace1;
 721                 }
 722             }
 723           else
 724             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 725                                     value1.value);
 726
 727           old_addr1 = ip1->dst_address.as_u32;
 728           ip1->dst_address = s1->in2out.addr;
 729           new_addr1 = ip1->dst_address.as_u32;
 730           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 731
 732           sum1 = ip1->checksum;
 733           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 734                                  ip4_header_t,
 735                                  dst_address /* changed member */);
 736           ip1->checksum = ip_csum_fold (sum1);
 737
 738           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 739             {
 740               old_port1 = tcp1->ports.dst;
 741               tcp1->ports.dst = s1->in2out.port;
 742               new_port1 = tcp1->ports.dst;
 743
 744               sum1 = tcp1->checksum;
 745               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 746                                      ip4_header_t,
 747                                      dst_address /* changed member */);
 748
 749               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 750                                      ip4_header_t /* cheat */,
 751                                      length /* changed member */);
 752               tcp1->checksum = ip_csum_fold(sum1);
 753             }
 754           else
 755             {
 756               old_port1 = udp1->dst_port;
 757               udp1->dst_port = s1->in2out.port;
 758               udp1->checksum = 0;
 759             }
 760
 761           /* Accounting */
 762           s1->last_heard = now;
 763           s1->total_pkts++;
 764           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 765           /* Per-user LRU list maintenance for dynamic translation */
 766           if (!snat_is_session_static (s1))
 767             {
 768               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 769                                  s1->per_user_index);
 770               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 771                                   s1->per_user_list_head_index,
 772                                   s1->per_user_index);
 773             }
 774         trace1:
 775
 776           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 777                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 778             {
 779               snat_out2in_trace_t *t =
 780                  vlib_add_trace (vm, node, b1, sizeof (*t));
 781               t->sw_if_index = sw_if_index1;
 782               t->next_index = next1;
 783               t->session_index = ~0;
 784               if (s1)
 785                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 786             }
 787
 788           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 789
 790           /* verify speculative enqueues, maybe switch current next frame */
 791           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 792                                            to_next, n_left_to_next,
 793                                            bi0, bi1, next0, next1);
 794         }
 795
 796       while (n_left_from > 0 && n_left_to_next > 0)
 797         {
 798           u32 bi0;
 799           vlib_buffer_t * b0;
 800           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 801           u32 sw_if_index0;
 802           ip4_header_t * ip0;
 803           ip_csum_t sum0;
 804           u32 new_addr0, old_addr0;
 805           u16 new_port0, old_port0;
 806           udp_header_t * udp0;
 807           tcp_header_t * tcp0;
 808           icmp46_header_t * icmp0;
 809           snat_session_key_t key0, sm0;
 810           u32 rx_fib_index0;
 811           u32 proto0;
 812           snat_session_t * s0 = 0;
 813           clib_bihash_kv_8_8_t kv0, value0;
 814
 815           /* speculatively enqueue b0 to the current next frame */
 816           bi0 = from[0];
 817           to_next[0] = bi0;
 818           from += 1;
 819           to_next += 1;
 820           n_left_from -= 1;
 821           n_left_to_next -= 1;
 822
 823           b0 = vlib_get_buffer (vm, bi0);
 824
 825           ip0 = vlib_buffer_get_current (b0);
 826           udp0 = ip4_next_header (ip0);
 827           tcp0 = (tcp_header_t *) udp0;
 828           icmp0 = (icmp46_header_t *) udp0;
 829
 830           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 831           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 832                                    sw_if_index0);
 833
 834           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 835
 836           if (PREDICT_FALSE (proto0 == ~0))
 837               goto trace00;
 838
 839           if (PREDICT_FALSE(ip0->ttl == 1))
 840             {
 841               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 842               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 843                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 844                                            0);
 845               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 846               goto trace00;
 847             }
 848
 849           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 850             {
 851               next0 = icmp_out2in_slow_path
 852                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 853                  next0, now, cpu_index, &s0);
 854               goto trace00;
 855             }
 856
 857           key0.addr = ip0->dst_address;
 858           key0.port = udp0->dst_port;
 859           key0.protocol = proto0;
 860           key0.fib_index = rx_fib_index0;
 861
 862           kv0.key = key0.as_u64;
 863
 864           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 865             {
 866               /* Try to match static mapping by external address and port,
 867                  destination address and port in packet */
 868               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 869                 {
 870                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 871                   /*
 872                    * Send DHCP packets to the ipv4 stack, or we won't
 873                    * be able to use dhcp client on the outside interface
 874                    */
 875                   if (proto0 != SNAT_PROTOCOL_UDP
 876                       || (udp0->dst_port
 877                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 878
 879                     next0 = SNAT_OUT2IN_NEXT_DROP;
 880                   goto trace00;
 881                 }
 882
 883               /* Create session initiated by host from external network */
 884               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 885                                                      cpu_index);
 886               if (!s0)
 887                 {
 888                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 889                     next0 = SNAT_OUT2IN_NEXT_DROP;
 890                   goto trace00;
 891                 }
 892             }
 893           else
 894             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 895                                     value0.value);
 896
 897           old_addr0 = ip0->dst_address.as_u32;
 898           ip0->dst_address = s0->in2out.addr;
 899           new_addr0 = ip0->dst_address.as_u32;
 900           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 901
 902           sum0 = ip0->checksum;
 903           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 904                                  ip4_header_t,
 905                                  dst_address /* changed member */);
 906           ip0->checksum = ip_csum_fold (sum0);
 907
 908           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 909             {
 910               old_port0 = tcp0->ports.dst;
 911               tcp0->ports.dst = s0->in2out.port;
 912               new_port0 = tcp0->ports.dst;
 913
 914               sum0 = tcp0->checksum;
 915               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 916                                      ip4_header_t,
 917                                      dst_address /* changed member */);
 918
 919               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 920                                      ip4_header_t /* cheat */,
 921                                      length /* changed member */);
 922               tcp0->checksum = ip_csum_fold(sum0);
 923             }
 924           else
 925             {
 926               old_port0 = udp0->dst_port;
 927               udp0->dst_port = s0->in2out.port;
 928               udp0->checksum = 0;
 929             }
 930
 931           /* Accounting */
 932           s0->last_heard = now;
 933           s0->total_pkts++;
 934           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 935           /* Per-user LRU list maintenance for dynamic translation */
 936           if (!snat_is_session_static (s0))
 937             {
 938               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 939                                  s0->per_user_index);
 940               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 941                                   s0->per_user_list_head_index,
 942                                   s0->per_user_index);
 943             }
 944         trace00:
 945
 946           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 947                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 948             {
 949               snat_out2in_trace_t *t =
 950                  vlib_add_trace (vm, node, b0, sizeof (*t));
 951               t->sw_if_index = sw_if_index0;
 952               t->next_index = next0;
 953               t->session_index = ~0;
 954               if (s0)
 955                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 956             }
 957
 958           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 959
 960           /* verify speculative enqueue, maybe switch current next frame */
 961           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 962                                            to_next, n_left_to_next,
 963                                            bi0, next0);
 964         }
 965
 966       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 967     }
 968
 969   vlib_node_increment_counter (vm, snat_out2in_node.index,
 970                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 971                                pkts_processed);
 972   return frame->n_vectors;
 973 }
 974
 975 VLIB_REGISTER_NODE (snat_out2in_node) = {
 976   .function = snat_out2in_node_fn,
 977   .name = "snat-out2in",
 978   .vector_size = sizeof (u32),
 979   .format_trace = format_snat_out2in_trace,
 980   .type = VLIB_NODE_TYPE_INTERNAL,
 981
 982   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 983   .error_strings = snat_out2in_error_strings,
 984
 985   .runtime_data_bytes = sizeof (snat_runtime_t),
 986
 987   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 988
 989   /* edit / add dispositions here */
 990   .next_nodes = {
 991     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 992     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 993     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
 994   },
 995 };
 996 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 997
 998 static uword
 999 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1000                                vlib_node_runtime_t * node,
1001                                vlib_frame_t * frame)
1002 {
1003   snat_main_t *sm = &snat_main;
1004   vlib_thread_main_t *tm = vlib_get_thread_main ();
1005   u32 n_left_from, *from, *to_next = 0;
1006   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1007   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1008     = 0;
1009   vlib_frame_queue_elt_t *hf = 0;
1010   vlib_frame_t *f = 0;
1011   int i;
1012   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1013   u32 next_worker_index = 0;
1014   u32 current_worker_index = ~0;
1015   u32 cpu_index = os_get_cpu_number ();
1016
1017   ASSERT (vec_len (sm->workers));
1018
1019   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1020     {
1021       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1022
1023       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1024                                sm->first_worker_index + sm->num_workers - 1,
1025                                (vlib_frame_queue_t *) (~0));
1026     }
1027
1028   from = vlib_frame_vector_args (frame);
1029   n_left_from = frame->n_vectors;
1030
1031   while (n_left_from > 0)
1032     {
1033       u32 bi0;
1034       vlib_buffer_t *b0;
1035       u32 sw_if_index0;
1036       u32 rx_fib_index0;
1037       ip4_header_t * ip0;
1038       udp_header_t * udp0;
1039       snat_worker_key_t key0;
1040       clib_bihash_kv_8_8_t kv0, value0;
1041       u8 do_handoff;
1042
1043       bi0 = from[0];
1044       from += 1;
1045       n_left_from -= 1;
1046
1047       b0 = vlib_get_buffer (vm, bi0);
1048
1049       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1050       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1051
1052       ip0 = vlib_buffer_get_current (b0);
1053       udp0 = ip4_next_header (ip0);
1054
1055       key0.addr = ip0->dst_address;
1056       key0.port = udp0->dst_port;
1057       key0.fib_index = rx_fib_index0;
1058
1059       if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
1060         {
1061           icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
1062           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1063           key0.port = echo0->identifier;
1064         }
1065
1066       kv0.key = key0.as_u64;
1067
1068       /* Ever heard of of the "user" before? */
1069       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
1070         {
1071           key0.port = 0;
1072           kv0.key = key0.as_u64;
1073
1074           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
1075             {
1076               /* No, assign next available worker (RR) */
1077               next_worker_index = sm->first_worker_index;
1078               if (vec_len (sm->workers))
1079                 {
1080                   next_worker_index +=
1081                     sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
1082                 }
1083             }
1084           else
1085             {
1086               /* Static mapping without port */
1087               next_worker_index = value0.value;
1088             }
1089
1090           /* Add to translated packets worker lookup */
1091           kv0.value = next_worker_index;
1092           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
1093         }
1094       else
1095         next_worker_index = value0.value;
1096
1097       if (PREDICT_FALSE (next_worker_index != cpu_index))
1098         {
1099           do_handoff = 1;
1100
1101           if (next_worker_index != current_worker_index)
1102             {
1103               if (hf)
1104                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1105
1106               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1107                                                       next_worker_index,
1108                                                       handoff_queue_elt_by_worker_index);
1109
1110               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1111               to_next_worker = &hf->buffer_index[hf->n_vectors];
1112               current_worker_index = next_worker_index;
1113             }
1114
1115           /* enqueue to correct worker thread */
1116           to_next_worker[0] = bi0;
1117           to_next_worker++;
1118           n_left_to_next_worker--;
1119
1120           if (n_left_to_next_worker == 0)
1121             {
1122               hf->n_vectors = VLIB_FRAME_SIZE;
1123               vlib_put_frame_queue_elt (hf);
1124               current_worker_index = ~0;
1125               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1126               hf = 0;
1127             }
1128         }
1129       else
1130         {
1131           do_handoff = 0;
1132           /* if this is 1st frame */
1133           if (!f)
1134             {
1135               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
1136               to_next = vlib_frame_vector_args (f);
1137             }
1138
1139           to_next[0] = bi0;
1140           to_next += 1;
1141           f->n_vectors++;
1142         }
1143
1144       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1145                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1146         {
1147           snat_out2in_worker_handoff_trace_t *t =
1148             vlib_add_trace (vm, node, b0, sizeof (*t));
1149           t->next_worker_index = next_worker_index;
1150           t->do_handoff = do_handoff;
1151         }
1152     }
1153
1154   if (f)
1155     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
1156
1157   if (hf)
1158     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1159
1160   /* Ship frames to the worker nodes */
1161   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1162     {
1163       if (handoff_queue_elt_by_worker_index[i])
1164         {
1165           hf = handoff_queue_elt_by_worker_index[i];
1166           /*
1167            * It works better to let the handoff node
1168            * rate-adapt, always ship the handoff queue element.
1169            */
1170           if (1 || hf->n_vectors == hf->last_n_vectors)
1171             {
1172               vlib_put_frame_queue_elt (hf);
1173               handoff_queue_elt_by_worker_index[i] = 0;
1174             }
1175           else
1176             hf->last_n_vectors = hf->n_vectors;
1177         }
1178       congested_handoff_queue_by_worker_index[i] =
1179         (vlib_frame_queue_t *) (~0);
1180     }
1181   hf = 0;
1182   current_worker_index = ~0;
1183   return frame->n_vectors;
1184 }
1185
1186 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1187   .function = snat_out2in_worker_handoff_fn,
1188   .name = "snat-out2in-worker-handoff",
1189   .vector_size = sizeof (u32),
1190   .format_trace = format_snat_out2in_worker_handoff_trace,
1191   .type = VLIB_NODE_TYPE_INTERNAL,
1192
1193   .n_next_nodes = 1,
1194
1195   .next_nodes = {
1196     [0] = "error-drop",
1197   },
1198 };
1199
1200 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1201
1202 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1203                                     vlib_buffer_t * b0,
1204                                     ip4_header_t * ip0,
1205                                     icmp46_header_t * icmp0,
1206                                     u32 sw_if_index0,
1207                                     vlib_node_runtime_t * node,
1208                                     u32 next0,
1209                                     u32 rx_fib_index0)
1210 {
1211   snat_session_key_t key0, sm0;
1212   icmp_echo_header_t *echo0;
1213   u32 new_addr0, old_addr0;
1214   u16 old_id0, new_id0;
1215   ip_csum_t sum0;
1216   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1217
1218   echo0 = (icmp_echo_header_t *)(icmp0+1);
1219
1220   key0.addr = ip0->dst_address;
1221   key0.port = echo0->identifier;
1222   key0.fib_index = rx_fib_index0;
1223
1224   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1225     {
1226       ip4_address_t * first_int_addr;
1227
1228       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1229         {
1230           first_int_addr =
1231             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1232                                          0 /* just want the address */);
1233           rt->cached_sw_if_index = sw_if_index0;
1234           if (first_int_addr)
1235             rt->cached_ip4_address = first_int_addr->as_u32;
1236           else
1237             rt->cached_ip4_address = 0;
1238         }
1239
1240       /* Don't NAT packet aimed at the intfc address */
1241       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1242                         rt->cached_ip4_address))
1243         return next0;
1244
1245       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1246       return SNAT_OUT2IN_NEXT_DROP;
1247     }
1248
1249   new_addr0 = sm0.addr.as_u32;
1250   new_id0 = sm0.port;
1251   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1252
1253   old_addr0 = ip0->dst_address.as_u32;
1254   ip0->dst_address.as_u32 = new_addr0;
1255
1256   sum0 = ip0->checksum;
1257   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1258                          ip4_header_t,
1259                          dst_address /* changed member */);
1260   ip0->checksum = ip_csum_fold (sum0);
1261
1262   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1263     {
1264       old_id0 = echo0->identifier;
1265       echo0->identifier = new_id0;
1266
1267       sum0 = icmp0->checksum;
1268       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1269                              identifier);
1270       icmp0->checksum = ip_csum_fold (sum0);
1271     }
1272
1273   return next0;
1274 }
1275
1276 static uword
1277 snat_out2in_fast_node_fn (vlib_main_t * vm,
1278                           vlib_node_runtime_t * node,
1279                           vlib_frame_t * frame)
1280 {
1281   u32 n_left_from, * from, * to_next;
1282   snat_out2in_next_t next_index;
1283   u32 pkts_processed = 0;
1284   snat_main_t * sm = &snat_main;
1285
1286   from = vlib_frame_vector_args (frame);
1287   n_left_from = frame->n_vectors;
1288   next_index = node->cached_next_index;
1289
1290   while (n_left_from > 0)
1291     {
1292       u32 n_left_to_next;
1293
1294       vlib_get_next_frame (vm, node, next_index,
1295                            to_next, n_left_to_next);
1296
1297       while (n_left_from > 0 && n_left_to_next > 0)
1298         {
1299           u32 bi0;
1300           vlib_buffer_t * b0;
1301           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1302           u32 sw_if_index0;
1303           ip4_header_t * ip0;
1304           ip_csum_t sum0;
1305           u32 new_addr0, old_addr0;
1306           u16 new_port0, old_port0;
1307           udp_header_t * udp0;
1308           tcp_header_t * tcp0;
1309           icmp46_header_t * icmp0;
1310           snat_session_key_t key0, sm0;
1311           u32 proto0;
1312           u32 rx_fib_index0;
1313
1314           /* speculatively enqueue b0 to the current next frame */
1315           bi0 = from[0];
1316           to_next[0] = bi0;
1317           from += 1;
1318           to_next += 1;
1319           n_left_from -= 1;
1320           n_left_to_next -= 1;
1321
1322           b0 = vlib_get_buffer (vm, bi0);
1323
1324           ip0 = vlib_buffer_get_current (b0);
1325           udp0 = ip4_next_header (ip0);
1326           tcp0 = (tcp_header_t *) udp0;
1327           icmp0 = (icmp46_header_t *) udp0;
1328
1329           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1330           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1331
1332           vnet_feature_next (sw_if_index0, &next0, b0);
1333
1334           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1335
1336           if (PREDICT_FALSE (proto0 == ~0))
1337               goto trace00;
1338
1339           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1340             {
1341               next0 = icmp_out2in_fast
1342                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1343               goto trace00;
1344             }
1345
1346           key0.addr = ip0->dst_address;
1347           key0.port = udp0->dst_port;
1348           key0.fib_index = rx_fib_index0;
1349
1350           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1351             {
1352               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1353               goto trace00;
1354             }
1355
1356           new_addr0 = sm0.addr.as_u32;
1357           new_port0 = sm0.port;
1358           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1359           old_addr0 = ip0->dst_address.as_u32;
1360           ip0->dst_address.as_u32 = new_addr0;
1361
1362           sum0 = ip0->checksum;
1363           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1364                                  ip4_header_t,
1365                                  dst_address /* changed member */);
1366           ip0->checksum = ip_csum_fold (sum0);
1367
1368           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1369             {
1370                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1371                 {
1372                   old_port0 = tcp0->ports.dst;
1373                   tcp0->ports.dst = new_port0;
1374
1375                   sum0 = tcp0->checksum;
1376                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1377                                          ip4_header_t,
1378                                          dst_address /* changed member */);
1379
1380                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1381                                          ip4_header_t /* cheat */,
1382                                          length /* changed member */);
1383                   tcp0->checksum = ip_csum_fold(sum0);
1384                 }
1385               else
1386                 {
1387                   old_port0 = udp0->dst_port;
1388                   udp0->dst_port = new_port0;
1389                   udp0->checksum = 0;
1390                 }
1391             }
1392           else
1393             {
1394               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1395                 {
1396                   sum0 = tcp0->checksum;
1397                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1398                                          ip4_header_t,
1399                                          dst_address /* changed member */);
1400
1401                   tcp0->checksum = ip_csum_fold(sum0);
1402                 }
1403             }
1404
1405         trace00:
1406
1407           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1408                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1409             {
1410               snat_out2in_trace_t *t =
1411                  vlib_add_trace (vm, node, b0, sizeof (*t));
1412               t->sw_if_index = sw_if_index0;
1413               t->next_index = next0;
1414             }
1415
1416           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1417
1418           /* verify speculative enqueue, maybe switch current next frame */
1419           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1420                                            to_next, n_left_to_next,
1421                                            bi0, next0);
1422         }
1423
1424       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1425     }
1426
1427   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1428                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1429                                pkts_processed);
1430   return frame->n_vectors;
1431 }
1432
1433 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1434   .function = snat_out2in_fast_node_fn,
1435   .name = "snat-out2in-fast",
1436   .vector_size = sizeof (u32),
1437   .format_trace = format_snat_out2in_fast_trace,
1438   .type = VLIB_NODE_TYPE_INTERNAL,
1439
1440   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1441   .error_strings = snat_out2in_error_strings,
1442
1443   .runtime_data_bytes = sizeof (snat_runtime_t),
1444
1445   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1446
1447   /* edit / add dispositions here */
1448   .next_nodes = {
1449     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1450     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1451     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1452   },
1453 };
1454 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);