src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36 } snat_out2in_trace_t;
  37
  38 typedef struct {
  39   u32 next_worker_index;
  40   u8 do_handoff;
  41 } snat_out2in_worker_handoff_trace_t;
  42
  43 /* packet trace format function */
  44 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  45 {
  46   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  47   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  48   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  49
  50   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  51               t->sw_if_index, t->next_index, t->session_index);
  52   return s;
  53 }
  54
  55 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  56 {
  57   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  58   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  59   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  60
  61   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  62               t->sw_if_index, t->next_index);
  63   return s;
  64 }
  65
  66 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   snat_out2in_worker_handoff_trace_t * t =
  71     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  72   char * m;
  73
  74   m = t->do_handoff ? "next worker" : "same worker";
  75   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  76
  77   return s;
  78 }
  79
  80 vlib_node_registration_t snat_out2in_node;
  81 vlib_node_registration_t snat_out2in_fast_node;
  82 vlib_node_registration_t snat_out2in_worker_handoff_node;
  83
  84 #define foreach_snat_out2in_error                       \
  85 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  86 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  87 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  88 _(NO_TRANSLATION, "No translation")
  89
  90 typedef enum {
  91 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  92   foreach_snat_out2in_error
  93 #undef _
  94   SNAT_OUT2IN_N_ERROR,
  95 } snat_out2in_error_t;
  96
  97 static char * snat_out2in_error_strings[] = {
  98 #define _(sym,string) string,
  99   foreach_snat_out2in_error
 100 #undef _
 101 };
 102
 103 typedef enum {
 104   SNAT_OUT2IN_NEXT_DROP,
 105   SNAT_OUT2IN_NEXT_LOOKUP,
 106   SNAT_OUT2IN_N_NEXT,
 107 } snat_out2in_next_t;
 108
 109 /**
 110  * @brief Create session for static mapping.
 111  *
 112  * Create NAT session initiated by host from external network with static
 113  * mapping.
 114  *
 115  * @param sm     SNAT main.
 116  * @param b0     Vlib buffer.
 117  * @param in2out In2out SNAT session key.
 118  * @param out2in Out2in SNAT session key.
 119  * @param node   Vlib node.
 120  *
 121  * @returns SNAT session if successfully created otherwise 0.
 122  */
 123 static inline snat_session_t *
 124 create_session_for_static_mapping (snat_main_t *sm,
 125                                    vlib_buffer_t *b0,
 126                                    snat_session_key_t in2out,
 127                                    snat_session_key_t out2in,
 128                                    vlib_node_runtime_t * node,
 129                                    u32 cpu_index)
 130 {
 131   snat_user_t *u;
 132   snat_user_key_t user_key;
 133   snat_session_t *s;
 134   clib_bihash_kv_8_8_t kv0, value0;
 135   dlist_elt_t * per_user_translation_list_elt;
 136   dlist_elt_t * per_user_list_head_elt;
 137
 138   user_key.addr = in2out.addr;
 139   user_key.fib_index = in2out.fib_index;
 140   kv0.key = user_key.as_u64;
 141
 142   /* Ever heard of the "user" = inside ip4 address before? */
 143   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 144     {
 145       /* no, make a new one */
 146       pool_get (sm->per_thread_data[cpu_index].users, u);
 147       memset (u, 0, sizeof (*u));
 148       u->addr = in2out.addr;
 149
 150       pool_get (sm->per_thread_data[cpu_index].list_pool,
 151                 per_user_list_head_elt);
 152
 153       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 154         sm->per_thread_data[cpu_index].list_pool;
 155
 156       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 157                        u->sessions_per_user_list_head_index);
 158
 159       kv0.value = u - sm->per_thread_data[cpu_index].users;
 160
 161       /* add user */
 162       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 163
 164       /* add non-traslated packets worker lookup */
 165       kv0.value = cpu_index;
 166       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 167     }
 168   else
 169     {
 170       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 171                              value0.value);
 172     }
 173
 174   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 175   memset (s, 0, sizeof (*s));
 176
 177   s->outside_address_index = ~0;
 178   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 179   u->nstaticsessions++;
 180
 181   /* Create list elts */
 182   pool_get (sm->per_thread_data[cpu_index].list_pool,
 183             per_user_translation_list_elt);
 184   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 185                    per_user_translation_list_elt -
 186                    sm->per_thread_data[cpu_index].list_pool);
 187
 188   per_user_translation_list_elt->value =
 189     s - sm->per_thread_data[cpu_index].sessions;
 190   s->per_user_index =
 191     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 192   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 193
 194   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 195                       s->per_user_list_head_index,
 196                       per_user_translation_list_elt -
 197                       sm->per_thread_data[cpu_index].list_pool);
 198
 199   s->in2out = in2out;
 200   s->out2in = out2in;
 201   s->in2out.protocol = out2in.protocol;
 202
 203   /* Add to translation hashes */
 204   kv0.key = s->in2out.as_u64;
 205   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 206   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 207       clib_warning ("in2out key add failed");
 208
 209   kv0.key = s->out2in.as_u64;
 210   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 211
 212   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 213       clib_warning ("out2in key add failed");
 214
 215   /* log NAT event */
 216   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 217                                       s->out2in.addr.as_u32,
 218                                       s->in2out.protocol,
 219                                       s->in2out.port,
 220                                       s->out2in.port,
 221                                       s->in2out.fib_index);
 222    return s;
 223 }
 224
 225 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 226                                          vlib_buffer_t * b0,
 227                                          ip4_header_t * ip0,
 228                                          icmp46_header_t * icmp0,
 229                                          u32 sw_if_index0,
 230                                          u32 rx_fib_index0,
 231                                          vlib_node_runtime_t * node,
 232                                          u32 next0, f64 now,
 233                                          u32 cpu_index)
 234 {
 235   snat_session_key_t key0, sm0;
 236   icmp_echo_header_t *echo0;
 237   clib_bihash_kv_8_8_t kv0, value0;
 238   snat_session_t * s0;
 239   u32 new_addr0, old_addr0;
 240   u16 old_id0, new_id0;
 241   ip_csum_t sum0;
 242   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 243
 244   echo0 = (icmp_echo_header_t *)(icmp0+1);
 245
 246   key0.addr = ip0->dst_address;
 247   key0.port = echo0->identifier;
 248   key0.protocol = SNAT_PROTOCOL_ICMP;
 249   key0.fib_index = rx_fib_index0;
 250
 251   kv0.key = key0.as_u64;
 252
 253   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 254     {
 255       /* Try to match static mapping by external address and port,
 256          destination address and port in packet */
 257       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 258         {
 259            ip4_address_t * first_int_addr;
 260
 261           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 262             {
 263               first_int_addr =
 264                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 265                                              0 /* just want the address */);
 266               rt->cached_sw_if_index = sw_if_index0;
 267               if (first_int_addr)
 268                 rt->cached_ip4_address = first_int_addr->as_u32;
 269               else
 270                 rt->cached_ip4_address = 0;
 271             }
 272
 273           /* Don't NAT packet aimed at the intfc address */
 274           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 275                             rt->cached_ip4_address))
 276             return next0;
 277
 278           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 279           return SNAT_OUT2IN_NEXT_DROP;
 280         }
 281
 282       /* Create session initiated by host from external network */
 283       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 284                                              node, cpu_index);
 285       if (!s0)
 286         return SNAT_OUT2IN_NEXT_DROP;
 287     }
 288   else
 289     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 290                             value0.value);
 291
 292   old_addr0 = ip0->dst_address.as_u32;
 293   ip0->dst_address = s0->in2out.addr;
 294   new_addr0 = ip0->dst_address.as_u32;
 295   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 296
 297   sum0 = ip0->checksum;
 298   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 299                          ip4_header_t,
 300                          dst_address /* changed member */);
 301   ip0->checksum = ip_csum_fold (sum0);
 302
 303   old_id0 = echo0->identifier;
 304   new_id0 = s0->in2out.port;
 305   echo0->identifier = new_id0;
 306
 307   sum0 = icmp0->checksum;
 308   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 309                          identifier);
 310   icmp0->checksum = ip_csum_fold (sum0);
 311
 312   /* Accounting */
 313   s0->last_heard = now;
 314   s0->total_pkts++;
 315   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 316   /* Per-user LRU list maintenance for dynamic translation */
 317   if (!snat_is_session_static (s0))
 318     {
 319       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 320                          s0->per_user_index);
 321       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 322                           s0->per_user_list_head_index,
 323                           s0->per_user_index);
 324     }
 325
 326   return next0;
 327 }
 328
 329 static uword
 330 snat_out2in_node_fn (vlib_main_t * vm,
 331                   vlib_node_runtime_t * node,
 332                   vlib_frame_t * frame)
 333 {
 334   u32 n_left_from, * from, * to_next;
 335   snat_out2in_next_t next_index;
 336   u32 pkts_processed = 0;
 337   snat_main_t * sm = &snat_main;
 338   f64 now = vlib_time_now (vm);
 339   u32 cpu_index = os_get_cpu_number ();
 340
 341   from = vlib_frame_vector_args (frame);
 342   n_left_from = frame->n_vectors;
 343   next_index = node->cached_next_index;
 344
 345   while (n_left_from > 0)
 346     {
 347       u32 n_left_to_next;
 348
 349       vlib_get_next_frame (vm, node, next_index,
 350                            to_next, n_left_to_next);
 351
 352       while (n_left_from >= 4 && n_left_to_next >= 2)
 353         {
 354           u32 bi0, bi1;
 355           vlib_buffer_t * b0, * b1;
 356           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 357           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 358           u32 sw_if_index0, sw_if_index1;
 359           ip4_header_t * ip0, *ip1;
 360           ip_csum_t sum0, sum1;
 361           u32 new_addr0, old_addr0;
 362           u16 new_port0, old_port0;
 363           u32 new_addr1, old_addr1;
 364           u16 new_port1, old_port1;
 365           udp_header_t * udp0, * udp1;
 366           tcp_header_t * tcp0, * tcp1;
 367           icmp46_header_t * icmp0, * icmp1;
 368           snat_session_key_t key0, key1, sm0, sm1;
 369           u32 rx_fib_index0, rx_fib_index1;
 370           u32 proto0, proto1;
 371           snat_session_t * s0 = 0, * s1 = 0;
 372           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 373
 374           /* Prefetch next iteration. */
 375           {
 376             vlib_buffer_t * p2, * p3;
 377
 378             p2 = vlib_get_buffer (vm, from[2]);
 379             p3 = vlib_get_buffer (vm, from[3]);
 380
 381             vlib_prefetch_buffer_header (p2, LOAD);
 382             vlib_prefetch_buffer_header (p3, LOAD);
 383
 384             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 385             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 386           }
 387
 388           /* speculatively enqueue b0 and b1 to the current next frame */
 389           to_next[0] = bi0 = from[0];
 390           to_next[1] = bi1 = from[1];
 391           from += 2;
 392           to_next += 2;
 393           n_left_from -= 2;
 394           n_left_to_next -= 2;
 395
 396           b0 = vlib_get_buffer (vm, bi0);
 397           b1 = vlib_get_buffer (vm, bi1);
 398
 399           ip0 = vlib_buffer_get_current (b0);
 400           udp0 = ip4_next_header (ip0);
 401           tcp0 = (tcp_header_t *) udp0;
 402           icmp0 = (icmp46_header_t *) udp0;
 403
 404           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 405           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 406                                    sw_if_index0);
 407
 408           proto0 = ~0;
 409           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 410             ? SNAT_PROTOCOL_UDP : proto0;
 411           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 412             ? SNAT_PROTOCOL_TCP : proto0;
 413           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 414             ? SNAT_PROTOCOL_ICMP : proto0;
 415
 416           if (PREDICT_FALSE (proto0 == ~0))
 417               goto trace0;
 418
 419           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 420             {
 421               next0 = icmp_out2in_slow_path
 422                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 423                  next0, now, cpu_index);
 424               goto trace0;
 425             }
 426
 427           key0.addr = ip0->dst_address;
 428           key0.port = udp0->dst_port;
 429           key0.protocol = proto0;
 430           key0.fib_index = rx_fib_index0;
 431
 432           kv0.key = key0.as_u64;
 433
 434           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 435             {
 436               /* Try to match static mapping by external address and port,
 437                  destination address and port in packet */
 438               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 439                 {
 440                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 441                   /*
 442                    * Send DHCP packets to the ipv4 stack, or we won't
 443                    * be able to use dhcp client on the outside interface
 444                    */
 445                   if (proto0 != SNAT_PROTOCOL_UDP
 446                       || (udp0->dst_port
 447                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 448                     next0 = SNAT_OUT2IN_NEXT_DROP;
 449                   goto trace0;
 450                 }
 451
 452               /* Create session initiated by host from external network */
 453               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 454                                                      cpu_index);
 455               if (!s0)
 456                 {
 457                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 458                   next0 = SNAT_OUT2IN_NEXT_DROP;
 459                   goto trace0;
 460                 }
 461             }
 462           else
 463             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 464                                     value0.value);
 465
 466           old_addr0 = ip0->dst_address.as_u32;
 467           ip0->dst_address = s0->in2out.addr;
 468           new_addr0 = ip0->dst_address.as_u32;
 469           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 470
 471           sum0 = ip0->checksum;
 472           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 473                                  ip4_header_t,
 474                                  dst_address /* changed member */);
 475           ip0->checksum = ip_csum_fold (sum0);
 476
 477           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 478             {
 479               old_port0 = tcp0->ports.dst;
 480               tcp0->ports.dst = s0->in2out.port;
 481               new_port0 = tcp0->ports.dst;
 482
 483               sum0 = tcp0->checksum;
 484               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 485                                      ip4_header_t,
 486                                      dst_address /* changed member */);
 487
 488               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 489                                      ip4_header_t /* cheat */,
 490                                      length /* changed member */);
 491               tcp0->checksum = ip_csum_fold(sum0);
 492             }
 493           else
 494             {
 495               old_port0 = udp0->dst_port;
 496               udp0->dst_port = s0->in2out.port;
 497               udp0->checksum = 0;
 498             }
 499
 500           /* Accounting */
 501           s0->last_heard = now;
 502           s0->total_pkts++;
 503           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 504           /* Per-user LRU list maintenance for dynamic translation */
 505           if (!snat_is_session_static (s0))
 506             {
 507               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 508                                  s0->per_user_index);
 509               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 510                                   s0->per_user_list_head_index,
 511                                   s0->per_user_index);
 512             }
 513         trace0:
 514
 515           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 516                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 517             {
 518               snat_out2in_trace_t *t =
 519                  vlib_add_trace (vm, node, b0, sizeof (*t));
 520               t->sw_if_index = sw_if_index0;
 521               t->next_index = next0;
 522               t->session_index = ~0;
 523               if (s0)
 524                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 525             }
 526
 527           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 528
 529
 530           ip1 = vlib_buffer_get_current (b1);
 531           udp1 = ip4_next_header (ip1);
 532           tcp1 = (tcp_header_t *) udp1;
 533           icmp1 = (icmp46_header_t *) udp1;
 534
 535           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 536           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 537                                    sw_if_index1);
 538
 539           proto1 = ~0;
 540           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 541             ? SNAT_PROTOCOL_UDP : proto1;
 542           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 543             ? SNAT_PROTOCOL_TCP : proto1;
 544           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 545             ? SNAT_PROTOCOL_ICMP : proto1;
 546
 547           if (PREDICT_FALSE (proto1 == ~0))
 548               goto trace1;
 549
 550           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 551             {
 552               next1 = icmp_out2in_slow_path
 553                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 554                  next1, now, cpu_index);
 555               goto trace1;
 556             }
 557
 558           key1.addr = ip1->dst_address;
 559           key1.port = udp1->dst_port;
 560           key1.protocol = proto1;
 561           key1.fib_index = rx_fib_index1;
 562
 563           kv1.key = key1.as_u64;
 564
 565           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 566             {
 567               /* Try to match static mapping by external address and port,
 568                  destination address and port in packet */
 569               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 570                 {
 571                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 572                   /*
 573                    * Send DHCP packets to the ipv4 stack, or we won't
 574                    * be able to use dhcp client on the outside interface
 575                    */
 576                   if (proto1 != SNAT_PROTOCOL_UDP
 577                       || (udp1->dst_port
 578                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 579                     next1 = SNAT_OUT2IN_NEXT_DROP;
 580                   goto trace1;
 581                 }
 582
 583               /* Create session initiated by host from external network */
 584               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 585                                                      cpu_index);
 586               if (!s1)
 587                 {
 588                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 589                   next1 = SNAT_OUT2IN_NEXT_DROP;
 590                   goto trace1;
 591                 }
 592             }
 593           else
 594             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 595                                     value1.value);
 596
 597           old_addr1 = ip1->dst_address.as_u32;
 598           ip1->dst_address = s1->in2out.addr;
 599           new_addr1 = ip1->dst_address.as_u32;
 600           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 601
 602           sum1 = ip1->checksum;
 603           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 604                                  ip4_header_t,
 605                                  dst_address /* changed member */);
 606           ip1->checksum = ip_csum_fold (sum1);
 607
 608           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 609             {
 610               old_port1 = tcp1->ports.dst;
 611               tcp1->ports.dst = s1->in2out.port;
 612               new_port1 = tcp1->ports.dst;
 613
 614               sum1 = tcp1->checksum;
 615               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 616                                      ip4_header_t,
 617                                      dst_address /* changed member */);
 618
 619               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 620                                      ip4_header_t /* cheat */,
 621                                      length /* changed member */);
 622               tcp1->checksum = ip_csum_fold(sum1);
 623             }
 624           else
 625             {
 626               old_port1 = udp1->dst_port;
 627               udp1->dst_port = s1->in2out.port;
 628               udp1->checksum = 0;
 629             }
 630
 631           /* Accounting */
 632           s1->last_heard = now;
 633           s1->total_pkts++;
 634           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 635           /* Per-user LRU list maintenance for dynamic translation */
 636           if (!snat_is_session_static (s1))
 637             {
 638               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 639                                  s1->per_user_index);
 640               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 641                                   s1->per_user_list_head_index,
 642                                   s1->per_user_index);
 643             }
 644         trace1:
 645
 646           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 647                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 648             {
 649               snat_out2in_trace_t *t =
 650                  vlib_add_trace (vm, node, b1, sizeof (*t));
 651               t->sw_if_index = sw_if_index1;
 652               t->next_index = next1;
 653               t->session_index = ~0;
 654               if (s1)
 655                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 656             }
 657
 658           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 659
 660           /* verify speculative enqueues, maybe switch current next frame */
 661           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 662                                            to_next, n_left_to_next,
 663                                            bi0, bi1, next0, next1);
 664         }
 665
 666       while (n_left_from > 0 && n_left_to_next > 0)
 667         {
 668           u32 bi0;
 669           vlib_buffer_t * b0;
 670           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 671           u32 sw_if_index0;
 672           ip4_header_t * ip0;
 673           ip_csum_t sum0;
 674           u32 new_addr0, old_addr0;
 675           u16 new_port0, old_port0;
 676           udp_header_t * udp0;
 677           tcp_header_t * tcp0;
 678           icmp46_header_t * icmp0;
 679           snat_session_key_t key0, sm0;
 680           u32 rx_fib_index0;
 681           u32 proto0;
 682           snat_session_t * s0 = 0;
 683           clib_bihash_kv_8_8_t kv0, value0;
 684
 685           /* speculatively enqueue b0 to the current next frame */
 686           bi0 = from[0];
 687           to_next[0] = bi0;
 688           from += 1;
 689           to_next += 1;
 690           n_left_from -= 1;
 691           n_left_to_next -= 1;
 692
 693           b0 = vlib_get_buffer (vm, bi0);
 694
 695           ip0 = vlib_buffer_get_current (b0);
 696           udp0 = ip4_next_header (ip0);
 697           tcp0 = (tcp_header_t *) udp0;
 698           icmp0 = (icmp46_header_t *) udp0;
 699
 700           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 701           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 702                                    sw_if_index0);
 703
 704           proto0 = ~0;
 705           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 706             ? SNAT_PROTOCOL_UDP : proto0;
 707           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 708             ? SNAT_PROTOCOL_TCP : proto0;
 709           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 710             ? SNAT_PROTOCOL_ICMP : proto0;
 711
 712           if (PREDICT_FALSE (proto0 == ~0))
 713               goto trace00;
 714
 715           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 716             {
 717               next0 = icmp_out2in_slow_path
 718                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 719                  next0, now, cpu_index);
 720               goto trace00;
 721             }
 722
 723           key0.addr = ip0->dst_address;
 724           key0.port = udp0->dst_port;
 725           key0.protocol = proto0;
 726           key0.fib_index = rx_fib_index0;
 727
 728           kv0.key = key0.as_u64;
 729
 730           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 731             {
 732               /* Try to match static mapping by external address and port,
 733                  destination address and port in packet */
 734               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 735                 {
 736                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 737                   /*
 738                    * Send DHCP packets to the ipv4 stack, or we won't
 739                    * be able to use dhcp client on the outside interface
 740                    */
 741                   if (proto0 != SNAT_PROTOCOL_UDP
 742                       || (udp0->dst_port
 743                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 744
 745                     next0 = SNAT_OUT2IN_NEXT_DROP;
 746                   goto trace00;
 747                 }
 748
 749               /* Create session initiated by host from external network */
 750               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 751                                                      cpu_index);
 752               if (!s0)
 753                 {
 754                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 755                     next0 = SNAT_OUT2IN_NEXT_DROP;
 756                   goto trace00;
 757                 }
 758             }
 759           else
 760             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 761                                     value0.value);
 762
 763           old_addr0 = ip0->dst_address.as_u32;
 764           ip0->dst_address = s0->in2out.addr;
 765           new_addr0 = ip0->dst_address.as_u32;
 766           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 767
 768           sum0 = ip0->checksum;
 769           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 770                                  ip4_header_t,
 771                                  dst_address /* changed member */);
 772           ip0->checksum = ip_csum_fold (sum0);
 773
 774           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 775             {
 776               old_port0 = tcp0->ports.dst;
 777               tcp0->ports.dst = s0->in2out.port;
 778               new_port0 = tcp0->ports.dst;
 779
 780               sum0 = tcp0->checksum;
 781               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 782                                      ip4_header_t,
 783                                      dst_address /* changed member */);
 784
 785               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 786                                      ip4_header_t /* cheat */,
 787                                      length /* changed member */);
 788               tcp0->checksum = ip_csum_fold(sum0);
 789             }
 790           else
 791             {
 792               old_port0 = udp0->dst_port;
 793               udp0->dst_port = s0->in2out.port;
 794               udp0->checksum = 0;
 795             }
 796
 797           /* Accounting */
 798           s0->last_heard = now;
 799           s0->total_pkts++;
 800           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 801           /* Per-user LRU list maintenance for dynamic translation */
 802           if (!snat_is_session_static (s0))
 803             {
 804               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 805                                  s0->per_user_index);
 806               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 807                                   s0->per_user_list_head_index,
 808                                   s0->per_user_index);
 809             }
 810         trace00:
 811
 812           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 813                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 814             {
 815               snat_out2in_trace_t *t =
 816                  vlib_add_trace (vm, node, b0, sizeof (*t));
 817               t->sw_if_index = sw_if_index0;
 818               t->next_index = next0;
 819               t->session_index = ~0;
 820               if (s0)
 821                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 822             }
 823
 824           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 825
 826           /* verify speculative enqueue, maybe switch current next frame */
 827           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 828                                            to_next, n_left_to_next,
 829                                            bi0, next0);
 830         }
 831
 832       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 833     }
 834
 835   vlib_node_increment_counter (vm, snat_out2in_node.index,
 836                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 837                                pkts_processed);
 838   return frame->n_vectors;
 839 }
 840
 841 VLIB_REGISTER_NODE (snat_out2in_node) = {
 842   .function = snat_out2in_node_fn,
 843   .name = "snat-out2in",
 844   .vector_size = sizeof (u32),
 845   .format_trace = format_snat_out2in_trace,
 846   .type = VLIB_NODE_TYPE_INTERNAL,
 847
 848   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 849   .error_strings = snat_out2in_error_strings,
 850
 851   .runtime_data_bytes = sizeof (snat_runtime_t),
 852
 853   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 854
 855   /* edit / add dispositions here */
 856   .next_nodes = {
 857     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 858     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 859   },
 860 };
 861 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 862
 863 static uword
 864 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
 865                                vlib_node_runtime_t * node,
 866                                vlib_frame_t * frame)
 867 {
 868   snat_main_t *sm = &snat_main;
 869   vlib_thread_main_t *tm = vlib_get_thread_main ();
 870   u32 n_left_from, *from, *to_next = 0;
 871   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
 872   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
 873     = 0;
 874   vlib_frame_queue_elt_t *hf = 0;
 875   vlib_frame_t *f = 0;
 876   int i;
 877   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
 878   u32 next_worker_index = 0;
 879   u32 current_worker_index = ~0;
 880   u32 cpu_index = os_get_cpu_number ();
 881
 882   ASSERT (vec_len (sm->workers));
 883
 884   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
 885     {
 886       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
 887
 888       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
 889                                sm->first_worker_index + sm->num_workers - 1,
 890                                (vlib_frame_queue_t *) (~0));
 891     }
 892
 893   from = vlib_frame_vector_args (frame);
 894   n_left_from = frame->n_vectors;
 895
 896   while (n_left_from > 0)
 897     {
 898       u32 bi0;
 899       vlib_buffer_t *b0;
 900       u32 sw_if_index0;
 901       u32 rx_fib_index0;
 902       ip4_header_t * ip0;
 903       udp_header_t * udp0;
 904       snat_static_mapping_key_t key0;
 905       clib_bihash_kv_8_8_t kv0, value0;
 906       u8 do_handoff;
 907
 908       bi0 = from[0];
 909       from += 1;
 910       n_left_from -= 1;
 911
 912       b0 = vlib_get_buffer (vm, bi0);
 913
 914       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 915       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
 916
 917       ip0 = vlib_buffer_get_current (b0);
 918       udp0 = ip4_next_header (ip0);
 919
 920       key0.addr = ip0->dst_address;
 921       key0.port = udp0->dst_port;
 922       key0.fib_index = rx_fib_index0;
 923
 924       if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
 925         {
 926           icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
 927           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 928           key0.port = echo0->identifier;
 929         }
 930
 931       kv0.key = key0.as_u64;
 932
 933       /* Ever heard of of the "user" before? */
 934       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 935         {
 936           key0.port = 0;
 937           kv0.key = key0.as_u64;
 938
 939           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 940             {
 941               /* No, assign next available worker (RR) */
 942               next_worker_index = sm->first_worker_index;
 943               if (vec_len (sm->workers))
 944                 {
 945                   next_worker_index +=
 946                     sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
 947                 }
 948             }
 949           else
 950             {
 951               /* Static mapping without port */
 952               next_worker_index = value0.value;
 953             }
 954
 955           /* Add to translated packets worker lookup */
 956           kv0.value = next_worker_index;
 957           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 958         }
 959       else
 960         next_worker_index = value0.value;
 961
 962       if (PREDICT_FALSE (next_worker_index != cpu_index))
 963         {
 964           do_handoff = 1;
 965
 966           if (next_worker_index != current_worker_index)
 967             {
 968               if (hf)
 969                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 970
 971               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
 972                                                       next_worker_index,
 973                                                       handoff_queue_elt_by_worker_index);
 974
 975               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
 976               to_next_worker = &hf->buffer_index[hf->n_vectors];
 977               current_worker_index = next_worker_index;
 978             }
 979
 980           /* enqueue to correct worker thread */
 981           to_next_worker[0] = bi0;
 982           to_next_worker++;
 983           n_left_to_next_worker--;
 984
 985           if (n_left_to_next_worker == 0)
 986             {
 987               hf->n_vectors = VLIB_FRAME_SIZE;
 988               vlib_put_frame_queue_elt (hf);
 989               current_worker_index = ~0;
 990               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
 991               hf = 0;
 992             }
 993         }
 994       else
 995         {
 996           do_handoff = 0;
 997           /* if this is 1st frame */
 998           if (!f)
 999             {
1000               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
1001               to_next = vlib_frame_vector_args (f);
1002             }
1003
1004           to_next[0] = bi0;
1005           to_next += 1;
1006           f->n_vectors++;
1007         }
1008
1009       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1010                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1011         {
1012           snat_out2in_worker_handoff_trace_t *t =
1013             vlib_add_trace (vm, node, b0, sizeof (*t));
1014           t->next_worker_index = next_worker_index;
1015           t->do_handoff = do_handoff;
1016         }
1017     }
1018
1019   if (f)
1020     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
1021
1022   if (hf)
1023     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1024
1025   /* Ship frames to the worker nodes */
1026   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1027     {
1028       if (handoff_queue_elt_by_worker_index[i])
1029         {
1030           hf = handoff_queue_elt_by_worker_index[i];
1031           /*
1032            * It works better to let the handoff node
1033            * rate-adapt, always ship the handoff queue element.
1034            */
1035           if (1 || hf->n_vectors == hf->last_n_vectors)
1036             {
1037               vlib_put_frame_queue_elt (hf);
1038               handoff_queue_elt_by_worker_index[i] = 0;
1039             }
1040           else
1041             hf->last_n_vectors = hf->n_vectors;
1042         }
1043       congested_handoff_queue_by_worker_index[i] =
1044         (vlib_frame_queue_t *) (~0);
1045     }
1046   hf = 0;
1047   current_worker_index = ~0;
1048   return frame->n_vectors;
1049 }
1050
1051 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1052   .function = snat_out2in_worker_handoff_fn,
1053   .name = "snat-out2in-worker-handoff",
1054   .vector_size = sizeof (u32),
1055   .format_trace = format_snat_out2in_worker_handoff_trace,
1056   .type = VLIB_NODE_TYPE_INTERNAL,
1057
1058   .n_next_nodes = 1,
1059
1060   .next_nodes = {
1061     [0] = "error-drop",
1062   },
1063 };
1064
1065 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1066
1067 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1068                                     vlib_buffer_t * b0,
1069                                     ip4_header_t * ip0,
1070                                     icmp46_header_t * icmp0,
1071                                     u32 sw_if_index0,
1072                                     vlib_node_runtime_t * node,
1073                                     u32 next0,
1074                                     u32 rx_fib_index0)
1075 {
1076   snat_session_key_t key0, sm0;
1077   icmp_echo_header_t *echo0;
1078   u32 new_addr0, old_addr0;
1079   u16 old_id0, new_id0;
1080   ip_csum_t sum0;
1081   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1082
1083   echo0 = (icmp_echo_header_t *)(icmp0+1);
1084
1085   key0.addr = ip0->dst_address;
1086   key0.port = echo0->identifier;
1087   key0.fib_index = rx_fib_index0;
1088
1089   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1090     {
1091       ip4_address_t * first_int_addr;
1092
1093       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1094         {
1095           first_int_addr =
1096             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1097                                          0 /* just want the address */);
1098           rt->cached_sw_if_index = sw_if_index0;
1099           if (first_int_addr)
1100             rt->cached_ip4_address = first_int_addr->as_u32;
1101           else
1102             rt->cached_ip4_address = 0;
1103         }
1104
1105       /* Don't NAT packet aimed at the intfc address */
1106       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1107                         rt->cached_ip4_address))
1108         return next0;
1109
1110       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1111       return SNAT_OUT2IN_NEXT_DROP;
1112     }
1113
1114   new_addr0 = sm0.addr.as_u32;
1115   new_id0 = sm0.port;
1116   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1117
1118   old_addr0 = ip0->dst_address.as_u32;
1119   ip0->dst_address.as_u32 = new_addr0;
1120
1121   sum0 = ip0->checksum;
1122   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1123                          ip4_header_t,
1124                          dst_address /* changed member */);
1125   ip0->checksum = ip_csum_fold (sum0);
1126
1127   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1128     {
1129       old_id0 = echo0->identifier;
1130       echo0->identifier = new_id0;
1131
1132       sum0 = icmp0->checksum;
1133       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1134                              identifier);
1135       icmp0->checksum = ip_csum_fold (sum0);
1136     }
1137
1138   return next0;
1139 }
1140
1141 static uword
1142 snat_out2in_fast_node_fn (vlib_main_t * vm,
1143                           vlib_node_runtime_t * node,
1144                           vlib_frame_t * frame)
1145 {
1146   u32 n_left_from, * from, * to_next;
1147   snat_out2in_next_t next_index;
1148   u32 pkts_processed = 0;
1149   snat_main_t * sm = &snat_main;
1150
1151   from = vlib_frame_vector_args (frame);
1152   n_left_from = frame->n_vectors;
1153   next_index = node->cached_next_index;
1154
1155   while (n_left_from > 0)
1156     {
1157       u32 n_left_to_next;
1158
1159       vlib_get_next_frame (vm, node, next_index,
1160                            to_next, n_left_to_next);
1161
1162       while (n_left_from > 0 && n_left_to_next > 0)
1163         {
1164           u32 bi0;
1165           vlib_buffer_t * b0;
1166           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1167           u32 sw_if_index0;
1168           ip4_header_t * ip0;
1169           ip_csum_t sum0;
1170           u32 new_addr0, old_addr0;
1171           u16 new_port0, old_port0;
1172           udp_header_t * udp0;
1173           tcp_header_t * tcp0;
1174           icmp46_header_t * icmp0;
1175           snat_session_key_t key0, sm0;
1176           u32 proto0;
1177           u32 rx_fib_index0;
1178
1179           /* speculatively enqueue b0 to the current next frame */
1180           bi0 = from[0];
1181           to_next[0] = bi0;
1182           from += 1;
1183           to_next += 1;
1184           n_left_from -= 1;
1185           n_left_to_next -= 1;
1186
1187           b0 = vlib_get_buffer (vm, bi0);
1188
1189           ip0 = vlib_buffer_get_current (b0);
1190           udp0 = ip4_next_header (ip0);
1191           tcp0 = (tcp_header_t *) udp0;
1192           icmp0 = (icmp46_header_t *) udp0;
1193
1194           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1195           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1196
1197           vnet_feature_next (sw_if_index0, &next0, b0);
1198
1199           proto0 = ~0;
1200           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1201             ? SNAT_PROTOCOL_UDP : proto0;
1202           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1203             ? SNAT_PROTOCOL_TCP : proto0;
1204           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1205             ? SNAT_PROTOCOL_ICMP : proto0;
1206
1207           if (PREDICT_FALSE (proto0 == ~0))
1208               goto trace00;
1209
1210           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1211             {
1212               next0 = icmp_out2in_fast
1213                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1214               goto trace00;
1215             }
1216
1217           key0.addr = ip0->dst_address;
1218           key0.port = udp0->dst_port;
1219           key0.fib_index = rx_fib_index0;
1220
1221           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1222             {
1223               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1224               goto trace00;
1225             }
1226
1227           new_addr0 = sm0.addr.as_u32;
1228           new_port0 = sm0.port;
1229           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1230           old_addr0 = ip0->dst_address.as_u32;
1231           ip0->dst_address.as_u32 = new_addr0;
1232
1233           sum0 = ip0->checksum;
1234           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1235                                  ip4_header_t,
1236                                  dst_address /* changed member */);
1237           ip0->checksum = ip_csum_fold (sum0);
1238
1239           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1240             {
1241                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1242                 {
1243                   old_port0 = tcp0->ports.dst;
1244                   tcp0->ports.dst = new_port0;
1245
1246                   sum0 = tcp0->checksum;
1247                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1248                                          ip4_header_t,
1249                                          dst_address /* changed member */);
1250
1251                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1252                                          ip4_header_t /* cheat */,
1253                                          length /* changed member */);
1254                   tcp0->checksum = ip_csum_fold(sum0);
1255                 }
1256               else
1257                 {
1258                   old_port0 = udp0->dst_port;
1259                   udp0->dst_port = new_port0;
1260                   udp0->checksum = 0;
1261                 }
1262             }
1263           else
1264             {
1265               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1266                 {
1267                   sum0 = tcp0->checksum;
1268                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1269                                          ip4_header_t,
1270                                          dst_address /* changed member */);
1271
1272                   tcp0->checksum = ip_csum_fold(sum0);
1273                 }
1274             }
1275
1276         trace00:
1277
1278           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1279                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1280             {
1281               snat_out2in_trace_t *t =
1282                  vlib_add_trace (vm, node, b0, sizeof (*t));
1283               t->sw_if_index = sw_if_index0;
1284               t->next_index = next0;
1285             }
1286
1287           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1288
1289           /* verify speculative enqueue, maybe switch current next frame */
1290           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1291                                            to_next, n_left_to_next,
1292                                            bi0, next0);
1293         }
1294
1295       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1296     }
1297
1298   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1299                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1300                                pkts_processed);
1301   return frame->n_vectors;
1302 }
1303
1304 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1305   .function = snat_out2in_fast_node_fn,
1306   .name = "snat-out2in-fast",
1307   .vector_size = sizeof (u32),
1308   .format_trace = format_snat_out2in_fast_trace,
1309   .type = VLIB_NODE_TYPE_INTERNAL,
1310
1311   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1312   .error_strings = snat_out2in_error_strings,
1313
1314   .runtime_data_bytes = sizeof (snat_runtime_t),
1315
1316   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1317
1318   /* edit / add dispositions here */
1319   .next_nodes = {
1320     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1321     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1322   },
1323 };
1324 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);