src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30 #include <vppinfra/elog.h>
  31
  32 typedef struct {
  33   u32 sw_if_index;
  34   u32 next_index;
  35   u32 session_index;
  36 } snat_out2in_trace_t;
  37
  38 typedef struct {
  39   u32 next_worker_index;
  40   u8 do_handoff;
  41 } snat_out2in_worker_handoff_trace_t;
  42
  43 /* packet trace format function */
  44 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  45 {
  46   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  47   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  48   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  49
  50   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  51               t->sw_if_index, t->next_index, t->session_index);
  52   return s;
  53 }
  54
  55 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  56 {
  57   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  58   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  59   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  60
  61   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  62               t->sw_if_index, t->next_index);
  63   return s;
  64 }
  65
  66 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   snat_out2in_worker_handoff_trace_t * t =
  71     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  72   char * m;
  73
  74   m = t->do_handoff ? "next worker" : "same worker";
  75   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  76
  77   return s;
  78 }
  79
  80 vlib_node_registration_t snat_out2in_node;
  81 vlib_node_registration_t snat_out2in_fast_node;
  82 vlib_node_registration_t snat_out2in_worker_handoff_node;
  83
  84 #define foreach_snat_out2in_error                       \
  85 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  86 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  87 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  88 _(NO_TRANSLATION, "No translation")
  89
  90 typedef enum {
  91 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  92   foreach_snat_out2in_error
  93 #undef _
  94   SNAT_OUT2IN_N_ERROR,
  95 } snat_out2in_error_t;
  96
  97 static char * snat_out2in_error_strings[] = {
  98 #define _(sym,string) string,
  99   foreach_snat_out2in_error
 100 #undef _
 101 };
 102
 103 typedef enum {
 104   SNAT_OUT2IN_NEXT_DROP,
 105   SNAT_OUT2IN_NEXT_LOOKUP,
 106   SNAT_OUT2IN_N_NEXT,
 107 } snat_out2in_next_t;
 108
 109 /**
 110  * @brief Create session for static mapping.
 111  *
 112  * Create NAT session initiated by host from external network with static
 113  * mapping.
 114  *
 115  * @param sm     SNAT main.
 116  * @param b0     Vlib buffer.
 117  * @param in2out In2out SNAT session key.
 118  * @param out2in Out2in SNAT session key.
 119  * @param node   Vlib node.
 120  *
 121  * @returns SNAT session if successfully created otherwise 0.
 122  */
 123 static inline snat_session_t *
 124 create_session_for_static_mapping (snat_main_t *sm,
 125                                    vlib_buffer_t *b0,
 126                                    snat_session_key_t in2out,
 127                                    snat_session_key_t out2in,
 128                                    vlib_node_runtime_t * node,
 129                                    u32 cpu_index)
 130 {
 131   snat_user_t *u;
 132   snat_user_key_t user_key;
 133   snat_session_t *s;
 134   clib_bihash_kv_8_8_t kv0, value0;
 135   dlist_elt_t * per_user_translation_list_elt;
 136   dlist_elt_t * per_user_list_head_elt;
 137
 138   user_key.addr = in2out.addr;
 139   user_key.fib_index = in2out.fib_index;
 140   kv0.key = user_key.as_u64;
 141
 142   /* Ever heard of the "user" = inside ip4 address before? */
 143   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 144     {
 145       /* no, make a new one */
 146       pool_get (sm->per_thread_data[cpu_index].users, u);
 147       memset (u, 0, sizeof (*u));
 148       u->addr = in2out.addr;
 149
 150       pool_get (sm->per_thread_data[cpu_index].list_pool,
 151                 per_user_list_head_elt);
 152
 153       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 154         sm->per_thread_data[cpu_index].list_pool;
 155
 156       clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 157                        u->sessions_per_user_list_head_index);
 158
 159       kv0.value = u - sm->per_thread_data[cpu_index].users;
 160
 161       /* add user */
 162       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 163
 164       /* add non-traslated packets worker lookup */
 165       kv0.value = cpu_index;
 166       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 167     }
 168   else
 169     {
 170       u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
 171                              value0.value);
 172     }
 173
 174   pool_get (sm->per_thread_data[cpu_index].sessions, s);
 175   memset (s, 0, sizeof (*s));
 176
 177   s->outside_address_index = ~0;
 178   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 179   u->nstaticsessions++;
 180
 181   /* Create list elts */
 182   pool_get (sm->per_thread_data[cpu_index].list_pool,
 183             per_user_translation_list_elt);
 184   clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
 185                    per_user_translation_list_elt -
 186                    sm->per_thread_data[cpu_index].list_pool);
 187
 188   per_user_translation_list_elt->value =
 189     s - sm->per_thread_data[cpu_index].sessions;
 190   s->per_user_index =
 191     per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
 192   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 193
 194   clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 195                       s->per_user_list_head_index,
 196                       per_user_translation_list_elt -
 197                       sm->per_thread_data[cpu_index].list_pool);
 198
 199   s->in2out = in2out;
 200   s->out2in = out2in;
 201   s->in2out.protocol = out2in.protocol;
 202
 203   /* Add to translation hashes */
 204   kv0.key = s->in2out.as_u64;
 205   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 206   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 207       clib_warning ("in2out key add failed");
 208
 209   kv0.key = s->out2in.as_u64;
 210   kv0.value = s - sm->per_thread_data[cpu_index].sessions;
 211
 212   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 213       clib_warning ("out2in key add failed");
 214
 215   /* log NAT event */
 216   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 217                                       s->out2in.addr.as_u32,
 218                                       s->in2out.protocol,
 219                                       s->in2out.port,
 220                                       s->out2in.port,
 221                                       s->in2out.fib_index);
 222    return s;
 223 }
 224
 225 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 226                                          vlib_buffer_t * b0,
 227                                          ip4_header_t * ip0,
 228                                          icmp46_header_t * icmp0,
 229                                          u32 sw_if_index0,
 230                                          u32 rx_fib_index0,
 231                                          vlib_node_runtime_t * node,
 232                                          u32 next0, f64 now,
 233                                          u32 cpu_index)
 234 {
 235   snat_session_key_t key0, sm0;
 236   icmp_echo_header_t *echo0;
 237   clib_bihash_kv_8_8_t kv0, value0;
 238   snat_session_t * s0;
 239   u32 new_addr0, old_addr0;
 240   u16 old_id0, new_id0;
 241   ip_csum_t sum0;
 242   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 243
 244   echo0 = (icmp_echo_header_t *)(icmp0+1);
 245
 246   key0.addr = ip0->dst_address;
 247   key0.port = echo0->identifier;
 248   key0.protocol = SNAT_PROTOCOL_ICMP;
 249   key0.fib_index = rx_fib_index0;
 250
 251   kv0.key = key0.as_u64;
 252
 253   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 254     {
 255       /* Try to match static mapping by external address and port,
 256          destination address and port in packet */
 257       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 258         {
 259            ip4_address_t * first_int_addr;
 260
 261           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 262             {
 263               first_int_addr =
 264                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 265                                              0 /* just want the address */);
 266               rt->cached_sw_if_index = sw_if_index0;
 267               if (first_int_addr)
 268                 rt->cached_ip4_address = first_int_addr->as_u32;
 269               else
 270                 rt->cached_ip4_address = 0;
 271             }
 272
 273           /* Don't NAT packet aimed at the intfc address */
 274           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 275                             rt->cached_ip4_address))
 276             return next0;
 277
 278           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 279           return SNAT_OUT2IN_NEXT_DROP;
 280         }
 281
 282       /* Create session initiated by host from external network */
 283       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 284                                              node, cpu_index);
 285       if (!s0)
 286         return SNAT_OUT2IN_NEXT_DROP;
 287     }
 288   else
 289     s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 290                             value0.value);
 291
 292   old_addr0 = ip0->dst_address.as_u32;
 293   ip0->dst_address = s0->in2out.addr;
 294   new_addr0 = ip0->dst_address.as_u32;
 295   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 296
 297   sum0 = ip0->checksum;
 298   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 299                          ip4_header_t,
 300                          dst_address /* changed member */);
 301   ip0->checksum = ip_csum_fold (sum0);
 302
 303   old_id0 = echo0->identifier;
 304   new_id0 = s0->in2out.port;
 305   echo0->identifier = new_id0;
 306
 307   sum0 = icmp0->checksum;
 308   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 309                          identifier);
 310   icmp0->checksum = ip_csum_fold (sum0);
 311
 312   /* Accounting */
 313   s0->last_heard = now;
 314   s0->total_pkts++;
 315   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 316   /* Per-user LRU list maintenance for dynamic translation */
 317   if (!snat_is_session_static (s0))
 318     {
 319       clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 320                          s0->per_user_index);
 321       clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 322                           s0->per_user_list_head_index,
 323                           s0->per_user_index);
 324     }
 325
 326   return next0;
 327 }
 328
 329 static uword
 330 snat_out2in_node_fn (vlib_main_t * vm,
 331                   vlib_node_runtime_t * node,
 332                   vlib_frame_t * frame)
 333 {
 334   u32 n_left_from, * from, * to_next;
 335   snat_out2in_next_t next_index;
 336   u32 pkts_processed = 0;
 337   snat_main_t * sm = &snat_main;
 338   f64 now = vlib_time_now (vm);
 339   u32 cpu_index = os_get_cpu_number ();
 340
 341   from = vlib_frame_vector_args (frame);
 342   n_left_from = frame->n_vectors;
 343   next_index = node->cached_next_index;
 344
 345   while (n_left_from > 0)
 346     {
 347       u32 n_left_to_next;
 348
 349       vlib_get_next_frame (vm, node, next_index,
 350                            to_next, n_left_to_next);
 351
 352       while (n_left_from >= 4 && n_left_to_next >= 2)
 353         {
 354           u32 bi0, bi1;
 355           vlib_buffer_t * b0, * b1;
 356           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 357           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 358           u32 sw_if_index0, sw_if_index1;
 359           ip4_header_t * ip0, *ip1;
 360           ip_csum_t sum0, sum1;
 361           u32 new_addr0, old_addr0;
 362           u16 new_port0, old_port0;
 363           u32 new_addr1, old_addr1;
 364           u16 new_port1, old_port1;
 365           udp_header_t * udp0, * udp1;
 366           tcp_header_t * tcp0, * tcp1;
 367           icmp46_header_t * icmp0, * icmp1;
 368           snat_session_key_t key0, key1, sm0, sm1;
 369           u32 rx_fib_index0, rx_fib_index1;
 370           u32 proto0, proto1;
 371           snat_session_t * s0 = 0, * s1 = 0;
 372           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 373
 374           /* Prefetch next iteration. */
 375           {
 376             vlib_buffer_t * p2, * p3;
 377
 378             p2 = vlib_get_buffer (vm, from[2]);
 379             p3 = vlib_get_buffer (vm, from[3]);
 380
 381             vlib_prefetch_buffer_header (p2, LOAD);
 382             vlib_prefetch_buffer_header (p3, LOAD);
 383
 384             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 385             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 386           }
 387
 388           /* speculatively enqueue b0 and b1 to the current next frame */
 389           to_next[0] = bi0 = from[0];
 390           to_next[1] = bi1 = from[1];
 391           from += 2;
 392           to_next += 2;
 393           n_left_from -= 2;
 394           n_left_to_next -= 2;
 395
 396           b0 = vlib_get_buffer (vm, bi0);
 397           b1 = vlib_get_buffer (vm, bi1);
 398
 399           ip0 = vlib_buffer_get_current (b0);
 400           udp0 = ip4_next_header (ip0);
 401           tcp0 = (tcp_header_t *) udp0;
 402           icmp0 = (icmp46_header_t *) udp0;
 403
 404           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 405           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 406                                    sw_if_index0);
 407
 408           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 409
 410           if (PREDICT_FALSE (proto0 == ~0))
 411               goto trace0;
 412
 413           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 414             {
 415               next0 = icmp_out2in_slow_path
 416                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 417                  next0, now, cpu_index);
 418               goto trace0;
 419             }
 420
 421           key0.addr = ip0->dst_address;
 422           key0.port = udp0->dst_port;
 423           key0.protocol = proto0;
 424           key0.fib_index = rx_fib_index0;
 425
 426           kv0.key = key0.as_u64;
 427
 428           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 429             {
 430               /* Try to match static mapping by external address and port,
 431                  destination address and port in packet */
 432               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 433                 {
 434                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 435                   /*
 436                    * Send DHCP packets to the ipv4 stack, or we won't
 437                    * be able to use dhcp client on the outside interface
 438                    */
 439                   if (proto0 != SNAT_PROTOCOL_UDP
 440                       || (udp0->dst_port
 441                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 442                     next0 = SNAT_OUT2IN_NEXT_DROP;
 443                   goto trace0;
 444                 }
 445
 446               /* Create session initiated by host from external network */
 447               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 448                                                      cpu_index);
 449               if (!s0)
 450                 {
 451                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 452                   next0 = SNAT_OUT2IN_NEXT_DROP;
 453                   goto trace0;
 454                 }
 455             }
 456           else
 457             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 458                                     value0.value);
 459
 460           old_addr0 = ip0->dst_address.as_u32;
 461           ip0->dst_address = s0->in2out.addr;
 462           new_addr0 = ip0->dst_address.as_u32;
 463           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 464
 465           sum0 = ip0->checksum;
 466           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 467                                  ip4_header_t,
 468                                  dst_address /* changed member */);
 469           ip0->checksum = ip_csum_fold (sum0);
 470
 471           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 472             {
 473               old_port0 = tcp0->ports.dst;
 474               tcp0->ports.dst = s0->in2out.port;
 475               new_port0 = tcp0->ports.dst;
 476
 477               sum0 = tcp0->checksum;
 478               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 479                                      ip4_header_t,
 480                                      dst_address /* changed member */);
 481
 482               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 483                                      ip4_header_t /* cheat */,
 484                                      length /* changed member */);
 485               tcp0->checksum = ip_csum_fold(sum0);
 486             }
 487           else
 488             {
 489               old_port0 = udp0->dst_port;
 490               udp0->dst_port = s0->in2out.port;
 491               udp0->checksum = 0;
 492             }
 493
 494           /* Accounting */
 495           s0->last_heard = now;
 496           s0->total_pkts++;
 497           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 498           /* Per-user LRU list maintenance for dynamic translation */
 499           if (!snat_is_session_static (s0))
 500             {
 501               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 502                                  s0->per_user_index);
 503               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 504                                   s0->per_user_list_head_index,
 505                                   s0->per_user_index);
 506             }
 507         trace0:
 508
 509           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 510                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 511             {
 512               snat_out2in_trace_t *t =
 513                  vlib_add_trace (vm, node, b0, sizeof (*t));
 514               t->sw_if_index = sw_if_index0;
 515               t->next_index = next0;
 516               t->session_index = ~0;
 517               if (s0)
 518                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 519             }
 520
 521           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 522
 523
 524           ip1 = vlib_buffer_get_current (b1);
 525           udp1 = ip4_next_header (ip1);
 526           tcp1 = (tcp_header_t *) udp1;
 527           icmp1 = (icmp46_header_t *) udp1;
 528
 529           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 530           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 531                                    sw_if_index1);
 532
 533           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 534
 535           if (PREDICT_FALSE (proto1 == ~0))
 536               goto trace1;
 537
 538           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 539             {
 540               next1 = icmp_out2in_slow_path
 541                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 542                  next1, now, cpu_index);
 543               goto trace1;
 544             }
 545
 546           key1.addr = ip1->dst_address;
 547           key1.port = udp1->dst_port;
 548           key1.protocol = proto1;
 549           key1.fib_index = rx_fib_index1;
 550
 551           kv1.key = key1.as_u64;
 552
 553           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 554             {
 555               /* Try to match static mapping by external address and port,
 556                  destination address and port in packet */
 557               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 558                 {
 559                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 560                   /*
 561                    * Send DHCP packets to the ipv4 stack, or we won't
 562                    * be able to use dhcp client on the outside interface
 563                    */
 564                   if (proto1 != SNAT_PROTOCOL_UDP
 565                       || (udp1->dst_port
 566                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 567                     next1 = SNAT_OUT2IN_NEXT_DROP;
 568                   goto trace1;
 569                 }
 570
 571               /* Create session initiated by host from external network */
 572               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 573                                                      cpu_index);
 574               if (!s1)
 575                 {
 576                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 577                   next1 = SNAT_OUT2IN_NEXT_DROP;
 578                   goto trace1;
 579                 }
 580             }
 581           else
 582             s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 583                                     value1.value);
 584
 585           old_addr1 = ip1->dst_address.as_u32;
 586           ip1->dst_address = s1->in2out.addr;
 587           new_addr1 = ip1->dst_address.as_u32;
 588           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 589
 590           sum1 = ip1->checksum;
 591           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 592                                  ip4_header_t,
 593                                  dst_address /* changed member */);
 594           ip1->checksum = ip_csum_fold (sum1);
 595
 596           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 597             {
 598               old_port1 = tcp1->ports.dst;
 599               tcp1->ports.dst = s1->in2out.port;
 600               new_port1 = tcp1->ports.dst;
 601
 602               sum1 = tcp1->checksum;
 603               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 604                                      ip4_header_t,
 605                                      dst_address /* changed member */);
 606
 607               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 608                                      ip4_header_t /* cheat */,
 609                                      length /* changed member */);
 610               tcp1->checksum = ip_csum_fold(sum1);
 611             }
 612           else
 613             {
 614               old_port1 = udp1->dst_port;
 615               udp1->dst_port = s1->in2out.port;
 616               udp1->checksum = 0;
 617             }
 618
 619           /* Accounting */
 620           s1->last_heard = now;
 621           s1->total_pkts++;
 622           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 623           /* Per-user LRU list maintenance for dynamic translation */
 624           if (!snat_is_session_static (s1))
 625             {
 626               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 627                                  s1->per_user_index);
 628               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 629                                   s1->per_user_list_head_index,
 630                                   s1->per_user_index);
 631             }
 632         trace1:
 633
 634           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 635                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 636             {
 637               snat_out2in_trace_t *t =
 638                  vlib_add_trace (vm, node, b1, sizeof (*t));
 639               t->sw_if_index = sw_if_index1;
 640               t->next_index = next1;
 641               t->session_index = ~0;
 642               if (s1)
 643                 t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
 644             }
 645
 646           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 647
 648           /* verify speculative enqueues, maybe switch current next frame */
 649           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 650                                            to_next, n_left_to_next,
 651                                            bi0, bi1, next0, next1);
 652         }
 653
 654       while (n_left_from > 0 && n_left_to_next > 0)
 655         {
 656           u32 bi0;
 657           vlib_buffer_t * b0;
 658           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 659           u32 sw_if_index0;
 660           ip4_header_t * ip0;
 661           ip_csum_t sum0;
 662           u32 new_addr0, old_addr0;
 663           u16 new_port0, old_port0;
 664           udp_header_t * udp0;
 665           tcp_header_t * tcp0;
 666           icmp46_header_t * icmp0;
 667           snat_session_key_t key0, sm0;
 668           u32 rx_fib_index0;
 669           u32 proto0;
 670           snat_session_t * s0 = 0;
 671           clib_bihash_kv_8_8_t kv0, value0;
 672
 673           /* speculatively enqueue b0 to the current next frame */
 674           bi0 = from[0];
 675           to_next[0] = bi0;
 676           from += 1;
 677           to_next += 1;
 678           n_left_from -= 1;
 679           n_left_to_next -= 1;
 680
 681           b0 = vlib_get_buffer (vm, bi0);
 682
 683           ip0 = vlib_buffer_get_current (b0);
 684           udp0 = ip4_next_header (ip0);
 685           tcp0 = (tcp_header_t *) udp0;
 686           icmp0 = (icmp46_header_t *) udp0;
 687
 688           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 689           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 690                                    sw_if_index0);
 691
 692           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 693
 694           if (PREDICT_FALSE (proto0 == ~0))
 695               goto trace00;
 696
 697           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 698             {
 699               next0 = icmp_out2in_slow_path
 700                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 701                  next0, now, cpu_index);
 702               goto trace00;
 703             }
 704
 705           key0.addr = ip0->dst_address;
 706           key0.port = udp0->dst_port;
 707           key0.protocol = proto0;
 708           key0.fib_index = rx_fib_index0;
 709
 710           kv0.key = key0.as_u64;
 711
 712           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 713             {
 714               /* Try to match static mapping by external address and port,
 715                  destination address and port in packet */
 716               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 717                 {
 718                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 719                   /*
 720                    * Send DHCP packets to the ipv4 stack, or we won't
 721                    * be able to use dhcp client on the outside interface
 722                    */
 723                   if (proto0 != SNAT_PROTOCOL_UDP
 724                       || (udp0->dst_port
 725                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 726
 727                     next0 = SNAT_OUT2IN_NEXT_DROP;
 728                   goto trace00;
 729                 }
 730
 731               /* Create session initiated by host from external network */
 732               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 733                                                      cpu_index);
 734               if (!s0)
 735                 {
 736                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 737                     next0 = SNAT_OUT2IN_NEXT_DROP;
 738                   goto trace00;
 739                 }
 740             }
 741           else
 742             s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
 743                                     value0.value);
 744
 745           old_addr0 = ip0->dst_address.as_u32;
 746           ip0->dst_address = s0->in2out.addr;
 747           new_addr0 = ip0->dst_address.as_u32;
 748           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 749
 750           sum0 = ip0->checksum;
 751           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 752                                  ip4_header_t,
 753                                  dst_address /* changed member */);
 754           ip0->checksum = ip_csum_fold (sum0);
 755
 756           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 757             {
 758               old_port0 = tcp0->ports.dst;
 759               tcp0->ports.dst = s0->in2out.port;
 760               new_port0 = tcp0->ports.dst;
 761
 762               sum0 = tcp0->checksum;
 763               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 764                                      ip4_header_t,
 765                                      dst_address /* changed member */);
 766
 767               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 768                                      ip4_header_t /* cheat */,
 769                                      length /* changed member */);
 770               tcp0->checksum = ip_csum_fold(sum0);
 771             }
 772           else
 773             {
 774               old_port0 = udp0->dst_port;
 775               udp0->dst_port = s0->in2out.port;
 776               udp0->checksum = 0;
 777             }
 778
 779           /* Accounting */
 780           s0->last_heard = now;
 781           s0->total_pkts++;
 782           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 783           /* Per-user LRU list maintenance for dynamic translation */
 784           if (!snat_is_session_static (s0))
 785             {
 786               clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool,
 787                                  s0->per_user_index);
 788               clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool,
 789                                   s0->per_user_list_head_index,
 790                                   s0->per_user_index);
 791             }
 792         trace00:
 793
 794           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 795                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 796             {
 797               snat_out2in_trace_t *t =
 798                  vlib_add_trace (vm, node, b0, sizeof (*t));
 799               t->sw_if_index = sw_if_index0;
 800               t->next_index = next0;
 801               t->session_index = ~0;
 802               if (s0)
 803                 t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
 804             }
 805
 806           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 807
 808           /* verify speculative enqueue, maybe switch current next frame */
 809           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 810                                            to_next, n_left_to_next,
 811                                            bi0, next0);
 812         }
 813
 814       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 815     }
 816
 817   vlib_node_increment_counter (vm, snat_out2in_node.index,
 818                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 819                                pkts_processed);
 820   return frame->n_vectors;
 821 }
 822
 823 VLIB_REGISTER_NODE (snat_out2in_node) = {
 824   .function = snat_out2in_node_fn,
 825   .name = "snat-out2in",
 826   .vector_size = sizeof (u32),
 827   .format_trace = format_snat_out2in_trace,
 828   .type = VLIB_NODE_TYPE_INTERNAL,
 829
 830   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 831   .error_strings = snat_out2in_error_strings,
 832
 833   .runtime_data_bytes = sizeof (snat_runtime_t),
 834
 835   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 836
 837   /* edit / add dispositions here */
 838   .next_nodes = {
 839     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 840     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
 841   },
 842 };
 843 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 844
 845 static uword
 846 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
 847                                vlib_node_runtime_t * node,
 848                                vlib_frame_t * frame)
 849 {
 850   snat_main_t *sm = &snat_main;
 851   vlib_thread_main_t *tm = vlib_get_thread_main ();
 852   u32 n_left_from, *from, *to_next = 0;
 853   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
 854   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
 855     = 0;
 856   vlib_frame_queue_elt_t *hf = 0;
 857   vlib_frame_t *f = 0;
 858   int i;
 859   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
 860   u32 next_worker_index = 0;
 861   u32 current_worker_index = ~0;
 862   u32 cpu_index = os_get_cpu_number ();
 863
 864   ASSERT (vec_len (sm->workers));
 865
 866   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
 867     {
 868       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
 869
 870       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
 871                                sm->first_worker_index + sm->num_workers - 1,
 872                                (vlib_frame_queue_t *) (~0));
 873     }
 874
 875   from = vlib_frame_vector_args (frame);
 876   n_left_from = frame->n_vectors;
 877
 878   while (n_left_from > 0)
 879     {
 880       u32 bi0;
 881       vlib_buffer_t *b0;
 882       u32 sw_if_index0;
 883       u32 rx_fib_index0;
 884       ip4_header_t * ip0;
 885       udp_header_t * udp0;
 886       snat_worker_key_t key0;
 887       clib_bihash_kv_8_8_t kv0, value0;
 888       u8 do_handoff;
 889
 890       bi0 = from[0];
 891       from += 1;
 892       n_left_from -= 1;
 893
 894       b0 = vlib_get_buffer (vm, bi0);
 895
 896       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 897       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
 898
 899       ip0 = vlib_buffer_get_current (b0);
 900       udp0 = ip4_next_header (ip0);
 901
 902       key0.addr = ip0->dst_address;
 903       key0.port = udp0->dst_port;
 904       key0.fib_index = rx_fib_index0;
 905
 906       if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
 907         {
 908           icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
 909           icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 910           key0.port = echo0->identifier;
 911         }
 912
 913       kv0.key = key0.as_u64;
 914
 915       /* Ever heard of of the "user" before? */
 916       if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 917         {
 918           key0.port = 0;
 919           kv0.key = key0.as_u64;
 920
 921           if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
 922             {
 923               /* No, assign next available worker (RR) */
 924               next_worker_index = sm->first_worker_index;
 925               if (vec_len (sm->workers))
 926                 {
 927                   next_worker_index +=
 928                     sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
 929                 }
 930             }
 931           else
 932             {
 933               /* Static mapping without port */
 934               next_worker_index = value0.value;
 935             }
 936
 937           /* Add to translated packets worker lookup */
 938           kv0.value = next_worker_index;
 939           clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
 940         }
 941       else
 942         next_worker_index = value0.value;
 943
 944       if (PREDICT_FALSE (next_worker_index != cpu_index))
 945         {
 946           do_handoff = 1;
 947
 948           if (next_worker_index != current_worker_index)
 949             {
 950               if (hf)
 951                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 952
 953               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
 954                                                       next_worker_index,
 955                                                       handoff_queue_elt_by_worker_index);
 956
 957               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
 958               to_next_worker = &hf->buffer_index[hf->n_vectors];
 959               current_worker_index = next_worker_index;
 960             }
 961
 962           /* enqueue to correct worker thread */
 963           to_next_worker[0] = bi0;
 964           to_next_worker++;
 965           n_left_to_next_worker--;
 966
 967           if (n_left_to_next_worker == 0)
 968             {
 969               hf->n_vectors = VLIB_FRAME_SIZE;
 970               vlib_put_frame_queue_elt (hf);
 971               current_worker_index = ~0;
 972               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
 973               hf = 0;
 974             }
 975         }
 976       else
 977         {
 978           do_handoff = 0;
 979           /* if this is 1st frame */
 980           if (!f)
 981             {
 982               f = vlib_get_frame_to_node (vm, snat_out2in_node.index);
 983               to_next = vlib_frame_vector_args (f);
 984             }
 985
 986           to_next[0] = bi0;
 987           to_next += 1;
 988           f->n_vectors++;
 989         }
 990
 991       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 992                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 993         {
 994           snat_out2in_worker_handoff_trace_t *t =
 995             vlib_add_trace (vm, node, b0, sizeof (*t));
 996           t->next_worker_index = next_worker_index;
 997           t->do_handoff = do_handoff;
 998         }
 999     }
1000
1001   if (f)
1002     vlib_put_frame_to_node (vm, snat_out2in_node.index, f);
1003
1004   if (hf)
1005     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1006
1007   /* Ship frames to the worker nodes */
1008   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1009     {
1010       if (handoff_queue_elt_by_worker_index[i])
1011         {
1012           hf = handoff_queue_elt_by_worker_index[i];
1013           /*
1014            * It works better to let the handoff node
1015            * rate-adapt, always ship the handoff queue element.
1016            */
1017           if (1 || hf->n_vectors == hf->last_n_vectors)
1018             {
1019               vlib_put_frame_queue_elt (hf);
1020               handoff_queue_elt_by_worker_index[i] = 0;
1021             }
1022           else
1023             hf->last_n_vectors = hf->n_vectors;
1024         }
1025       congested_handoff_queue_by_worker_index[i] =
1026         (vlib_frame_queue_t *) (~0);
1027     }
1028   hf = 0;
1029   current_worker_index = ~0;
1030   return frame->n_vectors;
1031 }
1032
1033 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1034   .function = snat_out2in_worker_handoff_fn,
1035   .name = "snat-out2in-worker-handoff",
1036   .vector_size = sizeof (u32),
1037   .format_trace = format_snat_out2in_worker_handoff_trace,
1038   .type = VLIB_NODE_TYPE_INTERNAL,
1039
1040   .n_next_nodes = 1,
1041
1042   .next_nodes = {
1043     [0] = "error-drop",
1044   },
1045 };
1046
1047 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1048
1049 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1050                                     vlib_buffer_t * b0,
1051                                     ip4_header_t * ip0,
1052                                     icmp46_header_t * icmp0,
1053                                     u32 sw_if_index0,
1054                                     vlib_node_runtime_t * node,
1055                                     u32 next0,
1056                                     u32 rx_fib_index0)
1057 {
1058   snat_session_key_t key0, sm0;
1059   icmp_echo_header_t *echo0;
1060   u32 new_addr0, old_addr0;
1061   u16 old_id0, new_id0;
1062   ip_csum_t sum0;
1063   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1064
1065   echo0 = (icmp_echo_header_t *)(icmp0+1);
1066
1067   key0.addr = ip0->dst_address;
1068   key0.port = echo0->identifier;
1069   key0.fib_index = rx_fib_index0;
1070
1071   if (snat_static_mapping_match(sm, key0, &sm0, 1))
1072     {
1073       ip4_address_t * first_int_addr;
1074
1075       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1076         {
1077           first_int_addr =
1078             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1079                                          0 /* just want the address */);
1080           rt->cached_sw_if_index = sw_if_index0;
1081           if (first_int_addr)
1082             rt->cached_ip4_address = first_int_addr->as_u32;
1083           else
1084             rt->cached_ip4_address = 0;
1085         }
1086
1087       /* Don't NAT packet aimed at the intfc address */
1088       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1089                         rt->cached_ip4_address))
1090         return next0;
1091
1092       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1093       return SNAT_OUT2IN_NEXT_DROP;
1094     }
1095
1096   new_addr0 = sm0.addr.as_u32;
1097   new_id0 = sm0.port;
1098   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1099
1100   old_addr0 = ip0->dst_address.as_u32;
1101   ip0->dst_address.as_u32 = new_addr0;
1102
1103   sum0 = ip0->checksum;
1104   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1105                          ip4_header_t,
1106                          dst_address /* changed member */);
1107   ip0->checksum = ip_csum_fold (sum0);
1108
1109   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1110     {
1111       old_id0 = echo0->identifier;
1112       echo0->identifier = new_id0;
1113
1114       sum0 = icmp0->checksum;
1115       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1116                              identifier);
1117       icmp0->checksum = ip_csum_fold (sum0);
1118     }
1119
1120   return next0;
1121 }
1122
1123 static uword
1124 snat_out2in_fast_node_fn (vlib_main_t * vm,
1125                           vlib_node_runtime_t * node,
1126                           vlib_frame_t * frame)
1127 {
1128   u32 n_left_from, * from, * to_next;
1129   snat_out2in_next_t next_index;
1130   u32 pkts_processed = 0;
1131   snat_main_t * sm = &snat_main;
1132
1133   from = vlib_frame_vector_args (frame);
1134   n_left_from = frame->n_vectors;
1135   next_index = node->cached_next_index;
1136
1137   while (n_left_from > 0)
1138     {
1139       u32 n_left_to_next;
1140
1141       vlib_get_next_frame (vm, node, next_index,
1142                            to_next, n_left_to_next);
1143
1144       while (n_left_from > 0 && n_left_to_next > 0)
1145         {
1146           u32 bi0;
1147           vlib_buffer_t * b0;
1148           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1149           u32 sw_if_index0;
1150           ip4_header_t * ip0;
1151           ip_csum_t sum0;
1152           u32 new_addr0, old_addr0;
1153           u16 new_port0, old_port0;
1154           udp_header_t * udp0;
1155           tcp_header_t * tcp0;
1156           icmp46_header_t * icmp0;
1157           snat_session_key_t key0, sm0;
1158           u32 proto0;
1159           u32 rx_fib_index0;
1160
1161           /* speculatively enqueue b0 to the current next frame */
1162           bi0 = from[0];
1163           to_next[0] = bi0;
1164           from += 1;
1165           to_next += 1;
1166           n_left_from -= 1;
1167           n_left_to_next -= 1;
1168
1169           b0 = vlib_get_buffer (vm, bi0);
1170
1171           ip0 = vlib_buffer_get_current (b0);
1172           udp0 = ip4_next_header (ip0);
1173           tcp0 = (tcp_header_t *) udp0;
1174           icmp0 = (icmp46_header_t *) udp0;
1175
1176           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1177           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1178
1179           vnet_feature_next (sw_if_index0, &next0, b0);
1180
1181           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1182
1183           if (PREDICT_FALSE (proto0 == ~0))
1184               goto trace00;
1185
1186           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1187             {
1188               next0 = icmp_out2in_fast
1189                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1190               goto trace00;
1191             }
1192
1193           key0.addr = ip0->dst_address;
1194           key0.port = udp0->dst_port;
1195           key0.fib_index = rx_fib_index0;
1196
1197           if (snat_static_mapping_match(sm, key0, &sm0, 1))
1198             {
1199               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1200               goto trace00;
1201             }
1202
1203           new_addr0 = sm0.addr.as_u32;
1204           new_port0 = sm0.port;
1205           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1206           old_addr0 = ip0->dst_address.as_u32;
1207           ip0->dst_address.as_u32 = new_addr0;
1208
1209           sum0 = ip0->checksum;
1210           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1211                                  ip4_header_t,
1212                                  dst_address /* changed member */);
1213           ip0->checksum = ip_csum_fold (sum0);
1214
1215           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1216             {
1217                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1218                 {
1219                   old_port0 = tcp0->ports.dst;
1220                   tcp0->ports.dst = new_port0;
1221
1222                   sum0 = tcp0->checksum;
1223                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1224                                          ip4_header_t,
1225                                          dst_address /* changed member */);
1226
1227                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1228                                          ip4_header_t /* cheat */,
1229                                          length /* changed member */);
1230                   tcp0->checksum = ip_csum_fold(sum0);
1231                 }
1232               else
1233                 {
1234                   old_port0 = udp0->dst_port;
1235                   udp0->dst_port = new_port0;
1236                   udp0->checksum = 0;
1237                 }
1238             }
1239           else
1240             {
1241               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1242                 {
1243                   sum0 = tcp0->checksum;
1244                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1245                                          ip4_header_t,
1246                                          dst_address /* changed member */);
1247
1248                   tcp0->checksum = ip_csum_fold(sum0);
1249                 }
1250             }
1251
1252         trace00:
1253
1254           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1255                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1256             {
1257               snat_out2in_trace_t *t =
1258                  vlib_add_trace (vm, node, b0, sizeof (*t));
1259               t->sw_if_index = sw_if_index0;
1260               t->next_index = next0;
1261             }
1262
1263           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1264
1265           /* verify speculative enqueue, maybe switch current next frame */
1266           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1267                                            to_next, n_left_to_next,
1268                                            bi0, next0);
1269         }
1270
1271       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1272     }
1273
1274   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1275                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1276                                pkts_processed);
1277   return frame->n_vectors;
1278 }
1279
1280 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1281   .function = snat_out2in_fast_node_fn,
1282   .name = "snat-out2in-fast",
1283   .vector_size = sizeof (u32),
1284   .format_trace = format_snat_out2in_fast_trace,
1285   .type = VLIB_NODE_TYPE_INTERNAL,
1286
1287   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1288   .error_strings = snat_out2in_error_strings,
1289
1290   .runtime_data_bytes = sizeof (snat_runtime_t),
1291
1292   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1293
1294   /* edit / add dispositions here */
1295   .next_nodes = {
1296     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1297     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1298   },
1299 };
1300 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);