plugins/snat-plugin/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <snat/snat.h>
  24
  25 #include <vppinfra/hash.h>
  26 #include <vppinfra/error.h>
  27 #include <vppinfra/elog.h>
  28
  29 typedef struct {
  30   u32 sw_if_index;
  31   u32 next_index;
  32   u32 session_index;
  33   u32 is_slow_path;
  34 } snat_in2out_trace_t;
  35
  36 /* packet trace format function */
  37 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  38 {
  39   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  40   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  41   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  42   char * tag;
  43
  44   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  45
  46   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  47               t->sw_if_index, t->next_index, t->session_index);
  48
  49   return s;
  50 }
  51
  52 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  53 {
  54   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  55   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  56   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  57
  58   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  59               t->sw_if_index, t->next_index);
  60
  61   return s;
  62 }
  63
  64 vlib_node_registration_t snat_in2out_node;
  65 vlib_node_registration_t snat_in2out_slowpath_node;
  66 vlib_node_registration_t snat_in2out_fast_node;
  67
  68 #define foreach_snat_in2out_error                       \
  69 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  70 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  71 _(OUT_OF_PORTS, "Out of ports")                         \
  72 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  73 _(BAD_ICMP_TYPE, "icmp type not echo-request")          \
  74 _(NO_TRANSLATION, "No translation")
  75
  76 typedef enum {
  77 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  78   foreach_snat_in2out_error
  79 #undef _
  80   SNAT_IN2OUT_N_ERROR,
  81 } snat_in2out_error_t;
  82
  83 static char * snat_in2out_error_strings[] = {
  84 #define _(sym,string) string,
  85   foreach_snat_in2out_error
  86 #undef _
  87 };
  88
  89 typedef enum {
  90   SNAT_IN2OUT_NEXT_LOOKUP,
  91   SNAT_IN2OUT_NEXT_DROP,
  92   SNAT_IN2OUT_NEXT_SLOW_PATH,
  93   SNAT_IN2OUT_N_NEXT,
  94 } snat_in2out_next_t;
  95
  96
  97 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
  98                       ip4_header_t * ip0,
  99                       u32 rx_fib_index0,
 100                       snat_session_key_t * key0,
 101                       snat_session_t ** sessionp,
 102                       vlib_node_runtime_t * node,
 103                       u32 next0)
 104 {
 105   snat_user_t *u;
 106   snat_user_key_t user_key;
 107   snat_session_t *s;
 108   clib_bihash_kv_8_8_t kv0, value0;
 109   u32 oldest_per_user_translation_list_index;
 110   dlist_elt_t * oldest_per_user_translation_list_elt;
 111   dlist_elt_t * per_user_translation_list_elt;
 112   dlist_elt_t * per_user_list_head_elt;
 113   u32 session_index;
 114   snat_session_key_t key1;
 115   u32 address_index = ~0;
 116   u32 outside_fib_index;
 117   uword * p;
 118
 119   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 120   if (! p)
 121     {
 122       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 123       return SNAT_IN2OUT_NEXT_DROP;
 124     }
 125   outside_fib_index = p[0];
 126
 127   user_key.addr = ip0->src_address;
 128   user_key.fib_index = rx_fib_index0;
 129   kv0.key = user_key.as_u64;
 130
 131   /* Ever heard of the "user" = src ip4 address before? */
 132   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 133     {
 134       /* no, make a new one */
 135       pool_get (sm->users, u);
 136       memset (u, 0, sizeof (*u));
 137       u->addr = ip0->src_address;
 138
 139       pool_get (sm->list_pool, per_user_list_head_elt);
 140
 141       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 142         sm->list_pool;
 143
 144       clib_dlist_init (sm->list_pool, u->sessions_per_user_list_head_index);
 145
 146       kv0.value = u - sm->users;
 147
 148       /* add user */
 149       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 150     }
 151   else
 152     {
 153       u = pool_elt_at_index (sm->users, value0.value);
 154     }
 155
 156   /* Over quota? Recycle the least recently used dynamic translation */
 157   if (u->nsessions >= sm->max_translations_per_user)
 158     {
 159       /* Remove the oldest dynamic translation */
 160       do {
 161           oldest_per_user_translation_list_index =
 162             clib_dlist_remove_head
 163             (sm->list_pool, u->sessions_per_user_list_head_index);
 164
 165           ASSERT (oldest_per_user_translation_list_index != ~0);
 166
 167           /* add it back to the end of the LRU list */
 168           clib_dlist_addtail (sm->list_pool,
 169                               u->sessions_per_user_list_head_index,
 170                               oldest_per_user_translation_list_index);
 171           /* Get the list element */
 172           oldest_per_user_translation_list_elt =
 173             pool_elt_at_index (sm->list_pool,
 174                                oldest_per_user_translation_list_index);
 175
 176           /* Get the session index from the list element */
 177           session_index = oldest_per_user_translation_list_elt->value;
 178
 179           /* Get the session */
 180           s = pool_elt_at_index (sm->sessions, session_index);
 181       } while (!snat_is_session_static (s));
 182
 183       /* Remove in2out, out2in keys */
 184       kv0.key = s->in2out.as_u64;
 185       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 186           clib_warning ("in2out key delete failed");
 187       kv0.key = s->out2in.as_u64;
 188       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 189           clib_warning ("out2in key delete failed");
 190
 191       snat_free_outside_address_and_port
 192         (sm, &s->out2in, s->outside_address_index);
 193       s->outside_address_index = ~0;
 194
 195       if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 196         {
 197           ASSERT(0);
 198
 199           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 200           return SNAT_IN2OUT_NEXT_DROP;
 201         }
 202       s->outside_address_index = address_index;
 203     }
 204   else
 205     {
 206       u8 static_mapping = 1;
 207
 208       /* First try to match static mapping by local address and port */
 209       if (snat_static_mapping_match (sm, *key0, &key1, 0))
 210         {
 211           static_mapping = 0;
 212           /* Try to create dynamic translation */
 213           if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 214             {
 215               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 216               return SNAT_IN2OUT_NEXT_DROP;
 217             }
 218         }
 219
 220       /* Create a new session */
 221       pool_get (sm->sessions, s);
 222       memset (s, 0, sizeof (*s));
 223
 224       s->outside_address_index = address_index;
 225
 226       if (static_mapping)
 227         {
 228           u->nstaticsessions++;
 229           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 230         }
 231       else
 232         {
 233           u->nsessions++;
 234         }
 235
 236       /* Create list elts */
 237       pool_get (sm->list_pool, per_user_translation_list_elt);
 238       clib_dlist_init (sm->list_pool, per_user_translation_list_elt -
 239                        sm->list_pool);
 240
 241       per_user_translation_list_elt->value = s - sm->sessions;
 242       s->per_user_index = per_user_translation_list_elt - sm->list_pool;
 243       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 244
 245       clib_dlist_addtail (sm->list_pool, s->per_user_list_head_index,
 246                           per_user_translation_list_elt - sm->list_pool);
 247    }
 248
 249   s->in2out = *key0;
 250   s->out2in = key1;
 251   s->out2in.protocol = key0->protocol;
 252   s->out2in.fib_index = outside_fib_index;
 253   *sessionp = s;
 254
 255   /* Add to translation hashes */
 256   kv0.key = s->in2out.as_u64;
 257   kv0.value = s - sm->sessions;
 258   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 259       clib_warning ("in2out key add failed");
 260
 261   kv0.key = s->out2in.as_u64;
 262   kv0.value = s - sm->sessions;
 263
 264   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 265       clib_warning ("out2in key add failed");
 266
 267   return next0;
 268 }
 269
 270 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 271                                          vlib_buffer_t * b0,
 272                                          ip4_header_t * ip0,
 273                                          icmp46_header_t * icmp0,
 274                                          u32 sw_if_index0,
 275                                          u32 rx_fib_index0,
 276                                          vlib_node_runtime_t * node,
 277                                          u32 next0,
 278                                          f64 now)
 279 {
 280   snat_session_key_t key0;
 281   icmp_echo_header_t *echo0;
 282   clib_bihash_kv_8_8_t kv0, value0;
 283   snat_session_t * s0;
 284   u32 new_addr0, old_addr0;
 285   u16 old_id0, new_id0;
 286   ip_csum_t sum0;
 287   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 288
 289   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request))
 290     {
 291       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 292       return SNAT_IN2OUT_NEXT_DROP;
 293     }
 294
 295   echo0 = (icmp_echo_header_t *)(icmp0+1);
 296
 297   key0.addr = ip0->src_address;
 298   key0.port = echo0->identifier;
 299   key0.protocol = SNAT_PROTOCOL_ICMP;
 300   key0.fib_index = rx_fib_index0;
 301
 302   kv0.key = key0.as_u64;
 303
 304   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 305     {
 306       ip4_address_t * first_int_addr;
 307
 308       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 309         {
 310           first_int_addr =
 311             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 312                                          0 /* just want the address */);
 313           rt->cached_sw_if_index = sw_if_index0;
 314           rt->cached_ip4_address = first_int_addr->as_u32;
 315         }
 316
 317       /* Don't NAT packet aimed at the intfc address */
 318       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 319                                 rt->cached_ip4_address))
 320         return next0;
 321
 322       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 323                          &s0, node, next0);
 324
 325       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 326         return next0;
 327     }
 328   else
 329     s0 = pool_elt_at_index (sm->sessions, value0.value);
 330
 331   old_addr0 = ip0->src_address.as_u32;
 332   ip0->src_address = s0->out2in.addr;
 333   new_addr0 = ip0->src_address.as_u32;
 334   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 335
 336   sum0 = ip0->checksum;
 337   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 338                          ip4_header_t,
 339                          src_address /* changed member */);
 340   ip0->checksum = ip_csum_fold (sum0);
 341
 342   old_id0 = echo0->identifier;
 343   new_id0 = s0->out2in.port;
 344   echo0->identifier = new_id0;
 345
 346   sum0 = icmp0->checksum;
 347   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 348                          identifier);
 349   icmp0->checksum = ip_csum_fold (sum0);
 350
 351   /* Accounting */
 352   s0->last_heard = now;
 353   s0->total_pkts++;
 354   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 355   /* Per-user LRU list maintenance for dynamic translations */
 356   if (!snat_is_session_static (s0))
 357     {
 358       clib_dlist_remove (sm->list_pool, s0->per_user_index);
 359       clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 360                           s0->per_user_index);
 361     }
 362
 363   return next0;
 364 }
 365
 366 static inline uword
 367 snat_in2out_node_fn_inline (vlib_main_t * vm,
 368                             vlib_node_runtime_t * node,
 369                             vlib_frame_t * frame, int is_slow_path)
 370 {
 371   u32 n_left_from, * from, * to_next;
 372   snat_in2out_next_t next_index;
 373   u32 pkts_processed = 0;
 374   snat_main_t * sm = &snat_main;
 375   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 376   f64 now = vlib_time_now (vm);
 377   u32 stats_node_index;
 378
 379   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 380     snat_in2out_node.index;
 381
 382   from = vlib_frame_vector_args (frame);
 383   n_left_from = frame->n_vectors;
 384   next_index = node->cached_next_index;
 385
 386   while (n_left_from > 0)
 387     {
 388       u32 n_left_to_next;
 389
 390       vlib_get_next_frame (vm, node, next_index,
 391                            to_next, n_left_to_next);
 392
 393       while (n_left_from >= 4 && n_left_to_next >= 2)
 394         {
 395           u32 bi0, bi1;
 396           vlib_buffer_t * b0, * b1;
 397           u32 next0, next1;
 398           u32 sw_if_index0, sw_if_index1;
 399           ip4_header_t * ip0, * ip1;
 400           ip_csum_t sum0, sum1;
 401           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 402           u16 old_port0, new_port0, old_port1, new_port1;
 403           udp_header_t * udp0, * udp1;
 404           tcp_header_t * tcp0, * tcp1;
 405           icmp46_header_t * icmp0, * icmp1;
 406           snat_session_key_t key0, key1;
 407           u32 rx_fib_index0, rx_fib_index1;
 408           u32 proto0, proto1;
 409           snat_session_t * s0 = 0, * s1 = 0;
 410           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 411
 412           /* Prefetch next iteration. */
 413           {
 414             vlib_buffer_t * p2, * p3;
 415
 416             p2 = vlib_get_buffer (vm, from[2]);
 417             p3 = vlib_get_buffer (vm, from[3]);
 418
 419             vlib_prefetch_buffer_header (p2, LOAD);
 420             vlib_prefetch_buffer_header (p3, LOAD);
 421
 422             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 423             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 424           }
 425
 426           /* speculatively enqueue b0 and b1 to the current next frame */
 427           to_next[0] = bi0 = from[0];
 428           to_next[1] = bi1 = from[1];
 429           from += 2;
 430           to_next += 2;
 431           n_left_from -= 2;
 432           n_left_to_next -= 2;
 433
 434           b0 = vlib_get_buffer (vm, bi0);
 435           b1 = vlib_get_buffer (vm, bi1);
 436
 437           ip0 = vlib_buffer_get_current (b0);
 438           udp0 = ip4_next_header (ip0);
 439           tcp0 = (tcp_header_t *) udp0;
 440           icmp0 = (icmp46_header_t *) udp0;
 441
 442           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 443           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 444                                    sw_if_index0);
 445
 446           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 447
 448 #if 0
 449           /* Formally correct, but we send to slowpath, lookup or drop */
 450           vnet_get_config_data (&cm->config_main,
 451                                 &b0->current_config_index,
 452                                 &next0,
 453                                 0 /* sizeof config data */);
 454 #endif
 455
 456           proto0 = ~0;
 457           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 458             ? SNAT_PROTOCOL_UDP : proto0;
 459           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 460             ? SNAT_PROTOCOL_TCP : proto0;
 461           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 462             ? SNAT_PROTOCOL_ICMP : proto0;
 463
 464           /* Next configured feature, probably ip4-lookup */
 465           if (is_slow_path)
 466             {
 467               if (PREDICT_FALSE (proto0 == ~0))
 468                 goto trace00;
 469
 470               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 471                 {
 472                   next0 = icmp_in2out_slow_path
 473                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 474                      node, next0, now);
 475                   goto trace00;
 476                 }
 477             }
 478           else
 479             {
 480               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 481                 {
 482                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 483                   goto trace00;
 484                 }
 485             }
 486
 487           key0.addr = ip0->src_address;
 488           key0.port = udp0->src_port;
 489           key0.protocol = proto0;
 490           key0.fib_index = rx_fib_index0;
 491
 492           kv0.key = key0.as_u64;
 493
 494           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
 495             {
 496               if (is_slow_path)
 497                 {
 498                   ip4_address_t * first_int_addr;
 499
 500                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 501                     {
 502                       first_int_addr =
 503                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 504                                                      0 /* just want the address */);
 505                       rt->cached_sw_if_index = sw_if_index0;
 506                       rt->cached_ip4_address = first_int_addr->as_u32;
 507                     }
 508
 509                   /* Don't NAT packet aimed at the intfc address */
 510                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 511                                     rt->cached_ip4_address))
 512                     goto trace00;
 513
 514                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 515                                      &s0, node, next0);
 516                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 517                     goto trace00;
 518                 }
 519               else
 520                 {
 521                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 522                   goto trace00;
 523                 }
 524             }
 525           else
 526             s0 = pool_elt_at_index (sm->sessions, value0.value);
 527
 528           old_addr0 = ip0->src_address.as_u32;
 529           ip0->src_address = s0->out2in.addr;
 530           new_addr0 = ip0->src_address.as_u32;
 531           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 532
 533           sum0 = ip0->checksum;
 534           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 535                                  ip4_header_t,
 536                                  src_address /* changed member */);
 537           ip0->checksum = ip_csum_fold (sum0);
 538
 539           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 540             {
 541               old_port0 = tcp0->ports.src;
 542               tcp0->ports.src = s0->out2in.port;
 543               new_port0 = tcp0->ports.src;
 544
 545               sum0 = tcp0->checksum;
 546               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 547                                      ip4_header_t,
 548                                      dst_address /* changed member */);
 549               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 550                                      ip4_header_t /* cheat */,
 551                                      length /* changed member */);
 552               tcp0->checksum = ip_csum_fold(sum0);
 553             }
 554           else
 555             {
 556               old_port0 = udp0->src_port;
 557               udp0->src_port = s0->out2in.port;
 558               udp0->checksum = 0;
 559             }
 560
 561           /* Accounting */
 562           s0->last_heard = now;
 563           s0->total_pkts++;
 564           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 565           /* Per-user LRU list maintenance for dynamic translation */
 566           if (!snat_is_session_static (s0))
 567             {
 568               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 569               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 570                                   s0->per_user_index);
 571             }
 572         trace00:
 573
 574           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 575                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 576             {
 577               snat_in2out_trace_t *t =
 578                  vlib_add_trace (vm, node, b0, sizeof (*t));
 579               t->is_slow_path = is_slow_path;
 580               t->sw_if_index = sw_if_index0;
 581               t->next_index = next0;
 582                   t->session_index = ~0;
 583               if (s0)
 584                   t->session_index = s0 - sm->sessions;
 585             }
 586
 587           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 588
 589           ip1 = vlib_buffer_get_current (b1);
 590           udp1 = ip4_next_header (ip1);
 591           tcp1 = (tcp_header_t *) udp1;
 592           icmp1 = (icmp46_header_t *) udp1;
 593
 594           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 595           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 596                                    sw_if_index1);
 597
 598 #if 0
 599           vnet_get_config_data (&cm->config_main,
 600                                 &b1->current_config_index,
 601                                 &next1,
 602                                 0 /* sizeof config data */);
 603 #endif
 604
 605           proto1 = ~0;
 606           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 607             ? SNAT_PROTOCOL_UDP : proto1;
 608           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 609             ? SNAT_PROTOCOL_TCP : proto1;
 610           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 611             ? SNAT_PROTOCOL_ICMP : proto1;
 612
 613           /* Next configured feature, probably ip4-lookup */
 614           if (is_slow_path)
 615             {
 616               if (PREDICT_FALSE (proto1 == ~0))
 617                 goto trace01;
 618
 619               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 620                 {
 621                   next1 = icmp_in2out_slow_path
 622                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, next1,
 623                      now);
 624                   goto trace01;
 625                 }
 626             }
 627           else
 628             {
 629               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
 630                 {
 631                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 632                   goto trace01;
 633                 }
 634             }
 635
 636           key1.addr = ip1->src_address;
 637           key1.port = udp1->src_port;
 638           key1.protocol = proto1;
 639           key1.fib_index = rx_fib_index1;
 640
 641           kv1.key = key1.as_u64;
 642
 643             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
 644             {
 645               if (is_slow_path)
 646                 {
 647                   ip4_address_t * first_int_addr;
 648
 649                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index1))
 650                     {
 651                       first_int_addr =
 652                         ip4_interface_first_address (sm->ip4_main, sw_if_index1,
 653                                                      0 /* just want the address */);
 654                       rt->cached_sw_if_index = sw_if_index1;
 655                       rt->cached_ip4_address = first_int_addr->as_u32;
 656                     }
 657
 658                   /* Don't NAT packet aimed at the intfc address */
 659                   if (PREDICT_FALSE(ip1->dst_address.as_u32 ==
 660                                     rt->cached_ip4_address))
 661                     goto trace01;
 662
 663                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
 664                                      &s1, node, next1);
 665                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
 666                     goto trace01;
 667                 }
 668               else
 669                 {
 670                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 671                   goto trace01;
 672                 }
 673             }
 674           else
 675             s1 = pool_elt_at_index (sm->sessions, value1.value);
 676
 677           old_addr1 = ip1->src_address.as_u32;
 678           ip1->src_address = s1->out2in.addr;
 679           new_addr1 = ip1->src_address.as_u32;
 680           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
 681
 682           sum1 = ip1->checksum;
 683           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 684                                  ip4_header_t,
 685                                  src_address /* changed member */);
 686           ip1->checksum = ip_csum_fold (sum1);
 687
 688           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 689             {
 690               old_port1 = tcp1->ports.src;
 691               tcp1->ports.src = s1->out2in.port;
 692               new_port1 = tcp1->ports.src;
 693
 694               sum1 = tcp1->checksum;
 695               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 696                                      ip4_header_t,
 697                                      dst_address /* changed member */);
 698               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 699                                      ip4_header_t /* cheat */,
 700                                      length /* changed member */);
 701               tcp1->checksum = ip_csum_fold(sum1);
 702             }
 703           else
 704             {
 705               old_port1 = udp1->src_port;
 706               udp1->src_port = s1->out2in.port;
 707               udp1->checksum = 0;
 708             }
 709
 710           /* Accounting */
 711           s1->last_heard = now;
 712           s1->total_pkts++;
 713           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 714           /* Per-user LRU list maintenance for dynamic translation */
 715           if (!snat_is_session_static (s1))
 716             {
 717               clib_dlist_remove (sm->list_pool, s1->per_user_index);
 718               clib_dlist_addtail (sm->list_pool, s1->per_user_list_head_index,
 719                                   s1->per_user_index);
 720             }
 721         trace01:
 722
 723           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 724                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 725             {
 726               snat_in2out_trace_t *t =
 727                  vlib_add_trace (vm, node, b1, sizeof (*t));
 728               t->sw_if_index = sw_if_index1;
 729               t->next_index = next1;
 730               t->session_index = ~0;
 731               if (s1)
 732                 t->session_index = s1 - sm->sessions;
 733             }
 734
 735           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
 736
 737           /* verify speculative enqueues, maybe switch current next frame */
 738           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 739                                            to_next, n_left_to_next,
 740                                            bi0, bi1, next0, next1);
 741         }
 742
 743       while (n_left_from > 0 && n_left_to_next > 0)
 744         {
 745           u32 bi0;
 746           vlib_buffer_t * b0;
 747           u32 next0;
 748           u32 sw_if_index0;
 749           ip4_header_t * ip0;
 750           ip_csum_t sum0;
 751           u32 new_addr0, old_addr0;
 752           u16 old_port0, new_port0;
 753           udp_header_t * udp0;
 754           tcp_header_t * tcp0;
 755           icmp46_header_t * icmp0;
 756           snat_session_key_t key0;
 757           u32 rx_fib_index0;
 758           u32 proto0;
 759           snat_session_t * s0 = 0;
 760           clib_bihash_kv_8_8_t kv0, value0;
 761
 762           /* speculatively enqueue b0 to the current next frame */
 763           bi0 = from[0];
 764           to_next[0] = bi0;
 765           from += 1;
 766           to_next += 1;
 767           n_left_from -= 1;
 768           n_left_to_next -= 1;
 769
 770           b0 = vlib_get_buffer (vm, bi0);
 771           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
 772
 773           ip0 = vlib_buffer_get_current (b0);
 774           udp0 = ip4_next_header (ip0);
 775           tcp0 = (tcp_header_t *) udp0;
 776           icmp0 = (icmp46_header_t *) udp0;
 777
 778           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 779           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 780                                    sw_if_index0);
 781
 782
 783 #if 0
 784           vnet_get_config_data (&cm->config_main,
 785                                 &b0->current_config_index,
 786                                 &next0,
 787                                 0 /* sizeof config data */);
 788 #endif
 789
 790           proto0 = ~0;
 791           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 792             ? SNAT_PROTOCOL_UDP : proto0;
 793           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 794             ? SNAT_PROTOCOL_TCP : proto0;
 795           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 796             ? SNAT_PROTOCOL_ICMP : proto0;
 797
 798           /* Next configured feature, probably ip4-lookup */
 799           if (is_slow_path)
 800             {
 801               if (PREDICT_FALSE (proto0 == ~0))
 802                 goto trace0;
 803
 804               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 805                 {
 806                   next0 = icmp_in2out_slow_path
 807                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0,
 808                      now);
 809                   goto trace0;
 810                 }
 811             }
 812           else
 813             {
 814               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 815                 {
 816                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 817                   goto trace0;
 818                 }
 819             }
 820
 821           key0.addr = ip0->src_address;
 822           key0.port = udp0->src_port;
 823           key0.protocol = proto0;
 824           key0.fib_index = rx_fib_index0;
 825
 826           kv0.key = key0.as_u64;
 827
 828           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 829             {
 830               if (is_slow_path)
 831                 {
 832                   ip4_address_t * first_int_addr;
 833
 834                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 835                     {
 836                       first_int_addr =
 837                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 838                                                      0 /* just want the address */);
 839                       rt->cached_sw_if_index = sw_if_index0;
 840                       rt->cached_ip4_address = first_int_addr->as_u32;
 841                     }
 842
 843                   /* Don't NAT packet aimed at the intfc address */
 844                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 845                                     rt->cached_ip4_address))
 846                     goto trace0;
 847
 848                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 849                                      &s0, node, next0);
 850                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 851                     goto trace0;
 852                 }
 853               else
 854                 {
 855                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 856                   goto trace0;
 857                 }
 858             }
 859           else
 860             s0 = pool_elt_at_index (sm->sessions, value0.value);
 861
 862           old_addr0 = ip0->src_address.as_u32;
 863           ip0->src_address = s0->out2in.addr;
 864           new_addr0 = ip0->src_address.as_u32;
 865           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 866
 867           sum0 = ip0->checksum;
 868           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 869                                  ip4_header_t,
 870                                  src_address /* changed member */);
 871           ip0->checksum = ip_csum_fold (sum0);
 872
 873           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 874             {
 875               old_port0 = tcp0->ports.src;
 876               tcp0->ports.src = s0->out2in.port;
 877               new_port0 = tcp0->ports.src;
 878
 879               sum0 = tcp0->checksum;
 880               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 881                                      ip4_header_t,
 882                                      dst_address /* changed member */);
 883               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 884                                      ip4_header_t /* cheat */,
 885                                      length /* changed member */);
 886               tcp0->checksum = ip_csum_fold(sum0);
 887             }
 888           else
 889             {
 890               old_port0 = udp0->src_port;
 891               udp0->src_port = s0->out2in.port;
 892               udp0->checksum = 0;
 893             }
 894
 895           /* Accounting */
 896           s0->last_heard = now;
 897           s0->total_pkts++;
 898           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 899           /* Per-user LRU list maintenance for dynamic translation */
 900           if (!snat_is_session_static (s0))
 901             {
 902               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 903               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 904                                   s0->per_user_index);
 905             }
 906
 907         trace0:
 908           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 909                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 910             {
 911               snat_in2out_trace_t *t =
 912                  vlib_add_trace (vm, node, b0, sizeof (*t));
 913               t->is_slow_path = is_slow_path;
 914               t->sw_if_index = sw_if_index0;
 915               t->next_index = next0;
 916                   t->session_index = ~0;
 917               if (s0)
 918                   t->session_index = s0 - sm->sessions;
 919             }
 920
 921           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 922
 923           /* verify speculative enqueue, maybe switch current next frame */
 924           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 925                                            to_next, n_left_to_next,
 926                                            bi0, next0);
 927         }
 928
 929       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 930     }
 931
 932   vlib_node_increment_counter (vm, stats_node_index,
 933                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
 934                                pkts_processed);
 935   return frame->n_vectors;
 936 }
 937
 938 static uword
 939 snat_in2out_fast_path_fn (vlib_main_t * vm,
 940                           vlib_node_runtime_t * node,
 941                           vlib_frame_t * frame)
 942 {
 943   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
 944 }
 945
 946 VLIB_REGISTER_NODE (snat_in2out_node) = {
 947   .function = snat_in2out_fast_path_fn,
 948   .name = "snat-in2out",
 949   .vector_size = sizeof (u32),
 950   .format_trace = format_snat_in2out_trace,
 951   .type = VLIB_NODE_TYPE_INTERNAL,
 952
 953   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 954   .error_strings = snat_in2out_error_strings,
 955
 956   .runtime_data_bytes = sizeof (snat_runtime_t),
 957
 958   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 959
 960   /* edit / add dispositions here */
 961   .next_nodes = {
 962     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 963     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 964     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 965   },
 966 };
 967
 968 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
 969
 970 static uword
 971 snat_in2out_slow_path_fn (vlib_main_t * vm,
 972                           vlib_node_runtime_t * node,
 973                           vlib_frame_t * frame)
 974 {
 975   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
 976 }
 977
 978 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
 979   .function = snat_in2out_slow_path_fn,
 980   .name = "snat-in2out-slowpath",
 981   .vector_size = sizeof (u32),
 982   .format_trace = format_snat_in2out_trace,
 983   .type = VLIB_NODE_TYPE_INTERNAL,
 984
 985   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 986   .error_strings = snat_in2out_error_strings,
 987
 988   .runtime_data_bytes = sizeof (snat_runtime_t),
 989
 990   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 991
 992   /* edit / add dispositions here */
 993   .next_nodes = {
 994     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 995     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 996     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 997   },
 998 };
 999
1000 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1001
1002 static inline u32 icmp_in2out_static_map (snat_main_t *sm,
1003                                           vlib_buffer_t * b0,
1004                                           ip4_header_t * ip0,
1005                                           icmp46_header_t * icmp0,
1006                                           u32 sw_if_index0,
1007                                           vlib_node_runtime_t * node,
1008                                           u32 next0,
1009                                           u32 rx_fib_index0)
1010 {
1011   snat_session_key_t key0, sm0;
1012   icmp_echo_header_t *echo0;
1013   u32 new_addr0, old_addr0;
1014   u16 old_id0, new_id0;
1015   ip_csum_t sum0;
1016   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1017
1018   echo0 = (icmp_echo_header_t *)(icmp0+1);
1019
1020   key0.addr = ip0->src_address;
1021   key0.port = echo0->identifier;
1022   key0.fib_index = rx_fib_index0;
1023
1024   if (snat_static_mapping_match(sm, key0, &sm0, 0))
1025     {
1026       ip4_address_t * first_int_addr;
1027
1028       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1029         {
1030           first_int_addr =
1031             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1032                                          0 /* just want the address */);
1033           rt->cached_sw_if_index = sw_if_index0;
1034           rt->cached_ip4_address = first_int_addr->as_u32;
1035         }
1036
1037       /* Don't NAT packet aimed at the intfc address */
1038       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1039                                 rt->cached_ip4_address))
1040         return next0;
1041
1042       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1043       return SNAT_IN2OUT_NEXT_DROP;
1044     }
1045
1046   new_addr0 = sm0.addr.as_u32;
1047   new_id0 = sm0.port;
1048   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1049   old_addr0 = ip0->src_address.as_u32;
1050   ip0->src_address.as_u32 = new_addr0;
1051
1052   sum0 = ip0->checksum;
1053   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1054                          ip4_header_t,
1055                          src_address /* changed member */);
1056   ip0->checksum = ip_csum_fold (sum0);
1057
1058   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1059     {
1060       old_id0 = echo0->identifier;
1061       echo0->identifier = new_id0;
1062
1063       sum0 = icmp0->checksum;
1064       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1065                              identifier);
1066       icmp0->checksum = ip_csum_fold (sum0);
1067     }
1068
1069   return next0;
1070 }
1071
1072 static uword
1073 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1074                                 vlib_node_runtime_t * node,
1075                                 vlib_frame_t * frame)
1076 {
1077   u32 n_left_from, * from, * to_next;
1078   snat_in2out_next_t next_index;
1079   u32 pkts_processed = 0;
1080   snat_main_t * sm = &snat_main;
1081   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1082   u32 stats_node_index;
1083
1084   stats_node_index = snat_in2out_fast_node.index;
1085
1086   from = vlib_frame_vector_args (frame);
1087   n_left_from = frame->n_vectors;
1088   next_index = node->cached_next_index;
1089
1090   while (n_left_from > 0)
1091     {
1092       u32 n_left_to_next;
1093
1094       vlib_get_next_frame (vm, node, next_index,
1095                            to_next, n_left_to_next);
1096
1097       while (n_left_from > 0 && n_left_to_next > 0)
1098         {
1099           u32 bi0;
1100           vlib_buffer_t * b0;
1101           u32 next0;
1102           u32 sw_if_index0;
1103           ip4_header_t * ip0;
1104           ip_csum_t sum0;
1105           u32 new_addr0, old_addr0;
1106           u16 old_port0, new_port0;
1107           udp_header_t * udp0;
1108           tcp_header_t * tcp0;
1109           icmp46_header_t * icmp0;
1110           snat_session_key_t key0, sm0;
1111           u32 proto0;
1112           u32 rx_fib_index0;
1113
1114           /* speculatively enqueue b0 to the current next frame */
1115           bi0 = from[0];
1116           to_next[0] = bi0;
1117           from += 1;
1118           to_next += 1;
1119           n_left_from -= 1;
1120           n_left_to_next -= 1;
1121
1122           b0 = vlib_get_buffer (vm, bi0);
1123           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1124
1125           ip0 = vlib_buffer_get_current (b0);
1126           udp0 = ip4_next_header (ip0);
1127           tcp0 = (tcp_header_t *) udp0;
1128           icmp0 = (icmp46_header_t *) udp0;
1129
1130           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1131           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1132
1133           proto0 = ~0;
1134           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1135             ? SNAT_PROTOCOL_UDP : proto0;
1136           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1137             ? SNAT_PROTOCOL_TCP : proto0;
1138           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1139             ? SNAT_PROTOCOL_ICMP : proto0;
1140
1141           if (PREDICT_FALSE (proto0 == ~0))
1142               goto trace0;
1143
1144           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1145             {
1146               ip4_address_t * first_int_addr;
1147
1148               if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1149                 {
1150                   first_int_addr =
1151                     ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1152                                                  0 /* just want the address */);
1153                   rt->cached_sw_if_index = sw_if_index0;
1154                   rt->cached_ip4_address = first_int_addr->as_u32;
1155                 }
1156
1157               /* Don't NAT packet aimed at the intfc address */
1158               if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1159                                 rt->cached_ip4_address))
1160                 goto trace0;
1161
1162               next0 = icmp_in2out_static_map
1163                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1164               goto trace0;
1165             }
1166
1167           key0.addr = ip0->src_address;
1168           key0.port = udp0->src_port;
1169           key0.fib_index = rx_fib_index0;
1170
1171           if (snat_static_mapping_match(sm, key0, &sm0, 0))
1172             {
1173               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1174               next0= SNAT_IN2OUT_NEXT_DROP;
1175               goto trace0;
1176             }
1177
1178           new_addr0 = sm0.addr.as_u32;
1179           new_port0 = sm0.port;
1180           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1181           old_addr0 = ip0->src_address.as_u32;
1182           ip0->src_address.as_u32 = new_addr0;
1183
1184           sum0 = ip0->checksum;
1185           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1186                                  ip4_header_t,
1187                                  src_address /* changed member */);
1188           ip0->checksum = ip_csum_fold (sum0);
1189
1190           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1191             {
1192               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1193                 {
1194                   old_port0 = tcp0->ports.src;
1195                   tcp0->ports.src = new_port0;
1196
1197                   sum0 = tcp0->checksum;
1198                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1199                                          ip4_header_t,
1200                                          dst_address /* changed member */);
1201                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1202                                          ip4_header_t /* cheat */,
1203                                          length /* changed member */);
1204                   tcp0->checksum = ip_csum_fold(sum0);
1205                 }
1206               else
1207                 {
1208                   old_port0 = udp0->src_port;
1209                   udp0->src_port = new_port0;
1210                   udp0->checksum = 0;
1211                 }
1212             }
1213           else
1214             {
1215               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1216                 {
1217                   sum0 = tcp0->checksum;
1218                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1219                                          ip4_header_t,
1220                                          dst_address /* changed member */);
1221                   tcp0->checksum = ip_csum_fold(sum0);
1222                 }
1223             }
1224
1225         trace0:
1226           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1227                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1228             {
1229               snat_in2out_trace_t *t =
1230                  vlib_add_trace (vm, node, b0, sizeof (*t));
1231               t->sw_if_index = sw_if_index0;
1232               t->next_index = next0;
1233             }
1234
1235           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1236
1237           /* verify speculative enqueue, maybe switch current next frame */
1238           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1239                                            to_next, n_left_to_next,
1240                                            bi0, next0);
1241         }
1242
1243       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1244     }
1245
1246   vlib_node_increment_counter (vm, stats_node_index,
1247                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1248                                pkts_processed);
1249   return frame->n_vectors;
1250 }
1251
1252
1253 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
1254   .function = snat_in2out_fast_static_map_fn,
1255   .name = "snat-in2out-fast",
1256   .vector_size = sizeof (u32),
1257   .format_trace = format_snat_in2out_fast_trace,
1258   .type = VLIB_NODE_TYPE_INTERNAL,
1259
1260   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1261   .error_strings = snat_in2out_error_strings,
1262
1263   .runtime_data_bytes = sizeof (snat_runtime_t),
1264
1265   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1266
1267   /* edit / add dispositions here */
1268   .next_nodes = {
1269     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1270     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1271     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1272   },
1273 };
1274
1275 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);