plugins/snat-plugin/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <snat/snat.h>
  23
  24 #include <vppinfra/hash.h>
  25 #include <vppinfra/error.h>
  26 #include <vppinfra/elog.h>
  27
  28 typedef struct {
  29   u32 sw_if_index;
  30   u32 next_index;
  31   u32 session_index;
  32   u32 is_slow_path;
  33 } snat_in2out_trace_t;
  34
  35 /* packet trace format function */
  36 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  37 {
  38   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  39   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  40   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  41   char * tag;
  42
  43   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  44
  45   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  46               t->sw_if_index, t->next_index, t->session_index);
  47
  48   return s;
  49 }
  50
  51 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  56
  57   s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
  58               t->sw_if_index, t->next_index);
  59
  60   return s;
  61 }
  62
  63 vlib_node_registration_t snat_in2out_node;
  64 vlib_node_registration_t snat_in2out_slowpath_node;
  65 vlib_node_registration_t snat_in2out_fast_node;
  66
  67 #define foreach_snat_in2out_error                       \
  68 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  69 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  70 _(OUT_OF_PORTS, "Out of ports")                         \
  71 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  72 _(BAD_ICMP_TYPE, "icmp type not echo-request")          \
  73 _(NO_TRANSLATION, "No translation")
  74
  75 typedef enum {
  76 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  77   foreach_snat_in2out_error
  78 #undef _
  79   SNAT_IN2OUT_N_ERROR,
  80 } snat_in2out_error_t;
  81
  82 static char * snat_in2out_error_strings[] = {
  83 #define _(sym,string) string,
  84   foreach_snat_in2out_error
  85 #undef _
  86 };
  87
  88 typedef enum {
  89   SNAT_IN2OUT_NEXT_LOOKUP,
  90   SNAT_IN2OUT_NEXT_DROP,
  91   SNAT_IN2OUT_NEXT_SLOW_PATH,
  92   SNAT_IN2OUT_N_NEXT,
  93 } snat_in2out_next_t;
  94
  95
  96 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
  97                       ip4_header_t * ip0,
  98                       u32 rx_fib_index0,
  99                       snat_session_key_t * key0,
 100                       snat_session_t ** sessionp,
 101                       vlib_node_runtime_t * node,
 102                       u32 next0)
 103 {
 104   snat_user_t *u;
 105   snat_user_key_t user_key;
 106   snat_session_t *s;
 107   clib_bihash_kv_8_8_t kv0, value0;
 108   u32 oldest_per_user_translation_list_index;
 109   dlist_elt_t * oldest_per_user_translation_list_elt;
 110   dlist_elt_t * per_user_translation_list_elt;
 111   dlist_elt_t * per_user_list_head_elt;
 112   u32 session_index;
 113   snat_session_key_t key1;
 114   u32 address_index = ~0;
 115   u32 outside_fib_index;
 116   uword * p;
 117
 118   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 119   if (! p)
 120     {
 121       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 122       return SNAT_IN2OUT_NEXT_DROP;
 123     }
 124   outside_fib_index = p[0];
 125
 126   user_key.addr = ip0->src_address;
 127   user_key.fib_index = rx_fib_index0;
 128   kv0.key = user_key.as_u64;
 129
 130   /* Ever heard of the "user" = src ip4 address before? */
 131   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 132     {
 133       /* no, make a new one */
 134       pool_get (sm->users, u);
 135       memset (u, 0, sizeof (*u));
 136       u->addr = ip0->src_address;
 137
 138       pool_get (sm->list_pool, per_user_list_head_elt);
 139
 140       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 141         sm->list_pool;
 142
 143       clib_dlist_init (sm->list_pool, u->sessions_per_user_list_head_index);
 144
 145       kv0.value = u - sm->users;
 146
 147       /* add user */
 148       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 149     }
 150   else
 151     {
 152       u = pool_elt_at_index (sm->users, value0.value);
 153     }
 154
 155   /* Over quota? Recycle the least recently used dynamic translation */
 156   if (u->nsessions >= sm->max_translations_per_user)
 157     {
 158       /* Remove the oldest dynamic translation */
 159       do {
 160           oldest_per_user_translation_list_index =
 161             clib_dlist_remove_head
 162             (sm->list_pool, u->sessions_per_user_list_head_index);
 163
 164           ASSERT (oldest_per_user_translation_list_index != ~0);
 165
 166           /* add it back to the end of the LRU list */
 167           clib_dlist_addtail (sm->list_pool,
 168                               u->sessions_per_user_list_head_index,
 169                               oldest_per_user_translation_list_index);
 170           /* Get the list element */
 171           oldest_per_user_translation_list_elt =
 172             pool_elt_at_index (sm->list_pool,
 173                                oldest_per_user_translation_list_index);
 174
 175           /* Get the session index from the list element */
 176           session_index = oldest_per_user_translation_list_elt->value;
 177
 178           /* Get the session */
 179           s = pool_elt_at_index (sm->sessions, session_index);
 180       } while (!snat_is_session_static (s));
 181
 182       /* Remove in2out, out2in keys */
 183       kv0.key = s->in2out.as_u64;
 184       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 185           clib_warning ("in2out key delete failed");
 186       kv0.key = s->out2in.as_u64;
 187       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 188           clib_warning ("out2in key delete failed");
 189
 190       snat_free_outside_address_and_port
 191         (sm, &s->out2in, s->outside_address_index);
 192       s->outside_address_index = ~0;
 193
 194       if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 195         {
 196           ASSERT(0);
 197
 198           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 199           return SNAT_IN2OUT_NEXT_DROP;
 200         }
 201       s->outside_address_index = address_index;
 202     }
 203   else
 204     {
 205       u8 static_mapping = 1;
 206
 207       /* First try to match static mapping by local address and port */
 208       if (snat_static_mapping_match (sm, *key0, &key1, 0))
 209         {
 210           static_mapping = 0;
 211           /* Try to create dynamic translation */
 212           if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 213             {
 214               b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 215               return SNAT_IN2OUT_NEXT_DROP;
 216             }
 217         }
 218
 219       /* Create a new session */
 220       pool_get (sm->sessions, s);
 221       memset (s, 0, sizeof (*s));
 222
 223       s->outside_address_index = address_index;
 224
 225       if (static_mapping)
 226         {
 227           u->nstaticsessions++;
 228           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 229         }
 230       else
 231         {
 232           u->nsessions++;
 233         }
 234
 235       /* Create list elts */
 236       pool_get (sm->list_pool, per_user_translation_list_elt);
 237       clib_dlist_init (sm->list_pool, per_user_translation_list_elt -
 238                        sm->list_pool);
 239
 240       per_user_translation_list_elt->value = s - sm->sessions;
 241       s->per_user_index = per_user_translation_list_elt - sm->list_pool;
 242       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 243
 244       clib_dlist_addtail (sm->list_pool, s->per_user_list_head_index,
 245                           per_user_translation_list_elt - sm->list_pool);
 246    }
 247
 248   s->in2out = *key0;
 249   s->out2in = key1;
 250   s->out2in.protocol = key0->protocol;
 251   s->out2in.fib_index = outside_fib_index;
 252   *sessionp = s;
 253
 254   /* Add to translation hashes */
 255   kv0.key = s->in2out.as_u64;
 256   kv0.value = s - sm->sessions;
 257   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 258       clib_warning ("in2out key add failed");
 259
 260   kv0.key = s->out2in.as_u64;
 261   kv0.value = s - sm->sessions;
 262
 263   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 264       clib_warning ("out2in key add failed");
 265
 266   return next0;
 267 }
 268
 269 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 270                                          vlib_buffer_t * b0,
 271                                          ip4_header_t * ip0,
 272                                          icmp46_header_t * icmp0,
 273                                          u32 sw_if_index0,
 274                                          u32 rx_fib_index0,
 275                                          vlib_node_runtime_t * node,
 276                                          u32 next0,
 277                                          f64 now)
 278 {
 279   snat_session_key_t key0;
 280   icmp_echo_header_t *echo0;
 281   clib_bihash_kv_8_8_t kv0, value0;
 282   snat_session_t * s0;
 283   u32 new_addr0, old_addr0;
 284   u16 old_id0, new_id0;
 285   ip_csum_t sum0;
 286   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 287
 288   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request))
 289     {
 290       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 291       return SNAT_IN2OUT_NEXT_DROP;
 292     }
 293
 294   echo0 = (icmp_echo_header_t *)(icmp0+1);
 295
 296   key0.addr = ip0->src_address;
 297   key0.port = echo0->identifier;
 298   key0.protocol = SNAT_PROTOCOL_ICMP;
 299   key0.fib_index = rx_fib_index0;
 300
 301   kv0.key = key0.as_u64;
 302
 303   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 304     {
 305       ip4_address_t * first_int_addr;
 306
 307       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 308         {
 309           first_int_addr =
 310             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 311                                          0 /* just want the address */);
 312           rt->cached_sw_if_index = sw_if_index0;
 313           rt->cached_ip4_address = first_int_addr->as_u32;
 314         }
 315
 316       /* Don't NAT packet aimed at the intfc address */
 317       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 318                                 rt->cached_ip4_address))
 319         return next0;
 320
 321       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 322                          &s0, node, next0);
 323
 324       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 325         return next0;
 326     }
 327   else
 328     s0 = pool_elt_at_index (sm->sessions, value0.value);
 329
 330   old_addr0 = ip0->src_address.as_u32;
 331   ip0->src_address = s0->out2in.addr;
 332   new_addr0 = ip0->src_address.as_u32;
 333   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 334
 335   sum0 = ip0->checksum;
 336   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 337                          ip4_header_t,
 338                          src_address /* changed member */);
 339   ip0->checksum = ip_csum_fold (sum0);
 340
 341   old_id0 = echo0->identifier;
 342   new_id0 = s0->out2in.port;
 343   echo0->identifier = new_id0;
 344
 345   sum0 = icmp0->checksum;
 346   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 347                          identifier);
 348   icmp0->checksum = ip_csum_fold (sum0);
 349
 350   /* Accounting */
 351   s0->last_heard = now;
 352   s0->total_pkts++;
 353   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 354   /* Per-user LRU list maintenance for dynamic translations */
 355   if (!snat_is_session_static (s0))
 356     {
 357       clib_dlist_remove (sm->list_pool, s0->per_user_index);
 358       clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 359                           s0->per_user_index);
 360     }
 361
 362   return next0;
 363 }
 364
 365 static inline uword
 366 snat_in2out_node_fn_inline (vlib_main_t * vm,
 367                             vlib_node_runtime_t * node,
 368                             vlib_frame_t * frame, int is_slow_path)
 369 {
 370   u32 n_left_from, * from, * to_next;
 371   snat_in2out_next_t next_index;
 372   u32 pkts_processed = 0;
 373   snat_main_t * sm = &snat_main;
 374   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 375   f64 now = vlib_time_now (vm);
 376   u32 stats_node_index;
 377
 378   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 379     snat_in2out_node.index;
 380
 381   from = vlib_frame_vector_args (frame);
 382   n_left_from = frame->n_vectors;
 383   next_index = node->cached_next_index;
 384
 385   while (n_left_from > 0)
 386     {
 387       u32 n_left_to_next;
 388
 389       vlib_get_next_frame (vm, node, next_index,
 390                            to_next, n_left_to_next);
 391
 392       while (n_left_from >= 4 && n_left_to_next >= 2)
 393         {
 394           u32 bi0, bi1;
 395           vlib_buffer_t * b0, * b1;
 396           u32 next0, next1;
 397           u32 sw_if_index0, sw_if_index1;
 398           ip4_header_t * ip0, * ip1;
 399           ip_csum_t sum0, sum1;
 400           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 401           u16 old_port0, new_port0, old_port1, new_port1;
 402           udp_header_t * udp0, * udp1;
 403           tcp_header_t * tcp0, * tcp1;
 404           icmp46_header_t * icmp0, * icmp1;
 405           snat_session_key_t key0, key1;
 406           u32 rx_fib_index0, rx_fib_index1;
 407           u32 proto0, proto1;
 408           snat_session_t * s0 = 0, * s1 = 0;
 409           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 410
 411           /* Prefetch next iteration. */
 412           {
 413             vlib_buffer_t * p2, * p3;
 414
 415             p2 = vlib_get_buffer (vm, from[2]);
 416             p3 = vlib_get_buffer (vm, from[3]);
 417
 418             vlib_prefetch_buffer_header (p2, LOAD);
 419             vlib_prefetch_buffer_header (p3, LOAD);
 420
 421             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 422             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 423           }
 424
 425           /* speculatively enqueue b0 and b1 to the current next frame */
 426           to_next[0] = bi0 = from[0];
 427           to_next[1] = bi1 = from[1];
 428           from += 2;
 429           to_next += 2;
 430           n_left_from -= 2;
 431           n_left_to_next -= 2;
 432
 433           b0 = vlib_get_buffer (vm, bi0);
 434           b1 = vlib_get_buffer (vm, bi1);
 435
 436           ip0 = vlib_buffer_get_current (b0);
 437           udp0 = ip4_next_header (ip0);
 438           tcp0 = (tcp_header_t *) udp0;
 439           icmp0 = (icmp46_header_t *) udp0;
 440
 441           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 442           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 443                                    sw_if_index0);
 444
 445           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 446
 447 #if 0
 448           /* Formally correct, but we send to slowpath, lookup or drop */
 449           vnet_get_config_data (&cm->config_main,
 450                                 &b0->current_config_index,
 451                                 &next0,
 452                                 0 /* sizeof config data */);
 453 #endif
 454
 455           proto0 = ~0;
 456           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 457             ? SNAT_PROTOCOL_UDP : proto0;
 458           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 459             ? SNAT_PROTOCOL_TCP : proto0;
 460           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 461             ? SNAT_PROTOCOL_ICMP : proto0;
 462
 463           /* Next configured feature, probably ip4-lookup */
 464           if (is_slow_path)
 465             {
 466               if (PREDICT_FALSE (proto0 == ~0))
 467                 goto trace00;
 468
 469               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 470                 {
 471                   next0 = icmp_in2out_slow_path
 472                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 473                      node, next0, now);
 474                   goto trace00;
 475                 }
 476             }
 477           else
 478             {
 479               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 480                 {
 481                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 482                   goto trace00;
 483                 }
 484             }
 485
 486           key0.addr = ip0->src_address;
 487           key0.port = udp0->src_port;
 488           key0.protocol = proto0;
 489           key0.fib_index = rx_fib_index0;
 490
 491           kv0.key = key0.as_u64;
 492
 493           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
 494             {
 495               if (is_slow_path)
 496                 {
 497                   ip4_address_t * first_int_addr;
 498
 499                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 500                     {
 501                       first_int_addr =
 502                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 503                                                      0 /* just want the address */);
 504                       rt->cached_sw_if_index = sw_if_index0;
 505                       rt->cached_ip4_address = first_int_addr->as_u32;
 506                     }
 507
 508                   /* Don't NAT packet aimed at the intfc address */
 509                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 510                                     rt->cached_ip4_address))
 511                     goto trace00;
 512
 513                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 514                                      &s0, node, next0);
 515                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 516                     goto trace00;
 517                 }
 518               else
 519                 {
 520                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 521                   goto trace00;
 522                 }
 523             }
 524           else
 525             s0 = pool_elt_at_index (sm->sessions, value0.value);
 526
 527           old_addr0 = ip0->src_address.as_u32;
 528           ip0->src_address = s0->out2in.addr;
 529           new_addr0 = ip0->src_address.as_u32;
 530           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 531
 532           sum0 = ip0->checksum;
 533           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 534                                  ip4_header_t,
 535                                  src_address /* changed member */);
 536           ip0->checksum = ip_csum_fold (sum0);
 537
 538           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 539             {
 540               old_port0 = tcp0->ports.src;
 541               tcp0->ports.src = s0->out2in.port;
 542               new_port0 = tcp0->ports.src;
 543
 544               sum0 = tcp0->checksum;
 545               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 546                                      ip4_header_t,
 547                                      dst_address /* changed member */);
 548               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 549                                      ip4_header_t /* cheat */,
 550                                      length /* changed member */);
 551               tcp0->checksum = ip_csum_fold(sum0);
 552             }
 553           else
 554             {
 555               old_port0 = udp0->src_port;
 556               udp0->src_port = s0->out2in.port;
 557               udp0->checksum = 0;
 558             }
 559
 560           /* Accounting */
 561           s0->last_heard = now;
 562           s0->total_pkts++;
 563           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 564           /* Per-user LRU list maintenance for dynamic translation */
 565           if (!snat_is_session_static (s0))
 566             {
 567               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 568               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 569                                   s0->per_user_index);
 570             }
 571         trace00:
 572
 573           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 574                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 575             {
 576               snat_in2out_trace_t *t =
 577                  vlib_add_trace (vm, node, b0, sizeof (*t));
 578               t->is_slow_path = is_slow_path;
 579               t->sw_if_index = sw_if_index0;
 580               t->next_index = next0;
 581                   t->session_index = ~0;
 582               if (s0)
 583                   t->session_index = s0 - sm->sessions;
 584             }
 585
 586           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 587
 588           ip1 = vlib_buffer_get_current (b1);
 589           udp1 = ip4_next_header (ip1);
 590           tcp1 = (tcp_header_t *) udp1;
 591           icmp1 = (icmp46_header_t *) udp1;
 592
 593           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 594           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 595                                    sw_if_index1);
 596
 597 #if 0
 598           vnet_get_config_data (&cm->config_main,
 599                                 &b1->current_config_index,
 600                                 &next1,
 601                                 0 /* sizeof config data */);
 602 #endif
 603
 604           proto1 = ~0;
 605           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 606             ? SNAT_PROTOCOL_UDP : proto1;
 607           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 608             ? SNAT_PROTOCOL_TCP : proto1;
 609           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 610             ? SNAT_PROTOCOL_ICMP : proto1;
 611
 612           /* Next configured feature, probably ip4-lookup */
 613           if (is_slow_path)
 614             {
 615               if (PREDICT_FALSE (proto1 == ~0))
 616                 goto trace01;
 617
 618               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 619                 {
 620                   next1 = icmp_in2out_slow_path
 621                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, next1,
 622                      now);
 623                   goto trace01;
 624                 }
 625             }
 626           else
 627             {
 628               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
 629                 {
 630                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 631                   goto trace01;
 632                 }
 633             }
 634
 635           key1.addr = ip1->src_address;
 636           key1.port = udp1->src_port;
 637           key1.protocol = proto1;
 638           key1.fib_index = rx_fib_index1;
 639
 640           kv1.key = key1.as_u64;
 641
 642             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
 643             {
 644               if (is_slow_path)
 645                 {
 646                   ip4_address_t * first_int_addr;
 647
 648                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index1))
 649                     {
 650                       first_int_addr =
 651                         ip4_interface_first_address (sm->ip4_main, sw_if_index1,
 652                                                      0 /* just want the address */);
 653                       rt->cached_sw_if_index = sw_if_index1;
 654                       rt->cached_ip4_address = first_int_addr->as_u32;
 655                     }
 656
 657                   /* Don't NAT packet aimed at the intfc address */
 658                   if (PREDICT_FALSE(ip1->dst_address.as_u32 ==
 659                                     rt->cached_ip4_address))
 660                     goto trace01;
 661
 662                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
 663                                      &s1, node, next1);
 664                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
 665                     goto trace01;
 666                 }
 667               else
 668                 {
 669                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 670                   goto trace01;
 671                 }
 672             }
 673           else
 674             s1 = pool_elt_at_index (sm->sessions, value1.value);
 675
 676           old_addr1 = ip1->src_address.as_u32;
 677           ip1->src_address = s1->out2in.addr;
 678           new_addr1 = ip1->src_address.as_u32;
 679           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
 680
 681           sum1 = ip1->checksum;
 682           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 683                                  ip4_header_t,
 684                                  src_address /* changed member */);
 685           ip1->checksum = ip_csum_fold (sum1);
 686
 687           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 688             {
 689               old_port1 = tcp1->ports.src;
 690               tcp1->ports.src = s1->out2in.port;
 691               new_port1 = tcp1->ports.src;
 692
 693               sum1 = tcp1->checksum;
 694               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 695                                      ip4_header_t,
 696                                      dst_address /* changed member */);
 697               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 698                                      ip4_header_t /* cheat */,
 699                                      length /* changed member */);
 700               tcp1->checksum = ip_csum_fold(sum1);
 701             }
 702           else
 703             {
 704               old_port1 = udp1->src_port;
 705               udp1->src_port = s1->out2in.port;
 706               udp1->checksum = 0;
 707             }
 708
 709           /* Accounting */
 710           s1->last_heard = now;
 711           s1->total_pkts++;
 712           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 713           /* Per-user LRU list maintenance for dynamic translation */
 714           if (!snat_is_session_static (s1))
 715             {
 716               clib_dlist_remove (sm->list_pool, s1->per_user_index);
 717               clib_dlist_addtail (sm->list_pool, s1->per_user_list_head_index,
 718                                   s1->per_user_index);
 719             }
 720         trace01:
 721
 722           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 723                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 724             {
 725               snat_in2out_trace_t *t =
 726                  vlib_add_trace (vm, node, b1, sizeof (*t));
 727               t->sw_if_index = sw_if_index1;
 728               t->next_index = next1;
 729               t->session_index = ~0;
 730               if (s1)
 731                 t->session_index = s1 - sm->sessions;
 732             }
 733
 734           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
 735
 736           /* verify speculative enqueues, maybe switch current next frame */
 737           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 738                                            to_next, n_left_to_next,
 739                                            bi0, bi1, next0, next1);
 740         }
 741
 742       while (n_left_from > 0 && n_left_to_next > 0)
 743         {
 744           u32 bi0;
 745           vlib_buffer_t * b0;
 746           u32 next0;
 747           u32 sw_if_index0;
 748           ip4_header_t * ip0;
 749           ip_csum_t sum0;
 750           u32 new_addr0, old_addr0;
 751           u16 old_port0, new_port0;
 752           udp_header_t * udp0;
 753           tcp_header_t * tcp0;
 754           icmp46_header_t * icmp0;
 755           snat_session_key_t key0;
 756           u32 rx_fib_index0;
 757           u32 proto0;
 758           snat_session_t * s0 = 0;
 759           clib_bihash_kv_8_8_t kv0, value0;
 760
 761           /* speculatively enqueue b0 to the current next frame */
 762           bi0 = from[0];
 763           to_next[0] = bi0;
 764           from += 1;
 765           to_next += 1;
 766           n_left_from -= 1;
 767           n_left_to_next -= 1;
 768
 769           b0 = vlib_get_buffer (vm, bi0);
 770           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
 771
 772           ip0 = vlib_buffer_get_current (b0);
 773           udp0 = ip4_next_header (ip0);
 774           tcp0 = (tcp_header_t *) udp0;
 775           icmp0 = (icmp46_header_t *) udp0;
 776
 777           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 778           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 779                                    sw_if_index0);
 780
 781
 782 #if 0
 783           vnet_get_config_data (&cm->config_main,
 784                                 &b0->current_config_index,
 785                                 &next0,
 786                                 0 /* sizeof config data */);
 787 #endif
 788
 789           proto0 = ~0;
 790           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 791             ? SNAT_PROTOCOL_UDP : proto0;
 792           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 793             ? SNAT_PROTOCOL_TCP : proto0;
 794           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 795             ? SNAT_PROTOCOL_ICMP : proto0;
 796
 797           /* Next configured feature, probably ip4-lookup */
 798           if (is_slow_path)
 799             {
 800               if (PREDICT_FALSE (proto0 == ~0))
 801                 goto trace0;
 802
 803               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 804                 {
 805                   next0 = icmp_in2out_slow_path
 806                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0,
 807                      now);
 808                   goto trace0;
 809                 }
 810             }
 811           else
 812             {
 813               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 814                 {
 815                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 816                   goto trace0;
 817                 }
 818             }
 819
 820           key0.addr = ip0->src_address;
 821           key0.port = udp0->src_port;
 822           key0.protocol = proto0;
 823           key0.fib_index = rx_fib_index0;
 824
 825           kv0.key = key0.as_u64;
 826
 827           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 828             {
 829               if (is_slow_path)
 830                 {
 831                   ip4_address_t * first_int_addr;
 832
 833                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 834                     {
 835                       first_int_addr =
 836                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 837                                                      0 /* just want the address */);
 838                       rt->cached_sw_if_index = sw_if_index0;
 839                       rt->cached_ip4_address = first_int_addr->as_u32;
 840                     }
 841
 842                   /* Don't NAT packet aimed at the intfc address */
 843                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 844                                     rt->cached_ip4_address))
 845                     goto trace0;
 846
 847                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 848                                      &s0, node, next0);
 849                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 850                     goto trace0;
 851                 }
 852               else
 853                 {
 854                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 855                   goto trace0;
 856                 }
 857             }
 858           else
 859             s0 = pool_elt_at_index (sm->sessions, value0.value);
 860
 861           old_addr0 = ip0->src_address.as_u32;
 862           ip0->src_address = s0->out2in.addr;
 863           new_addr0 = ip0->src_address.as_u32;
 864           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 865
 866           sum0 = ip0->checksum;
 867           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 868                                  ip4_header_t,
 869                                  src_address /* changed member */);
 870           ip0->checksum = ip_csum_fold (sum0);
 871
 872           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 873             {
 874               old_port0 = tcp0->ports.src;
 875               tcp0->ports.src = s0->out2in.port;
 876               new_port0 = tcp0->ports.src;
 877
 878               sum0 = tcp0->checksum;
 879               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 880                                      ip4_header_t,
 881                                      dst_address /* changed member */);
 882               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 883                                      ip4_header_t /* cheat */,
 884                                      length /* changed member */);
 885               tcp0->checksum = ip_csum_fold(sum0);
 886             }
 887           else
 888             {
 889               old_port0 = udp0->src_port;
 890               udp0->src_port = s0->out2in.port;
 891               udp0->checksum = 0;
 892             }
 893
 894           /* Accounting */
 895           s0->last_heard = now;
 896           s0->total_pkts++;
 897           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 898           /* Per-user LRU list maintenance for dynamic translation */
 899           if (!snat_is_session_static (s0))
 900             {
 901               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 902               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 903                                   s0->per_user_index);
 904             }
 905
 906         trace0:
 907           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 908                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 909             {
 910               snat_in2out_trace_t *t =
 911                  vlib_add_trace (vm, node, b0, sizeof (*t));
 912               t->is_slow_path = is_slow_path;
 913               t->sw_if_index = sw_if_index0;
 914               t->next_index = next0;
 915                   t->session_index = ~0;
 916               if (s0)
 917                   t->session_index = s0 - sm->sessions;
 918             }
 919
 920           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 921
 922           /* verify speculative enqueue, maybe switch current next frame */
 923           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 924                                            to_next, n_left_to_next,
 925                                            bi0, next0);
 926         }
 927
 928       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 929     }
 930
 931   vlib_node_increment_counter (vm, stats_node_index,
 932                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
 933                                pkts_processed);
 934   return frame->n_vectors;
 935 }
 936
 937 static uword
 938 snat_in2out_fast_path_fn (vlib_main_t * vm,
 939                           vlib_node_runtime_t * node,
 940                           vlib_frame_t * frame)
 941 {
 942   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
 943 }
 944
 945 VLIB_REGISTER_NODE (snat_in2out_node) = {
 946   .function = snat_in2out_fast_path_fn,
 947   .name = "snat-in2out",
 948   .vector_size = sizeof (u32),
 949   .format_trace = format_snat_in2out_trace,
 950   .type = VLIB_NODE_TYPE_INTERNAL,
 951
 952   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 953   .error_strings = snat_in2out_error_strings,
 954
 955   .runtime_data_bytes = sizeof (snat_runtime_t),
 956
 957   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 958
 959   /* edit / add dispositions here */
 960   .next_nodes = {
 961     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 962     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 963     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 964   },
 965 };
 966
 967 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
 968
 969 static uword
 970 snat_in2out_slow_path_fn (vlib_main_t * vm,
 971                           vlib_node_runtime_t * node,
 972                           vlib_frame_t * frame)
 973 {
 974   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
 975 }
 976
 977 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
 978   .function = snat_in2out_slow_path_fn,
 979   .name = "snat-in2out-slowpath",
 980   .vector_size = sizeof (u32),
 981   .format_trace = format_snat_in2out_trace,
 982   .type = VLIB_NODE_TYPE_INTERNAL,
 983
 984   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 985   .error_strings = snat_in2out_error_strings,
 986
 987   .runtime_data_bytes = sizeof (snat_runtime_t),
 988
 989   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 990
 991   /* edit / add dispositions here */
 992   .next_nodes = {
 993     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 994     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 995     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 996   },
 997 };
 998
 999 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);
1000
1001 static inline u32 icmp_in2out_static_map (snat_main_t *sm,
1002                                           vlib_buffer_t * b0,
1003                                           ip4_header_t * ip0,
1004                                           icmp46_header_t * icmp0,
1005                                           u32 sw_if_index0,
1006                                           vlib_node_runtime_t * node,
1007                                           u32 next0)
1008 {
1009   snat_session_key_t key0, sm0;
1010   icmp_echo_header_t *echo0;
1011   u32 new_addr0, old_addr0;
1012   u16 old_id0, new_id0;
1013   ip_csum_t sum0;
1014   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1015
1016   echo0 = (icmp_echo_header_t *)(icmp0+1);
1017
1018   key0.addr = ip0->src_address;
1019   key0.port = echo0->identifier;
1020
1021   if (snat_static_mapping_match(sm, key0, &sm0, 0))
1022     {
1023       ip4_address_t * first_int_addr;
1024
1025       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1026         {
1027           first_int_addr =
1028             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1029                                          0 /* just want the address */);
1030           rt->cached_sw_if_index = sw_if_index0;
1031           rt->cached_ip4_address = first_int_addr->as_u32;
1032         }
1033
1034       /* Don't NAT packet aimed at the intfc address */
1035       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1036                                 rt->cached_ip4_address))
1037         return next0;
1038
1039       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1040       return SNAT_IN2OUT_NEXT_DROP;
1041     }
1042
1043   new_addr0 = sm0.addr.as_u32;
1044   new_id0 = sm0.port;
1045   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1046   old_addr0 = ip0->src_address.as_u32;
1047   ip0->src_address.as_u32 = new_addr0;
1048
1049   sum0 = ip0->checksum;
1050   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1051                          ip4_header_t,
1052                          src_address /* changed member */);
1053   ip0->checksum = ip_csum_fold (sum0);
1054
1055   if (PREDICT_FALSE(new_id0 != echo0->identifier))
1056     {
1057       old_id0 = echo0->identifier;
1058       echo0->identifier = new_id0;
1059
1060       sum0 = icmp0->checksum;
1061       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1062                              identifier);
1063       icmp0->checksum = ip_csum_fold (sum0);
1064     }
1065
1066   return next0;
1067 }
1068
1069 static uword
1070 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1071                                 vlib_node_runtime_t * node,
1072                                 vlib_frame_t * frame)
1073 {
1074   u32 n_left_from, * from, * to_next;
1075   snat_in2out_next_t next_index;
1076   u32 pkts_processed = 0;
1077   snat_main_t * sm = &snat_main;
1078   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1079   u32 stats_node_index;
1080
1081   stats_node_index = snat_in2out_fast_node.index;
1082
1083   from = vlib_frame_vector_args (frame);
1084   n_left_from = frame->n_vectors;
1085   next_index = node->cached_next_index;
1086
1087   while (n_left_from > 0)
1088     {
1089       u32 n_left_to_next;
1090
1091       vlib_get_next_frame (vm, node, next_index,
1092                            to_next, n_left_to_next);
1093
1094       while (n_left_from > 0 && n_left_to_next > 0)
1095         {
1096           u32 bi0;
1097           vlib_buffer_t * b0;
1098           u32 next0;
1099           u32 sw_if_index0;
1100           ip4_header_t * ip0;
1101           ip_csum_t sum0;
1102           u32 new_addr0, old_addr0;
1103           u16 old_port0, new_port0;
1104           udp_header_t * udp0;
1105           tcp_header_t * tcp0;
1106           icmp46_header_t * icmp0;
1107           snat_session_key_t key0, sm0;
1108           u32 proto0;
1109
1110           /* speculatively enqueue b0 to the current next frame */
1111           bi0 = from[0];
1112           to_next[0] = bi0;
1113           from += 1;
1114           to_next += 1;
1115           n_left_from -= 1;
1116           n_left_to_next -= 1;
1117
1118           b0 = vlib_get_buffer (vm, bi0);
1119           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1120
1121           ip0 = vlib_buffer_get_current (b0);
1122           udp0 = ip4_next_header (ip0);
1123           tcp0 = (tcp_header_t *) udp0;
1124           icmp0 = (icmp46_header_t *) udp0;
1125
1126           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1127
1128           proto0 = ~0;
1129           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1130             ? SNAT_PROTOCOL_UDP : proto0;
1131           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1132             ? SNAT_PROTOCOL_TCP : proto0;
1133           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1134             ? SNAT_PROTOCOL_ICMP : proto0;
1135
1136           if (PREDICT_FALSE (proto0 == ~0))
1137               goto trace0;
1138
1139           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1140             {
1141               ip4_address_t * first_int_addr;
1142
1143               if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1144                 {
1145                   first_int_addr =
1146                     ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1147                                                  0 /* just want the address */);
1148                   rt->cached_sw_if_index = sw_if_index0;
1149                   rt->cached_ip4_address = first_int_addr->as_u32;
1150                 }
1151
1152               /* Don't NAT packet aimed at the intfc address */
1153               if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1154                                 rt->cached_ip4_address))
1155                 goto trace0;
1156
1157               next0 = icmp_in2out_static_map
1158                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0);
1159               goto trace0;
1160             }
1161
1162           key0.addr = ip0->src_address;
1163           key0.port = udp0->src_port;
1164
1165           if (snat_static_mapping_match(sm, key0, &sm0, 0))
1166             {
1167               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1168               next0= SNAT_IN2OUT_NEXT_DROP;
1169               goto trace0;
1170             }
1171
1172           new_addr0 = sm0.addr.as_u32;
1173           new_port0 = sm0.port;
1174           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1175           old_addr0 = ip0->src_address.as_u32;
1176           ip0->src_address.as_u32 = new_addr0;
1177
1178           sum0 = ip0->checksum;
1179           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1180                                  ip4_header_t,
1181                                  src_address /* changed member */);
1182           ip0->checksum = ip_csum_fold (sum0);
1183
1184           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1185             {
1186               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1187                 {
1188                   old_port0 = tcp0->ports.src;
1189                   tcp0->ports.src = new_port0;
1190
1191                   sum0 = tcp0->checksum;
1192                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1193                                          ip4_header_t,
1194                                          dst_address /* changed member */);
1195                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1196                                          ip4_header_t /* cheat */,
1197                                          length /* changed member */);
1198                   tcp0->checksum = ip_csum_fold(sum0);
1199                 }
1200               else
1201                 {
1202                   old_port0 = udp0->src_port;
1203                   udp0->src_port = new_port0;
1204                   udp0->checksum = 0;
1205                 }
1206             }
1207           else
1208             {
1209               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1210                 {
1211                   sum0 = tcp0->checksum;
1212                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1213                                          ip4_header_t,
1214                                          dst_address /* changed member */);
1215                   tcp0->checksum = ip_csum_fold(sum0);
1216                 }
1217             }
1218
1219         trace0:
1220           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1221                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1222             {
1223               snat_in2out_trace_t *t =
1224                  vlib_add_trace (vm, node, b0, sizeof (*t));
1225               t->sw_if_index = sw_if_index0;
1226               t->next_index = next0;
1227             }
1228
1229           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1230
1231           /* verify speculative enqueue, maybe switch current next frame */
1232           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1233                                            to_next, n_left_to_next,
1234                                            bi0, next0);
1235         }
1236
1237       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1238     }
1239
1240   vlib_node_increment_counter (vm, stats_node_index,
1241                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1242                                pkts_processed);
1243   return frame->n_vectors;
1244 }
1245
1246
1247 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
1248   .function = snat_in2out_fast_static_map_fn,
1249   .name = "snat-in2out-fast",
1250   .vector_size = sizeof (u32),
1251   .format_trace = format_snat_in2out_fast_trace,
1252   .type = VLIB_NODE_TYPE_INTERNAL,
1253
1254   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1255   .error_strings = snat_in2out_error_strings,
1256
1257   .runtime_data_bytes = sizeof (snat_runtime_t),
1258
1259   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1260
1261   /* edit / add dispositions here */
1262   .next_nodes = {
1263     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1264     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1265     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1266   },
1267 };
1268
1269 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);