plugins/snat-plugin/snat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <snat/snat.h>
  23
  24 #include <vppinfra/hash.h>
  25 #include <vppinfra/error.h>
  26 #include <vppinfra/elog.h>
  27
  28 vlib_node_registration_t snat_in2out_node, snat_in2out_slowpath_node;
  29
  30 typedef struct {
  31   u32 sw_if_index;
  32   u32 next_index;
  33   u32 session_index;
  34   u32 is_slow_path;
  35 } snat_in2out_trace_t;
  36
  37 /* packet trace format function */
  38 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  39 {
  40   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  41   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  42   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  43   char * tag;
  44
  45   tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
  46
  47   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  48               t->sw_if_index, t->next_index, t->session_index);
  49
  50   return s;
  51 }
  52
  53 vlib_node_registration_t snat_in2out_node;
  54
  55 #define foreach_snat_in2out_error                       \
  56 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  57 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  58 _(OUT_OF_PORTS, "Out of ports")                         \
  59 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  60 _(BAD_ICMP_TYPE, "icmp type not echo-request")
  61
  62 typedef enum {
  63 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  64   foreach_snat_in2out_error
  65 #undef _
  66   SNAT_IN2OUT_N_ERROR,
  67 } snat_in2out_error_t;
  68
  69 static char * snat_in2out_error_strings[] = {
  70 #define _(sym,string) string,
  71   foreach_snat_in2out_error
  72 #undef _
  73 };
  74
  75 typedef enum {
  76   SNAT_IN2OUT_NEXT_LOOKUP,
  77   SNAT_IN2OUT_NEXT_DROP,
  78   SNAT_IN2OUT_NEXT_SLOW_PATH,
  79   SNAT_IN2OUT_N_NEXT,
  80 } snat_in2out_next_t;
  81
  82
  83 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
  84                       ip4_header_t * ip0,
  85                       u32 rx_fib_index0,
  86                       snat_session_key_t * key0,
  87                       snat_session_t ** sessionp,
  88                       vlib_node_runtime_t * node,
  89                       u32 next0)
  90 {
  91   snat_user_t *u;
  92   snat_user_key_t user_key;
  93   snat_session_t *s;
  94   clib_bihash_kv_8_8_t kv0, value0;
  95   u32 oldest_per_user_translation_list_index;
  96   dlist_elt_t * oldest_per_user_translation_list_elt;
  97   dlist_elt_t * per_user_translation_list_elt;
  98   dlist_elt_t * per_user_list_head_elt;
  99   u32 session_index;
 100   snat_session_key_t key1;
 101   u32 address_index;
 102   u32 outside_fib_index;
 103   uword * p;
 104
 105   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 106   if (! p)
 107     {
 108       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 109       return SNAT_IN2OUT_NEXT_DROP;
 110     }
 111   outside_fib_index = p[0];
 112
 113   user_key.addr = ip0->src_address;
 114   user_key.fib_index = rx_fib_index0;
 115   kv0.key = user_key.as_u64;
 116
 117   /* Ever heard of the "user" = src ip4 address before? */
 118   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 119     {
 120       /* no, make a new one */
 121       pool_get (sm->users, u);
 122       memset (u, 0, sizeof (*u));
 123       u->addr = ip0->src_address;
 124
 125       pool_get (sm->list_pool, per_user_list_head_elt);
 126
 127       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 128         sm->list_pool;
 129
 130       clib_dlist_init (sm->list_pool, u->sessions_per_user_list_head_index);
 131
 132       kv0.value = u - sm->users;
 133
 134       /* add user */
 135       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 136     }
 137   else
 138     {
 139       u = pool_elt_at_index (sm->users, value0.value);
 140     }
 141
 142   /* Over quota? Recycle the least recently used translation */
 143   if (u->nsessions >= sm->max_translations_per_user)
 144     {
 145       /* Remove the oldest translation */
 146       oldest_per_user_translation_list_index =
 147         clib_dlist_remove_head
 148         (sm->list_pool, u->sessions_per_user_list_head_index);
 149
 150       ASSERT (oldest_per_user_translation_list_index != ~0);
 151
 152       /* add it back to the end of the LRU list */
 153       clib_dlist_addtail (sm->list_pool, u->sessions_per_user_list_head_index,
 154                           oldest_per_user_translation_list_index);
 155
 156       /* Get the list element */
 157       oldest_per_user_translation_list_elt =
 158         pool_elt_at_index (sm->list_pool,
 159                            oldest_per_user_translation_list_index);
 160
 161       /* Get the session index from the list element */
 162       session_index = oldest_per_user_translation_list_elt->value;
 163
 164       /* Get the session */
 165       s = pool_elt_at_index (sm->sessions, session_index);
 166
 167       /* Remove in2out, out2in keys */
 168       kv0.key = s->in2out.as_u64;
 169       if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
 170           clib_warning ("in2out key delete failed");
 171       kv0.key = s->out2in.as_u64;
 172       if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
 173           clib_warning ("out2in key delete failed");
 174
 175       snat_free_outside_address_and_port
 176         (sm, &s->out2in, s->outside_address_index);
 177       s->outside_address_index = ~0;
 178
 179       if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 180         {
 181           ASSERT(0);
 182
 183           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 184           return SNAT_IN2OUT_NEXT_DROP;
 185         }
 186       s->outside_address_index = address_index;
 187     }
 188   else
 189     {
 190       if (snat_alloc_outside_address_and_port (sm, &key1, &address_index))
 191         {
 192           ASSERT(0);
 193
 194           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 195           return SNAT_IN2OUT_NEXT_DROP;
 196         }
 197
 198       /* Create a new session */
 199       pool_get (sm->sessions, s);
 200       memset (s, 0, sizeof (*s));
 201
 202       s->outside_address_index = address_index;
 203
 204       /* Create list elts */
 205       pool_get (sm->list_pool, per_user_translation_list_elt);
 206       clib_dlist_init (sm->list_pool, per_user_translation_list_elt -
 207                        sm->list_pool);
 208
 209       per_user_translation_list_elt->value = s - sm->sessions;
 210       s->per_user_index = per_user_translation_list_elt - sm->list_pool;
 211       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 212
 213       clib_dlist_addtail (sm->list_pool, s->per_user_list_head_index,
 214                           per_user_translation_list_elt - sm->list_pool);
 215       u->nsessions++;
 216     }
 217
 218   s->in2out = *key0;
 219   s->out2in = key1;
 220   s->out2in.protocol = key0->protocol;
 221   s->out2in.fib_index = outside_fib_index;
 222   *sessionp = s;
 223
 224   /* Add to translation hashes */
 225   kv0.key = s->in2out.as_u64;
 226   kv0.value = s - sm->sessions;
 227   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 228       clib_warning ("in2out key add failed");
 229
 230   kv0.key = s->out2in.as_u64;
 231   kv0.value = s - sm->sessions;
 232
 233   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 234       clib_warning ("out2in key add failed");
 235
 236   return next0;
 237 }
 238
 239 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 240                                          vlib_buffer_t * b0,
 241                                          ip4_header_t * ip0,
 242                                          icmp46_header_t * icmp0,
 243                                          u32 sw_if_index0,
 244                                          u32 rx_fib_index0,
 245                                          vlib_node_runtime_t * node,
 246                                          u32 next0,
 247                                          f64 now)
 248 {
 249   snat_session_key_t key0;
 250   icmp_echo_header_t *echo0;
 251   clib_bihash_kv_8_8_t kv0, value0;
 252   snat_session_t * s0;
 253   u32 new_addr0, old_addr0;
 254   u16 old_id0, new_id0;
 255   ip_csum_t sum0;
 256   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 257
 258   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request))
 259     {
 260       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 261       return SNAT_IN2OUT_NEXT_DROP;
 262     }
 263
 264   echo0 = (icmp_echo_header_t *)(icmp0+1);
 265
 266   key0.addr = ip0->src_address;
 267   key0.port = echo0->identifier;
 268   key0.protocol = SNAT_PROTOCOL_ICMP;
 269   key0.fib_index = rx_fib_index0;
 270
 271   kv0.key = key0.as_u64;
 272
 273   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 274     {
 275       ip4_address_t * first_int_addr;
 276
 277       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 278         {
 279           first_int_addr =
 280             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 281                                          0 /* just want the address */);
 282           rt->cached_sw_if_index = sw_if_index0;
 283           rt->cached_ip4_address = first_int_addr->as_u32;
 284         }
 285
 286       /* Don't NAT packet aimed at the intfc address */
 287       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 288                                 rt->cached_ip4_address))
 289         return next0;
 290
 291       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 292                          &s0, node, next0);
 293
 294       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 295         return next0;
 296     }
 297   else
 298     s0 = pool_elt_at_index (sm->sessions, value0.value);
 299
 300   old_addr0 = ip0->src_address.as_u32;
 301   ip0->src_address = s0->out2in.addr;
 302   new_addr0 = ip0->src_address.as_u32;
 303   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 304
 305   sum0 = ip0->checksum;
 306   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 307                          ip4_header_t,
 308                          src_address /* changed member */);
 309   ip0->checksum = ip_csum_fold (sum0);
 310
 311   old_id0 = echo0->identifier;
 312   new_id0 = s0->out2in.port;
 313   echo0->identifier = new_id0;
 314
 315   sum0 = icmp0->checksum;
 316   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 317                          identifier);
 318   icmp0->checksum = ip_csum_fold (sum0);
 319
 320   /* Accounting, per-user LRU list maintenance */
 321   s0->last_heard = now;
 322   s0->total_pkts++;
 323   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 324   clib_dlist_remove (sm->list_pool, s0->per_user_index);
 325   clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 326                       s0->per_user_index);
 327
 328   return next0;
 329 }
 330
 331 static inline uword
 332 snat_in2out_node_fn_inline (vlib_main_t * vm,
 333                             vlib_node_runtime_t * node,
 334                             vlib_frame_t * frame, int is_slow_path)
 335 {
 336   u32 n_left_from, * from, * to_next;
 337   snat_in2out_next_t next_index;
 338   u32 pkts_processed = 0;
 339   snat_main_t * sm = &snat_main;
 340   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 341   f64 now = vlib_time_now (vm);
 342   u32 stats_node_index;
 343
 344   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 345     snat_in2out_node.index;
 346
 347   from = vlib_frame_vector_args (frame);
 348   n_left_from = frame->n_vectors;
 349   next_index = node->cached_next_index;
 350
 351   while (n_left_from > 0)
 352     {
 353       u32 n_left_to_next;
 354
 355       vlib_get_next_frame (vm, node, next_index,
 356                            to_next, n_left_to_next);
 357
 358       while (n_left_from >= 4 && n_left_to_next >= 2)
 359         {
 360           u32 bi0, bi1;
 361           vlib_buffer_t * b0, * b1;
 362           u32 next0, next1;
 363           u32 sw_if_index0, sw_if_index1;
 364           ip4_header_t * ip0, * ip1;
 365           ip_csum_t sum0, sum1;
 366           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 367           u16 old_port0, new_port0, old_port1, new_port1;
 368           udp_header_t * udp0, * udp1;
 369           tcp_header_t * tcp0, * tcp1;
 370           icmp46_header_t * icmp0, * icmp1;
 371           snat_session_key_t key0, key1;
 372           u32 rx_fib_index0, rx_fib_index1;
 373           u32 proto0, proto1;
 374           snat_session_t * s0 = 0, * s1 = 0;
 375           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 376
 377           /* Prefetch next iteration. */
 378           {
 379             vlib_buffer_t * p2, * p3;
 380
 381             p2 = vlib_get_buffer (vm, from[2]);
 382             p3 = vlib_get_buffer (vm, from[3]);
 383
 384             vlib_prefetch_buffer_header (p2, LOAD);
 385             vlib_prefetch_buffer_header (p3, LOAD);
 386
 387             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 388             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 389           }
 390
 391           /* speculatively enqueue b0 and b1 to the current next frame */
 392           to_next[0] = bi0 = from[0];
 393           to_next[1] = bi1 = from[1];
 394           from += 2;
 395           to_next += 2;
 396           n_left_from -= 2;
 397           n_left_to_next -= 2;
 398
 399           b0 = vlib_get_buffer (vm, bi0);
 400           b1 = vlib_get_buffer (vm, bi1);
 401
 402           ip0 = vlib_buffer_get_current (b0);
 403           udp0 = ip4_next_header (ip0);
 404           tcp0 = (tcp_header_t *) udp0;
 405           icmp0 = (icmp46_header_t *) udp0;
 406
 407           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 408           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 409                                    sw_if_index0);
 410
 411           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 412
 413 #if 0
 414           /* Formally correct, but we send to slowpath, lookup or drop */
 415           vnet_get_config_data (&cm->config_main,
 416                                 &b0->current_config_index,
 417                                 &next0,
 418                                 0 /* sizeof config data */);
 419 #endif
 420
 421           proto0 = ~0;
 422           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 423             ? SNAT_PROTOCOL_UDP : proto0;
 424           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 425             ? SNAT_PROTOCOL_TCP : proto0;
 426           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 427             ? SNAT_PROTOCOL_ICMP : proto0;
 428
 429           /* Next configured feature, probably ip4-lookup */
 430           if (is_slow_path)
 431             {
 432               if (PREDICT_FALSE (proto0 == ~0))
 433                 goto trace00;
 434
 435               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 436                 {
 437                   next0 = icmp_in2out_slow_path
 438                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 439                      node, next0, now);
 440                   goto trace00;
 441                 }
 442             }
 443           else
 444             {
 445               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 446                 {
 447                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 448                   goto trace00;
 449                 }
 450             }
 451
 452           key0.addr = ip0->src_address;
 453           key0.port = udp0->src_port;
 454           key0.protocol = proto0;
 455           key0.fib_index = rx_fib_index0;
 456
 457           kv0.key = key0.as_u64;
 458
 459           if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
 460             {
 461               if (is_slow_path)
 462                 {
 463                   ip4_address_t * first_int_addr;
 464
 465                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 466                     {
 467                       first_int_addr =
 468                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 469                                                      0 /* just want the address */);
 470                       rt->cached_sw_if_index = sw_if_index0;
 471                       rt->cached_ip4_address = first_int_addr->as_u32;
 472                     }
 473
 474                   /* Don't NAT packet aimed at the intfc address */
 475                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 476                                     rt->cached_ip4_address))
 477                     goto trace00;
 478
 479                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 480                                      &s0, node, next0);
 481                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 482                     goto trace00;
 483                 }
 484               else
 485                 {
 486                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 487                   goto trace00;
 488                 }
 489             }
 490           else
 491             s0 = pool_elt_at_index (sm->sessions, value0.value);
 492
 493           old_addr0 = ip0->src_address.as_u32;
 494           ip0->src_address = s0->out2in.addr;
 495           new_addr0 = ip0->src_address.as_u32;
 496           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 497
 498           sum0 = ip0->checksum;
 499           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 500                                  ip4_header_t,
 501                                  src_address /* changed member */);
 502           ip0->checksum = ip_csum_fold (sum0);
 503
 504           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 505             {
 506               old_port0 = tcp0->ports.src;
 507               tcp0->ports.src = s0->out2in.port;
 508               new_port0 = tcp0->ports.src;
 509
 510               sum0 = tcp0->checksum;
 511               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 512                                      ip4_header_t,
 513                                      dst_address /* changed member */);
 514               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 515                                      ip4_header_t /* cheat */,
 516                                      length /* changed member */);
 517               tcp0->checksum = ip_csum_fold(sum0);
 518             }
 519           else
 520             {
 521               old_port0 = udp0->src_port;
 522               udp0->src_port = s0->out2in.port;
 523               udp0->checksum = 0;
 524             }
 525
 526           /* Accounting, per-user LRU list maintenance */
 527           s0->last_heard = now;
 528           s0->total_pkts++;
 529           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 530           clib_dlist_remove (sm->list_pool, s0->per_user_index);
 531           clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 532                               s0->per_user_index);
 533         trace00:
 534
 535           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 536                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 537             {
 538               snat_in2out_trace_t *t =
 539                  vlib_add_trace (vm, node, b0, sizeof (*t));
 540               t->is_slow_path = is_slow_path;
 541               t->sw_if_index = sw_if_index0;
 542               t->next_index = next0;
 543                   t->session_index = ~0;
 544               if (s0)
 545                   t->session_index = s0 - sm->sessions;
 546             }
 547
 548           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 549
 550           ip1 = vlib_buffer_get_current (b1);
 551           udp1 = ip4_next_header (ip1);
 552           tcp1 = (tcp_header_t *) udp1;
 553           icmp1 = (icmp46_header_t *) udp1;
 554
 555           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 556           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 557                                    sw_if_index1);
 558
 559 #if 0
 560           vnet_get_config_data (&cm->config_main,
 561                                 &b1->current_config_index,
 562                                 &next1,
 563                                 0 /* sizeof config data */);
 564 #endif
 565
 566           proto1 = ~0;
 567           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 568             ? SNAT_PROTOCOL_UDP : proto1;
 569           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 570             ? SNAT_PROTOCOL_TCP : proto1;
 571           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 572             ? SNAT_PROTOCOL_ICMP : proto1;
 573
 574           /* Next configured feature, probably ip4-lookup */
 575           if (is_slow_path)
 576             {
 577               if (PREDICT_FALSE (proto1 == ~0))
 578                 goto trace01;
 579
 580               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 581                 {
 582                   next1 = icmp_in2out_slow_path
 583                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, next1,
 584                      now);
 585                   goto trace01;
 586                 }
 587             }
 588           else
 589             {
 590               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
 591                 {
 592                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 593                   goto trace01;
 594                 }
 595             }
 596
 597           key1.addr = ip1->src_address;
 598           key1.port = udp1->src_port;
 599           key1.protocol = proto1;
 600           key1.fib_index = rx_fib_index1;
 601
 602           kv1.key = key1.as_u64;
 603
 604             if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
 605             {
 606               if (is_slow_path)
 607                 {
 608                   ip4_address_t * first_int_addr;
 609
 610                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index1))
 611                     {
 612                       first_int_addr =
 613                         ip4_interface_first_address (sm->ip4_main, sw_if_index1,
 614                                                      0 /* just want the address */);
 615                       rt->cached_sw_if_index = sw_if_index1;
 616                       rt->cached_ip4_address = first_int_addr->as_u32;
 617                     }
 618
 619                   /* Don't NAT packet aimed at the intfc address */
 620                   if (PREDICT_FALSE(ip1->dst_address.as_u32 ==
 621                                     rt->cached_ip4_address))
 622                     goto trace01;
 623
 624                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
 625                                      &s1, node, next1);
 626                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
 627                     goto trace01;
 628                 }
 629               else
 630                 {
 631                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 632                   goto trace01;
 633                 }
 634             }
 635           else
 636             s1 = pool_elt_at_index (sm->sessions, value1.value);
 637
 638           old_addr1 = ip1->src_address.as_u32;
 639           ip1->src_address = s1->out2in.addr;
 640           new_addr1 = ip1->src_address.as_u32;
 641           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
 642
 643           sum1 = ip1->checksum;
 644           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 645                                  ip4_header_t,
 646                                  src_address /* changed member */);
 647           ip1->checksum = ip_csum_fold (sum1);
 648
 649           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 650             {
 651               old_port1 = tcp1->ports.src;
 652               tcp1->ports.src = s1->out2in.port;
 653               new_port1 = tcp1->ports.src;
 654
 655               sum1 = tcp1->checksum;
 656               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 657                                      ip4_header_t,
 658                                      dst_address /* changed member */);
 659               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 660                                      ip4_header_t /* cheat */,
 661                                      length /* changed member */);
 662               tcp1->checksum = ip_csum_fold(sum1);
 663             }
 664           else
 665             {
 666               old_port1 = udp1->src_port;
 667               udp1->src_port = s1->out2in.port;
 668               udp1->checksum = 0;
 669             }
 670
 671           /* Accounting, per-user LRU list maintenance */
 672           s1->last_heard = now;
 673           s1->total_pkts++;
 674           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 675           clib_dlist_remove (sm->list_pool, s1->per_user_index);
 676           clib_dlist_addtail (sm->list_pool, s1->per_user_list_head_index,
 677                               s1->per_user_index);
 678         trace01:
 679
 680           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 681                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 682             {
 683               snat_in2out_trace_t *t =
 684                  vlib_add_trace (vm, node, b1, sizeof (*t));
 685               t->sw_if_index = sw_if_index1;
 686               t->next_index = next1;
 687               t->session_index = ~0;
 688               if (s1)
 689                 t->session_index = s1 - sm->sessions;
 690             }
 691
 692           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
 693
 694           /* verify speculative enqueues, maybe switch current next frame */
 695           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 696                                            to_next, n_left_to_next,
 697                                            bi0, bi1, next0, next1);
 698         }
 699
 700       while (n_left_from > 0 && n_left_to_next > 0)
 701         {
 702           u32 bi0;
 703           vlib_buffer_t * b0;
 704           u32 next0;
 705           u32 sw_if_index0;
 706           ip4_header_t * ip0;
 707           ip_csum_t sum0;
 708           u32 new_addr0, old_addr0;
 709           u16 old_port0, new_port0;
 710           udp_header_t * udp0;
 711           tcp_header_t * tcp0;
 712           icmp46_header_t * icmp0;
 713           snat_session_key_t key0;
 714           u32 rx_fib_index0;
 715           u32 proto0;
 716           snat_session_t * s0 = 0;
 717           clib_bihash_kv_8_8_t kv0, value0;
 718
 719           /* speculatively enqueue b0 to the current next frame */
 720           bi0 = from[0];
 721           to_next[0] = bi0;
 722           from += 1;
 723           to_next += 1;
 724           n_left_from -= 1;
 725           n_left_to_next -= 1;
 726
 727           b0 = vlib_get_buffer (vm, bi0);
 728           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
 729
 730           ip0 = vlib_buffer_get_current (b0);
 731           udp0 = ip4_next_header (ip0);
 732           tcp0 = (tcp_header_t *) udp0;
 733           icmp0 = (icmp46_header_t *) udp0;
 734
 735           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 736           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 737                                    sw_if_index0);
 738
 739
 740 #if 0
 741           vnet_get_config_data (&cm->config_main,
 742                                 &b0->current_config_index,
 743                                 &next0,
 744                                 0 /* sizeof config data */);
 745 #endif
 746
 747           proto0 = ~0;
 748           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 749             ? SNAT_PROTOCOL_UDP : proto0;
 750           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 751             ? SNAT_PROTOCOL_TCP : proto0;
 752           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 753             ? SNAT_PROTOCOL_ICMP : proto0;
 754
 755           /* Next configured feature, probably ip4-lookup */
 756           if (is_slow_path)
 757             {
 758               if (PREDICT_FALSE (proto0 == ~0))
 759                 goto trace0;
 760
 761               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 762                 {
 763                   next0 = icmp_in2out_slow_path
 764                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0,
 765                      now);
 766                   goto trace0;
 767                 }
 768             }
 769           else
 770             {
 771               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 772                 {
 773                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 774                   goto trace0;
 775                 }
 776             }
 777
 778           key0.addr = ip0->src_address;
 779           key0.port = udp0->src_port;
 780           key0.protocol = proto0;
 781           key0.fib_index = rx_fib_index0;
 782
 783           kv0.key = key0.as_u64;
 784
 785           if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 786             {
 787               if (is_slow_path)
 788                 {
 789                   ip4_address_t * first_int_addr;
 790
 791                   if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 792                     {
 793                       first_int_addr =
 794                         ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 795                                                      0 /* just want the address */);
 796                       rt->cached_sw_if_index = sw_if_index0;
 797                       rt->cached_ip4_address = first_int_addr->as_u32;
 798                     }
 799
 800                   /* Don't NAT packet aimed at the intfc address */
 801                   if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 802                                     rt->cached_ip4_address))
 803                     goto trace0;
 804
 805                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 806                                      &s0, node, next0);
 807                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 808                     goto trace0;
 809                 }
 810               else
 811                 {
 812                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 813                   goto trace0;
 814                 }
 815             }
 816           else
 817             s0 = pool_elt_at_index (sm->sessions, value0.value);
 818
 819           old_addr0 = ip0->src_address.as_u32;
 820           ip0->src_address = s0->out2in.addr;
 821           new_addr0 = ip0->src_address.as_u32;
 822           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 823
 824           sum0 = ip0->checksum;
 825           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 826                                  ip4_header_t,
 827                                  src_address /* changed member */);
 828           ip0->checksum = ip_csum_fold (sum0);
 829
 830           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 831             {
 832               old_port0 = tcp0->ports.src;
 833               tcp0->ports.src = s0->out2in.port;
 834               new_port0 = tcp0->ports.src;
 835
 836               sum0 = tcp0->checksum;
 837               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 838                                      ip4_header_t,
 839                                      dst_address /* changed member */);
 840               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 841                                      ip4_header_t /* cheat */,
 842                                      length /* changed member */);
 843               tcp0->checksum = ip_csum_fold(sum0);
 844             }
 845           else
 846             {
 847               old_port0 = udp0->src_port;
 848               udp0->src_port = s0->out2in.port;
 849               udp0->checksum = 0;
 850             }
 851
 852           /* Accounting, per-user LRU list maintenance */
 853           s0->last_heard = now;
 854           s0->total_pkts++;
 855           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 856           clib_dlist_remove (sm->list_pool, s0->per_user_index);
 857           clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 858                               s0->per_user_index);
 859
 860         trace0:
 861           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 862                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 863             {
 864               snat_in2out_trace_t *t =
 865                  vlib_add_trace (vm, node, b0, sizeof (*t));
 866               t->is_slow_path = is_slow_path;
 867               t->sw_if_index = sw_if_index0;
 868               t->next_index = next0;
 869                   t->session_index = ~0;
 870               if (s0)
 871                   t->session_index = s0 - sm->sessions;
 872             }
 873
 874           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
 875
 876           /* verify speculative enqueue, maybe switch current next frame */
 877           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 878                                            to_next, n_left_to_next,
 879                                            bi0, next0);
 880         }
 881
 882       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 883     }
 884
 885   vlib_node_increment_counter (vm, stats_node_index,
 886                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
 887                                pkts_processed);
 888   return frame->n_vectors;
 889 }
 890
 891 static uword
 892 snat_in2out_fast_path_fn (vlib_main_t * vm,
 893                           vlib_node_runtime_t * node,
 894                           vlib_frame_t * frame)
 895 {
 896   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */);
 897 }
 898
 899 VLIB_REGISTER_NODE (snat_in2out_node) = {
 900   .function = snat_in2out_fast_path_fn,
 901   .name = "snat-in2out",
 902   .vector_size = sizeof (u32),
 903   .format_trace = format_snat_in2out_trace,
 904   .type = VLIB_NODE_TYPE_INTERNAL,
 905
 906   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 907   .error_strings = snat_in2out_error_strings,
 908
 909   .runtime_data_bytes = sizeof (snat_runtime_t),
 910
 911   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 912
 913   /* edit / add dispositions here */
 914   .next_nodes = {
 915     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 916     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 917     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 918   },
 919 };
 920
 921 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
 922
 923 static uword
 924 snat_in2out_slow_path_fn (vlib_main_t * vm,
 925                           vlib_node_runtime_t * node,
 926                           vlib_frame_t * frame)
 927 {
 928   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */);
 929 }
 930
 931 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
 932   .function = snat_in2out_slow_path_fn,
 933   .name = "snat-in2out-slowpath",
 934   .vector_size = sizeof (u32),
 935   .format_trace = format_snat_in2out_trace,
 936   .type = VLIB_NODE_TYPE_INTERNAL,
 937
 938   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
 939   .error_strings = snat_in2out_error_strings,
 940
 941   .runtime_data_bytes = sizeof (snat_runtime_t),
 942
 943   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
 944
 945   /* edit / add dispositions here */
 946   .next_nodes = {
 947     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
 948     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
 949     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
 950   },
 951 };
 952
 953 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn);