plugins/snat-plugin/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <snat/snat.h>
  24
  25 #include <vppinfra/hash.h>
  26 #include <vppinfra/error.h>
  27 #include <vppinfra/elog.h>
  28
  29 typedef struct {
  30   u32 sw_if_index;
  31   u32 next_index;
  32   u32 session_index;
  33 } snat_out2in_trace_t;
  34
  35 /* packet trace format function */
  36 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  37 {
  38   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  39   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  40   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  41
  42   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  43               t->sw_if_index, t->next_index, t->session_index);
  44   return s;
  45 }
  46
  47 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  52
  53   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  54               t->sw_if_index, t->next_index);
  55   return s;
  56 }
  57
  58
  59 vlib_node_registration_t snat_out2in_node;
  60 vlib_node_registration_t snat_out2in_fast_node;
  61
  62 #define foreach_snat_out2in_error                       \
  63 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  64 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  65 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  66 _(NO_TRANSLATION, "No translation")
  67
  68 typedef enum {
  69 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  70   foreach_snat_out2in_error
  71 #undef _
  72   SNAT_OUT2IN_N_ERROR,
  73 } snat_out2in_error_t;
  74
  75 static char * snat_out2in_error_strings[] = {
  76 #define _(sym,string) string,
  77   foreach_snat_out2in_error
  78 #undef _
  79 };
  80
  81 typedef enum {
  82   SNAT_OUT2IN_NEXT_DROP,
  83   SNAT_OUT2IN_N_NEXT,
  84 } snat_out2in_next_t;
  85
  86 /**
  87  * @brief Create session for static mapping.
  88  *
  89  * Create NAT session initiated by host from external network with static
  90  * mapping.
  91  *
  92  * @param sm     SNAT main.
  93  * @param b0     Vlib buffer.
  94  * @param in2out In2out SNAT session key.
  95  * @param out2in Out2in SNAT session key.
  96  * @param node   Vlib node.
  97  *
  98  * @returns SNAT session if successfully created otherwise 0.
  99  */
 100 static inline snat_session_t *
 101 create_session_for_static_mapping (snat_main_t *sm,
 102                                    vlib_buffer_t *b0,
 103                                    snat_session_key_t in2out,
 104                                    snat_session_key_t out2in,
 105                                    vlib_node_runtime_t * node)
 106 {
 107   snat_user_t *u;
 108   snat_user_key_t user_key;
 109   snat_session_t *s;
 110   clib_bihash_kv_8_8_t kv0, value0;
 111   dlist_elt_t * per_user_translation_list_elt;
 112   dlist_elt_t * per_user_list_head_elt;
 113
 114   user_key.addr = in2out.addr;
 115   user_key.fib_index = in2out.fib_index;
 116   kv0.key = user_key.as_u64;
 117
 118   /* Ever heard of the "user" = inside ip4 address before? */
 119   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 120     {
 121       /* no, make a new one */
 122       pool_get (sm->users, u);
 123       memset (u, 0, sizeof (*u));
 124       u->addr = in2out.addr;
 125
 126       pool_get (sm->list_pool, per_user_list_head_elt);
 127
 128       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 129         sm->list_pool;
 130
 131       clib_dlist_init (sm->list_pool, u->sessions_per_user_list_head_index);
 132
 133       kv0.value = u - sm->users;
 134
 135       /* add user */
 136       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 137     }
 138   else
 139     {
 140       u = pool_elt_at_index (sm->users, value0.value);
 141     }
 142
 143   pool_get (sm->sessions, s);
 144   memset (s, 0, sizeof (*s));
 145
 146   s->outside_address_index = ~0;
 147   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 148   u->nstaticsessions++;
 149
 150   /* Create list elts */
 151   pool_get (sm->list_pool, per_user_translation_list_elt);
 152   clib_dlist_init (sm->list_pool, per_user_translation_list_elt -
 153                    sm->list_pool);
 154
 155   per_user_translation_list_elt->value = s - sm->sessions;
 156   s->per_user_index = per_user_translation_list_elt - sm->list_pool;
 157   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 158
 159   clib_dlist_addtail (sm->list_pool, s->per_user_list_head_index,
 160                       per_user_translation_list_elt - sm->list_pool);
 161
 162   s->in2out = in2out;
 163   s->out2in = out2in;
 164   s->in2out.protocol = out2in.protocol;
 165
 166   /* Add to translation hashes */
 167   kv0.key = s->in2out.as_u64;
 168   kv0.value = s - sm->sessions;
 169   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 170       clib_warning ("in2out key add failed");
 171
 172   kv0.key = s->out2in.as_u64;
 173   kv0.value = s - sm->sessions;
 174
 175   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 176       clib_warning ("out2in key add failed");
 177
 178   return s;
 179 }
 180
 181 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 182                                          vlib_buffer_t * b0,
 183                                          ip4_header_t * ip0,
 184                                          icmp46_header_t * icmp0,
 185                                          u32 sw_if_index0,
 186                                          u32 rx_fib_index0,
 187                                          vlib_node_runtime_t * node,
 188                                          u32 next0, f64 now)
 189 {
 190   snat_session_key_t key0, sm0;
 191   icmp_echo_header_t *echo0;
 192   clib_bihash_kv_8_8_t kv0, value0;
 193   snat_session_t * s0;
 194   u32 new_addr0, old_addr0;
 195   u16 old_id0, new_id0;
 196   ip_csum_t sum0;
 197   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 198
 199   echo0 = (icmp_echo_header_t *)(icmp0+1);
 200
 201   key0.addr = ip0->dst_address;
 202   key0.port = echo0->identifier;
 203   key0.protocol = SNAT_PROTOCOL_ICMP;
 204   key0.fib_index = rx_fib_index0;
 205
 206   kv0.key = key0.as_u64;
 207
 208   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 209     {
 210       /* Try to match static mapping by external address and port,
 211          destination address and port in packet */
 212       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 213         {
 214            ip4_address_t * first_int_addr;
 215
 216           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 217             {
 218               first_int_addr =
 219                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 220                                              0 /* just want the address */);
 221               rt->cached_sw_if_index = sw_if_index0;
 222               rt->cached_ip4_address = first_int_addr->as_u32;
 223             }
 224
 225           /* Don't NAT packet aimed at the intfc address */
 226           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 227                             rt->cached_ip4_address))
 228             return next0;
 229
 230           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 231           return SNAT_OUT2IN_NEXT_DROP;
 232         }
 233
 234       /* Create session initiated by host from external network */
 235       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 236                                              node);
 237       if (!s0)
 238         return SNAT_OUT2IN_NEXT_DROP;
 239     }
 240   else
 241     s0 = pool_elt_at_index (sm->sessions, value0.value);
 242
 243   old_addr0 = ip0->dst_address.as_u32;
 244   ip0->dst_address = s0->in2out.addr;
 245   new_addr0 = ip0->dst_address.as_u32;
 246   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 247
 248   sum0 = ip0->checksum;
 249   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 250                          ip4_header_t,
 251                          dst_address /* changed member */);
 252   ip0->checksum = ip_csum_fold (sum0);
 253
 254   old_id0 = echo0->identifier;
 255   new_id0 = s0->in2out.port;
 256   echo0->identifier = new_id0;
 257
 258   sum0 = icmp0->checksum;
 259   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 260                          identifier);
 261   icmp0->checksum = ip_csum_fold (sum0);
 262
 263   /* Accounting */
 264   s0->last_heard = now;
 265   s0->total_pkts++;
 266   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 267   /* Per-user LRU list maintenance for dynamic translation */
 268   if (!snat_is_session_static (s0))
 269     {
 270       clib_dlist_remove (sm->list_pool, s0->per_user_index);
 271       clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 272                           s0->per_user_index);
 273     }
 274
 275   return next0;
 276 }
 277
 278 static uword
 279 snat_out2in_node_fn (vlib_main_t * vm,
 280                   vlib_node_runtime_t * node,
 281                   vlib_frame_t * frame)
 282 {
 283   u32 n_left_from, * from, * to_next;
 284   snat_out2in_next_t next_index;
 285   u32 pkts_processed = 0;
 286   snat_main_t * sm = &snat_main;
 287   ip_lookup_main_t * lm = sm->ip4_lookup_main;
 288   ip_config_main_t * cm = &lm->feature_config_mains[VNET_IP_RX_UNICAST_FEAT];
 289   f64 now = vlib_time_now (vm);
 290
 291   from = vlib_frame_vector_args (frame);
 292   n_left_from = frame->n_vectors;
 293   next_index = node->cached_next_index;
 294
 295   while (n_left_from > 0)
 296     {
 297       u32 n_left_to_next;
 298
 299       vlib_get_next_frame (vm, node, next_index,
 300                            to_next, n_left_to_next);
 301
 302       while (n_left_from >= 4 && n_left_to_next >= 2)
 303         {
 304           u32 bi0, bi1;
 305           vlib_buffer_t * b0, * b1;
 306           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
 307           u32 next1 = SNAT_OUT2IN_NEXT_DROP;
 308           u32 sw_if_index0, sw_if_index1;
 309           ip4_header_t * ip0, *ip1;
 310           ip_csum_t sum0, sum1;
 311           u32 new_addr0, old_addr0;
 312           u16 new_port0, old_port0;
 313           u32 new_addr1, old_addr1;
 314           u16 new_port1, old_port1;
 315           udp_header_t * udp0, * udp1;
 316           tcp_header_t * tcp0, * tcp1;
 317           icmp46_header_t * icmp0, * icmp1;
 318           snat_session_key_t key0, key1, sm0, sm1;
 319           u32 rx_fib_index0, rx_fib_index1;
 320           u32 proto0, proto1;
 321           snat_session_t * s0 = 0, * s1 = 0;
 322           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 323
 324           /* Prefetch next iteration. */
 325           {
 326             vlib_buffer_t * p2, * p3;
 327
 328             p2 = vlib_get_buffer (vm, from[2]);
 329             p3 = vlib_get_buffer (vm, from[3]);
 330
 331             vlib_prefetch_buffer_header (p2, LOAD);
 332             vlib_prefetch_buffer_header (p3, LOAD);
 333
 334             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 335             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 336           }
 337
 338           /* speculatively enqueue b0 and b1 to the current next frame */
 339           to_next[0] = bi0 = from[0];
 340           to_next[1] = bi1 = from[1];
 341           from += 2;
 342           to_next += 2;
 343           n_left_from -= 2;
 344           n_left_to_next -= 2;
 345
 346           b0 = vlib_get_buffer (vm, bi0);
 347           b1 = vlib_get_buffer (vm, bi1);
 348
 349           ip0 = vlib_buffer_get_current (b0);
 350           udp0 = ip4_next_header (ip0);
 351           tcp0 = (tcp_header_t *) udp0;
 352           icmp0 = (icmp46_header_t *) udp0;
 353
 354           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 355           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 356                                    sw_if_index0);
 357
 358           vnet_get_config_data (&cm->config_main,
 359                                 &b0->current_config_index,
 360                                 &next0,
 361                                 0 /* sizeof config data */);
 362           proto0 = ~0;
 363           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 364             ? SNAT_PROTOCOL_UDP : proto0;
 365           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 366             ? SNAT_PROTOCOL_TCP : proto0;
 367           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 368             ? SNAT_PROTOCOL_ICMP : proto0;
 369
 370           if (PREDICT_FALSE (proto0 == ~0))
 371               goto trace0;
 372
 373           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 374             {
 375               next0 = icmp_out2in_slow_path
 376                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 377                  next0, now);
 378               goto trace0;
 379             }
 380
 381           key0.addr = ip0->dst_address;
 382           key0.port = udp0->dst_port;
 383           key0.protocol = proto0;
 384           key0.fib_index = rx_fib_index0;
 385
 386           kv0.key = key0.as_u64;
 387
 388           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 389             {
 390               /* Try to match static mapping by external address and port,
 391                  destination address and port in packet */
 392               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 393                 {
 394                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 395                   goto trace0;
 396                 }
 397
 398               /* Create session initiated by host from external network */
 399               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node);
 400               if (!s0)
 401                 goto trace0;
 402             }
 403           else
 404             s0 = pool_elt_at_index (sm->sessions, value0.value);
 405
 406           old_addr0 = ip0->dst_address.as_u32;
 407           ip0->dst_address = s0->in2out.addr;
 408           new_addr0 = ip0->dst_address.as_u32;
 409           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 410
 411           sum0 = ip0->checksum;
 412           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 413                                  ip4_header_t,
 414                                  dst_address /* changed member */);
 415           ip0->checksum = ip_csum_fold (sum0);
 416
 417           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 418             {
 419               old_port0 = tcp0->ports.dst;
 420               tcp0->ports.dst = s0->in2out.port;
 421               new_port0 = tcp0->ports.dst;
 422
 423               sum0 = tcp0->checksum;
 424               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 425                                      ip4_header_t,
 426                                      dst_address /* changed member */);
 427
 428               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 429                                      ip4_header_t /* cheat */,
 430                                      length /* changed member */);
 431               tcp0->checksum = ip_csum_fold(sum0);
 432             }
 433           else
 434             {
 435               old_port0 = udp0->dst_port;
 436               udp0->dst_port = s0->in2out.port;
 437               udp0->checksum = 0;
 438             }
 439
 440           /* Accounting */
 441           s0->last_heard = now;
 442           s0->total_pkts++;
 443           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 444           /* Per-user LRU list maintenance for dynamic translation */
 445           if (!snat_is_session_static (s0))
 446             {
 447               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 448               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 449                                   s0->per_user_index);
 450             }
 451         trace0:
 452
 453           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 454                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 455             {
 456               snat_out2in_trace_t *t =
 457                  vlib_add_trace (vm, node, b0, sizeof (*t));
 458               t->sw_if_index = sw_if_index0;
 459               t->next_index = next0;
 460               t->session_index = ~0;
 461               if (s0)
 462                   t->session_index = s0 - sm->sessions;
 463             }
 464
 465           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 466
 467
 468           ip1 = vlib_buffer_get_current (b1);
 469           udp1 = ip4_next_header (ip1);
 470           tcp1 = (tcp_header_t *) udp1;
 471           icmp1 = (icmp46_header_t *) udp1;
 472
 473           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 474           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 475                                    sw_if_index1);
 476
 477           vnet_get_config_data (&cm->config_main,
 478                                 &b1->current_config_index,
 479                                 &next1,
 480                                 0 /* sizeof config data */);
 481           proto1 = ~0;
 482           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 483             ? SNAT_PROTOCOL_UDP : proto1;
 484           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 485             ? SNAT_PROTOCOL_TCP : proto1;
 486           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 487             ? SNAT_PROTOCOL_ICMP : proto1;
 488
 489           if (PREDICT_FALSE (proto1 == ~0))
 490               goto trace1;
 491
 492           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 493             {
 494               next1 = icmp_out2in_slow_path
 495                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 496                  next1, now);
 497               goto trace1;
 498             }
 499
 500           key1.addr = ip1->dst_address;
 501           key1.port = udp1->dst_port;
 502           key1.protocol = proto1;
 503           key1.fib_index = rx_fib_index1;
 504
 505           kv1.key = key1.as_u64;
 506
 507           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 508             {
 509               /* Try to match static mapping by external address and port,
 510                  destination address and port in packet */
 511               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 512                 {
 513                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 514                   goto trace1;
 515                 }
 516
 517               /* Create session initiated by host from external network */
 518               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node);
 519               if (!s1)
 520                 goto trace1;
 521             }
 522           else
 523             s1 = pool_elt_at_index (sm->sessions, value1.value);
 524
 525           old_addr1 = ip1->dst_address.as_u32;
 526           ip1->dst_address = s1->in2out.addr;
 527           new_addr1 = ip1->dst_address.as_u32;
 528           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 529
 530           sum1 = ip1->checksum;
 531           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 532                                  ip4_header_t,
 533                                  dst_address /* changed member */);
 534           ip1->checksum = ip_csum_fold (sum1);
 535
 536           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 537             {
 538               old_port1 = tcp1->ports.dst;
 539               tcp1->ports.dst = s1->in2out.port;
 540               new_port1 = tcp1->ports.dst;
 541
 542               sum1 = tcp1->checksum;
 543               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 544                                      ip4_header_t,
 545                                      dst_address /* changed member */);
 546
 547               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 548                                      ip4_header_t /* cheat */,
 549                                      length /* changed member */);
 550               tcp1->checksum = ip_csum_fold(sum1);
 551             }
 552           else
 553             {
 554               old_port1 = udp1->dst_port;
 555               udp1->dst_port = s1->in2out.port;
 556               udp1->checksum = 0;
 557             }
 558
 559           /* Accounting */
 560           s1->last_heard = now;
 561           s1->total_pkts++;
 562           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 563           /* Per-user LRU list maintenance for dynamic translation */
 564           if (!snat_is_session_static (s1))
 565             {
 566               clib_dlist_remove (sm->list_pool, s1->per_user_index);
 567               clib_dlist_addtail (sm->list_pool, s1->per_user_list_head_index,
 568                                   s1->per_user_index);
 569             }
 570         trace1:
 571
 572           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 573                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 574             {
 575               snat_out2in_trace_t *t =
 576                  vlib_add_trace (vm, node, b1, sizeof (*t));
 577               t->sw_if_index = sw_if_index1;
 578               t->next_index = next1;
 579               t->session_index = ~0;
 580               if (s1)
 581                   t->session_index = s1 - sm->sessions;
 582             }
 583
 584           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 585           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 586
 587           /* verify speculative enqueues, maybe switch current next frame */
 588           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 589                                            to_next, n_left_to_next,
 590                                            bi0, bi1, next0, next1);
 591         }
 592
 593       while (n_left_from > 0 && n_left_to_next > 0)
 594         {
 595           u32 bi0;
 596           vlib_buffer_t * b0;
 597           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
 598           u32 sw_if_index0;
 599           ip4_header_t * ip0;
 600           ip_csum_t sum0;
 601           u32 new_addr0, old_addr0;
 602           u16 new_port0, old_port0;
 603           udp_header_t * udp0;
 604           tcp_header_t * tcp0;
 605           icmp46_header_t * icmp0;
 606           snat_session_key_t key0, sm0;
 607           u32 rx_fib_index0;
 608           u32 proto0;
 609           snat_session_t * s0 = 0;
 610           clib_bihash_kv_8_8_t kv0, value0;
 611
 612           /* speculatively enqueue b0 to the current next frame */
 613           bi0 = from[0];
 614           to_next[0] = bi0;
 615           from += 1;
 616           to_next += 1;
 617           n_left_from -= 1;
 618           n_left_to_next -= 1;
 619
 620           b0 = vlib_get_buffer (vm, bi0);
 621
 622           ip0 = vlib_buffer_get_current (b0);
 623           udp0 = ip4_next_header (ip0);
 624           tcp0 = (tcp_header_t *) udp0;
 625           icmp0 = (icmp46_header_t *) udp0;
 626
 627           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 628           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 629                                    sw_if_index0);
 630
 631           vnet_get_config_data (&cm->config_main,
 632                                 &b0->current_config_index,
 633                                 &next0,
 634                                 0 /* sizeof config data */);
 635           proto0 = ~0;
 636           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 637             ? SNAT_PROTOCOL_UDP : proto0;
 638           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 639             ? SNAT_PROTOCOL_TCP : proto0;
 640           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 641             ? SNAT_PROTOCOL_ICMP : proto0;
 642
 643           if (PREDICT_FALSE (proto0 == ~0))
 644               goto trace00;
 645
 646           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 647             {
 648               next0 = icmp_out2in_slow_path
 649                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 650                  next0, now);
 651               goto trace00;
 652             }
 653
 654           key0.addr = ip0->dst_address;
 655           key0.port = udp0->dst_port;
 656           key0.protocol = proto0;
 657           key0.fib_index = rx_fib_index0;
 658
 659           kv0.key = key0.as_u64;
 660
 661           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 662             {
 663               /* Try to match static mapping by external address and port,
 664                  destination address and port in packet */
 665               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 666                 {
 667                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 668                   goto trace00;
 669                 }
 670
 671               /* Create session initiated by host from external network */
 672               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node);
 673               if (!s0)
 674                 goto trace00;
 675             }
 676           else
 677             s0 = pool_elt_at_index (sm->sessions, value0.value);
 678
 679           old_addr0 = ip0->dst_address.as_u32;
 680           ip0->dst_address = s0->in2out.addr;
 681           new_addr0 = ip0->dst_address.as_u32;
 682           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 683
 684           sum0 = ip0->checksum;
 685           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 686                                  ip4_header_t,
 687                                  dst_address /* changed member */);
 688           ip0->checksum = ip_csum_fold (sum0);
 689
 690           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 691             {
 692               old_port0 = tcp0->ports.dst;
 693               tcp0->ports.dst = s0->in2out.port;
 694               new_port0 = tcp0->ports.dst;
 695
 696               sum0 = tcp0->checksum;
 697               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 698                                      ip4_header_t,
 699                                      dst_address /* changed member */);
 700
 701               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 702                                      ip4_header_t /* cheat */,
 703                                      length /* changed member */);
 704               tcp0->checksum = ip_csum_fold(sum0);
 705             }
 706           else
 707             {
 708               old_port0 = udp0->dst_port;
 709               udp0->dst_port = s0->in2out.port;
 710               udp0->checksum = 0;
 711             }
 712
 713           /* Accounting */
 714           s0->last_heard = now;
 715           s0->total_pkts++;
 716           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 717           /* Per-user LRU list maintenance for dynamic translation */
 718           if (!snat_is_session_static (s0))
 719             {
 720               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 721               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 722                                   s0->per_user_index);
 723             }
 724         trace00:
 725
 726           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 727                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 728             {
 729               snat_out2in_trace_t *t =
 730                  vlib_add_trace (vm, node, b0, sizeof (*t));
 731               t->sw_if_index = sw_if_index0;
 732               t->next_index = next0;
 733               t->session_index = ~0;
 734               if (s0)
 735                   t->session_index = s0 - sm->sessions;
 736             }
 737
 738           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 739
 740           /* verify speculative enqueue, maybe switch current next frame */
 741           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 742                                            to_next, n_left_to_next,
 743                                            bi0, next0);
 744         }
 745
 746       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 747     }
 748
 749   vlib_node_increment_counter (vm, snat_out2in_node.index,
 750                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 751                                pkts_processed);
 752   return frame->n_vectors;
 753 }
 754
 755 VLIB_REGISTER_NODE (snat_out2in_node) = {
 756   .function = snat_out2in_node_fn,
 757   .name = "snat-out2in",
 758   .vector_size = sizeof (u32),
 759   .format_trace = format_snat_out2in_trace,
 760   .type = VLIB_NODE_TYPE_INTERNAL,
 761
 762   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 763   .error_strings = snat_out2in_error_strings,
 764
 765   .runtime_data_bytes = sizeof (snat_runtime_t),
 766
 767   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 768
 769   /* edit / add dispositions here */
 770   .next_nodes = {
 771     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 772   },
 773 };
 774 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 775
 776 static inline u32 icmp_out2in_fast (snat_main_t *sm,
 777                                     vlib_buffer_t * b0,
 778                                     ip4_header_t * ip0,
 779                                     icmp46_header_t * icmp0,
 780                                     u32 sw_if_index0,
 781                                     vlib_node_runtime_t * node,
 782                                     u32 next0,
 783                                     u32 rx_fib_index0)
 784 {
 785   snat_session_key_t key0, sm0;
 786   icmp_echo_header_t *echo0;
 787   u32 new_addr0, old_addr0;
 788   u16 old_id0, new_id0;
 789   ip_csum_t sum0;
 790   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 791
 792   echo0 = (icmp_echo_header_t *)(icmp0+1);
 793
 794   key0.addr = ip0->dst_address;
 795   key0.port = echo0->identifier;
 796   key0.fib_index = rx_fib_index0;
 797
 798   if (snat_static_mapping_match(sm, key0, &sm0, 1))
 799     {
 800       ip4_address_t * first_int_addr;
 801
 802       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 803         {
 804           first_int_addr =
 805             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 806                                          0 /* just want the address */);
 807           rt->cached_sw_if_index = sw_if_index0;
 808           rt->cached_ip4_address = first_int_addr->as_u32;
 809         }
 810
 811       /* Don't NAT packet aimed at the intfc address */
 812       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 813                         rt->cached_ip4_address))
 814         return next0;
 815
 816       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 817       return SNAT_OUT2IN_NEXT_DROP;
 818     }
 819
 820   new_addr0 = sm0.addr.as_u32;
 821   new_id0 = sm0.port;
 822   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 823
 824   old_addr0 = ip0->dst_address.as_u32;
 825   ip0->dst_address.as_u32 = new_addr0;
 826
 827   sum0 = ip0->checksum;
 828   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 829                          ip4_header_t,
 830                          dst_address /* changed member */);
 831   ip0->checksum = ip_csum_fold (sum0);
 832
 833   if (PREDICT_FALSE(new_id0 != echo0->identifier))
 834     {
 835       old_id0 = echo0->identifier;
 836       echo0->identifier = new_id0;
 837
 838       sum0 = icmp0->checksum;
 839       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 840                              identifier);
 841       icmp0->checksum = ip_csum_fold (sum0);
 842     }
 843
 844   return next0;
 845 }
 846
 847 static uword
 848 snat_out2in_fast_node_fn (vlib_main_t * vm,
 849                           vlib_node_runtime_t * node,
 850                           vlib_frame_t * frame)
 851 {
 852   u32 n_left_from, * from, * to_next;
 853   snat_out2in_next_t next_index;
 854   u32 pkts_processed = 0;
 855   snat_main_t * sm = &snat_main;
 856   ip_lookup_main_t * lm = sm->ip4_lookup_main;
 857   ip_config_main_t * cm = &lm->feature_config_mains[VNET_IP_RX_UNICAST_FEAT];
 858
 859   from = vlib_frame_vector_args (frame);
 860   n_left_from = frame->n_vectors;
 861   next_index = node->cached_next_index;
 862
 863   while (n_left_from > 0)
 864     {
 865       u32 n_left_to_next;
 866
 867       vlib_get_next_frame (vm, node, next_index,
 868                            to_next, n_left_to_next);
 869
 870       while (n_left_from > 0 && n_left_to_next > 0)
 871         {
 872           u32 bi0;
 873           vlib_buffer_t * b0;
 874           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
 875           u32 sw_if_index0;
 876           ip4_header_t * ip0;
 877           ip_csum_t sum0;
 878           u32 new_addr0, old_addr0;
 879           u16 new_port0, old_port0;
 880           udp_header_t * udp0;
 881           tcp_header_t * tcp0;
 882           icmp46_header_t * icmp0;
 883           snat_session_key_t key0, sm0;
 884           u32 proto0;
 885           u32 rx_fib_index0;
 886
 887           /* speculatively enqueue b0 to the current next frame */
 888           bi0 = from[0];
 889           to_next[0] = bi0;
 890           from += 1;
 891           to_next += 1;
 892           n_left_from -= 1;
 893           n_left_to_next -= 1;
 894
 895           b0 = vlib_get_buffer (vm, bi0);
 896
 897           ip0 = vlib_buffer_get_current (b0);
 898           udp0 = ip4_next_header (ip0);
 899           tcp0 = (tcp_header_t *) udp0;
 900           icmp0 = (icmp46_header_t *) udp0;
 901
 902           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 903           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
 904
 905           vnet_get_config_data (&cm->config_main,
 906                                 &b0->current_config_index,
 907                                 &next0,
 908                                 0 /* sizeof config data */);
 909           proto0 = ~0;
 910           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 911             ? SNAT_PROTOCOL_UDP : proto0;
 912           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 913             ? SNAT_PROTOCOL_TCP : proto0;
 914           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 915             ? SNAT_PROTOCOL_ICMP : proto0;
 916
 917           if (PREDICT_FALSE (proto0 == ~0))
 918               goto trace00;
 919
 920           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 921             {
 922               next0 = icmp_out2in_fast
 923                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
 924               goto trace00;
 925             }
 926
 927           key0.addr = ip0->dst_address;
 928           key0.port = udp0->dst_port;
 929           key0.fib_index = rx_fib_index0;
 930
 931           if (snat_static_mapping_match(sm, key0, &sm0, 1))
 932             {
 933               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 934               goto trace00;
 935             }
 936
 937           new_addr0 = sm0.addr.as_u32;
 938           new_port0 = sm0.port;
 939           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 940           old_addr0 = ip0->dst_address.as_u32;
 941           ip0->dst_address.as_u32 = new_addr0;
 942
 943           sum0 = ip0->checksum;
 944           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 945                                  ip4_header_t,
 946                                  dst_address /* changed member */);
 947           ip0->checksum = ip_csum_fold (sum0);
 948
 949           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
 950             {
 951                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 952                 {
 953                   old_port0 = tcp0->ports.dst;
 954                   tcp0->ports.dst = new_port0;
 955
 956                   sum0 = tcp0->checksum;
 957                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 958                                          ip4_header_t,
 959                                          dst_address /* changed member */);
 960
 961                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
 962                                          ip4_header_t /* cheat */,
 963                                          length /* changed member */);
 964                   tcp0->checksum = ip_csum_fold(sum0);
 965                 }
 966               else
 967                 {
 968                   old_port0 = udp0->dst_port;
 969                   udp0->dst_port = new_port0;
 970                   udp0->checksum = 0;
 971                 }
 972             }
 973           else
 974             {
 975               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 976                 {
 977                   sum0 = tcp0->checksum;
 978                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 979                                          ip4_header_t,
 980                                          dst_address /* changed member */);
 981
 982                   tcp0->checksum = ip_csum_fold(sum0);
 983                 }
 984             }
 985
 986         trace00:
 987
 988           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 989                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 990             {
 991               snat_out2in_trace_t *t =
 992                  vlib_add_trace (vm, node, b0, sizeof (*t));
 993               t->sw_if_index = sw_if_index0;
 994               t->next_index = next0;
 995             }
 996
 997           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 998
 999           /* verify speculative enqueue, maybe switch current next frame */
1000           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1001                                            to_next, n_left_to_next,
1002                                            bi0, next0);
1003         }
1004
1005       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1006     }
1007
1008   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1009                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1010                                pkts_processed);
1011   return frame->n_vectors;
1012 }
1013
1014 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1015   .function = snat_out2in_fast_node_fn,
1016   .name = "snat-out2in-fast",
1017   .vector_size = sizeof (u32),
1018   .format_trace = format_snat_out2in_fast_trace,
1019   .type = VLIB_NODE_TYPE_INTERNAL,
1020
1021   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1022   .error_strings = snat_out2in_error_strings,
1023
1024   .runtime_data_bytes = sizeof (snat_runtime_t),
1025
1026   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1027
1028   /* edit / add dispositions here */
1029   .next_nodes = {
1030     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1031   },
1032 };
1033 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);