plugins/snat-plugin/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <snat/snat.h>
  23
  24 #include <vppinfra/hash.h>
  25 #include <vppinfra/error.h>
  26 #include <vppinfra/elog.h>
  27
  28 typedef struct {
  29   u32 sw_if_index;
  30   u32 next_index;
  31   u32 session_index;
  32 } snat_out2in_trace_t;
  33
  34 /* packet trace format function */
  35 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  36 {
  37   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  38   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  39   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  40
  41   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  42               t->sw_if_index, t->next_index, t->session_index);
  43   return s;
  44 }
  45
  46 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  53               t->sw_if_index, t->next_index);
  54   return s;
  55 }
  56
  57
  58 vlib_node_registration_t snat_out2in_node;
  59 vlib_node_registration_t snat_out2in_fast_node;
  60
  61 #define foreach_snat_out2in_error                       \
  62 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  63 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  64 _(BAD_ICMP_TYPE, "icmp type not echo-reply")            \
  65 _(NO_TRANSLATION, "No translation")
  66
  67 typedef enum {
  68 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  69   foreach_snat_out2in_error
  70 #undef _
  71   SNAT_OUT2IN_N_ERROR,
  72 } snat_out2in_error_t;
  73
  74 static char * snat_out2in_error_strings[] = {
  75 #define _(sym,string) string,
  76   foreach_snat_out2in_error
  77 #undef _
  78 };
  79
  80 typedef enum {
  81   SNAT_OUT2IN_NEXT_DROP,
  82   SNAT_OUT2IN_N_NEXT,
  83 } snat_out2in_next_t;
  84
  85 /**
  86  * @brief Create session for static mapping.
  87  *
  88  * Create NAT session initiated by host from external network with static
  89  * mapping.
  90  *
  91  * @param sm     SNAT main.
  92  * @param b0     Vlib buffer.
  93  * @param in2out In2out SNAT session key.
  94  * @param out2in Out2in SNAT session key.
  95  * @param node   Vlib node.
  96  *
  97  * @returns SNAT session if successfully created otherwise 0.
  98  */
  99 static inline snat_session_t *
 100 create_session_for_static_mapping (snat_main_t *sm,
 101                                    vlib_buffer_t *b0,
 102                                    snat_session_key_t in2out,
 103                                    snat_session_key_t out2in,
 104                                    vlib_node_runtime_t * node)
 105 {
 106   snat_user_t *u;
 107   snat_user_key_t user_key;
 108   snat_session_t *s;
 109   clib_bihash_kv_8_8_t kv0, value0;
 110   dlist_elt_t * per_user_translation_list_elt;
 111   dlist_elt_t * per_user_list_head_elt;
 112
 113   user_key.addr = in2out.addr;
 114   user_key.fib_index = in2out.fib_index;
 115   kv0.key = user_key.as_u64;
 116
 117   /* Ever heard of the "user" = inside ip4 address before? */
 118   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 119     {
 120       /* no, make a new one */
 121       pool_get (sm->users, u);
 122       memset (u, 0, sizeof (*u));
 123       u->addr = in2out.addr;
 124
 125       pool_get (sm->list_pool, per_user_list_head_elt);
 126
 127       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 128         sm->list_pool;
 129
 130       clib_dlist_init (sm->list_pool, u->sessions_per_user_list_head_index);
 131
 132       kv0.value = u - sm->users;
 133
 134       /* add user */
 135       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 136     }
 137   else
 138     {
 139       u = pool_elt_at_index (sm->users, value0.value);
 140     }
 141
 142   pool_get (sm->sessions, s);
 143   memset (s, 0, sizeof (*s));
 144
 145   s->outside_address_index = ~0;
 146   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 147   u->nstaticsessions++;
 148
 149   /* Create list elts */
 150   pool_get (sm->list_pool, per_user_translation_list_elt);
 151   clib_dlist_init (sm->list_pool, per_user_translation_list_elt -
 152                    sm->list_pool);
 153
 154   per_user_translation_list_elt->value = s - sm->sessions;
 155   s->per_user_index = per_user_translation_list_elt - sm->list_pool;
 156   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 157
 158   clib_dlist_addtail (sm->list_pool, s->per_user_list_head_index,
 159                       per_user_translation_list_elt - sm->list_pool);
 160
 161   s->in2out = in2out;
 162   s->out2in = out2in;
 163   s->in2out.protocol = out2in.protocol;
 164
 165   /* Add to translation hashes */
 166   kv0.key = s->in2out.as_u64;
 167   kv0.value = s - sm->sessions;
 168   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 169       clib_warning ("in2out key add failed");
 170
 171   kv0.key = s->out2in.as_u64;
 172   kv0.value = s - sm->sessions;
 173
 174   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 175       clib_warning ("out2in key add failed");
 176
 177   return s;
 178 }
 179
 180 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 181                                          vlib_buffer_t * b0,
 182                                          ip4_header_t * ip0,
 183                                          icmp46_header_t * icmp0,
 184                                          u32 sw_if_index0,
 185                                          u32 rx_fib_index0,
 186                                          vlib_node_runtime_t * node,
 187                                          u32 next0, f64 now)
 188 {
 189   snat_session_key_t key0, sm0;
 190   icmp_echo_header_t *echo0;
 191   clib_bihash_kv_8_8_t kv0, value0;
 192   snat_session_t * s0;
 193   u32 new_addr0, old_addr0;
 194   u16 old_id0, new_id0;
 195   ip_csum_t sum0;
 196   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 197
 198   echo0 = (icmp_echo_header_t *)(icmp0+1);
 199
 200   key0.addr = ip0->dst_address;
 201   key0.port = echo0->identifier;
 202   key0.protocol = SNAT_PROTOCOL_ICMP;
 203   key0.fib_index = rx_fib_index0;
 204
 205   kv0.key = key0.as_u64;
 206
 207   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 208     {
 209       /* Try to match static mapping by external address and port,
 210          destination address and port in packet */
 211       if (snat_static_mapping_match(sm, key0, &sm0, 1))
 212         {
 213            ip4_address_t * first_int_addr;
 214
 215           if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 216             {
 217               first_int_addr =
 218                 ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 219                                              0 /* just want the address */);
 220               rt->cached_sw_if_index = sw_if_index0;
 221               rt->cached_ip4_address = first_int_addr->as_u32;
 222             }
 223
 224           /* Don't NAT packet aimed at the intfc address */
 225           if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 226                             rt->cached_ip4_address))
 227             return next0;
 228
 229           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 230           return SNAT_OUT2IN_NEXT_DROP;
 231         }
 232
 233       /* Create session initiated by host from external network */
 234       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 235                                              node);
 236       if (!s0)
 237         return SNAT_OUT2IN_NEXT_DROP;
 238     }
 239   else
 240     s0 = pool_elt_at_index (sm->sessions, value0.value);
 241
 242   old_addr0 = ip0->dst_address.as_u32;
 243   ip0->dst_address = s0->in2out.addr;
 244   new_addr0 = ip0->dst_address.as_u32;
 245   vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 246
 247   sum0 = ip0->checksum;
 248   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 249                          ip4_header_t,
 250                          dst_address /* changed member */);
 251   ip0->checksum = ip_csum_fold (sum0);
 252
 253   old_id0 = echo0->identifier;
 254   new_id0 = s0->in2out.port;
 255   echo0->identifier = new_id0;
 256
 257   sum0 = icmp0->checksum;
 258   sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 259                          identifier);
 260   icmp0->checksum = ip_csum_fold (sum0);
 261
 262   /* Accounting */
 263   s0->last_heard = now;
 264   s0->total_pkts++;
 265   s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 266   /* Per-user LRU list maintenance for dynamic translation */
 267   if (!snat_is_session_static (s0))
 268     {
 269       clib_dlist_remove (sm->list_pool, s0->per_user_index);
 270       clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 271                           s0->per_user_index);
 272     }
 273
 274   return next0;
 275 }
 276
 277 static uword
 278 snat_out2in_node_fn (vlib_main_t * vm,
 279                   vlib_node_runtime_t * node,
 280                   vlib_frame_t * frame)
 281 {
 282   u32 n_left_from, * from, * to_next;
 283   snat_out2in_next_t next_index;
 284   u32 pkts_processed = 0;
 285   snat_main_t * sm = &snat_main;
 286   ip_lookup_main_t * lm = sm->ip4_lookup_main;
 287   ip_config_main_t * cm = &lm->feature_config_mains[VNET_IP_RX_UNICAST_FEAT];
 288   f64 now = vlib_time_now (vm);
 289
 290   from = vlib_frame_vector_args (frame);
 291   n_left_from = frame->n_vectors;
 292   next_index = node->cached_next_index;
 293
 294   while (n_left_from > 0)
 295     {
 296       u32 n_left_to_next;
 297
 298       vlib_get_next_frame (vm, node, next_index,
 299                            to_next, n_left_to_next);
 300
 301       while (n_left_from >= 4 && n_left_to_next >= 2)
 302         {
 303           u32 bi0, bi1;
 304           vlib_buffer_t * b0, * b1;
 305           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
 306           u32 next1 = SNAT_OUT2IN_NEXT_DROP;
 307           u32 sw_if_index0, sw_if_index1;
 308           ip4_header_t * ip0, *ip1;
 309           ip_csum_t sum0, sum1;
 310           u32 new_addr0, old_addr0;
 311           u16 new_port0, old_port0;
 312           u32 new_addr1, old_addr1;
 313           u16 new_port1, old_port1;
 314           udp_header_t * udp0, * udp1;
 315           tcp_header_t * tcp0, * tcp1;
 316           icmp46_header_t * icmp0, * icmp1;
 317           snat_session_key_t key0, key1, sm0, sm1;
 318           u32 rx_fib_index0, rx_fib_index1;
 319           u32 proto0, proto1;
 320           snat_session_t * s0 = 0, * s1 = 0;
 321           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 322
 323           /* Prefetch next iteration. */
 324           {
 325             vlib_buffer_t * p2, * p3;
 326
 327             p2 = vlib_get_buffer (vm, from[2]);
 328             p3 = vlib_get_buffer (vm, from[3]);
 329
 330             vlib_prefetch_buffer_header (p2, LOAD);
 331             vlib_prefetch_buffer_header (p3, LOAD);
 332
 333             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 334             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 335           }
 336
 337           /* speculatively enqueue b0 and b1 to the current next frame */
 338           to_next[0] = bi0 = from[0];
 339           to_next[1] = bi1 = from[1];
 340           from += 2;
 341           to_next += 2;
 342           n_left_from -= 2;
 343           n_left_to_next -= 2;
 344
 345           b0 = vlib_get_buffer (vm, bi0);
 346           b1 = vlib_get_buffer (vm, bi1);
 347
 348           ip0 = vlib_buffer_get_current (b0);
 349           udp0 = ip4_next_header (ip0);
 350           tcp0 = (tcp_header_t *) udp0;
 351           icmp0 = (icmp46_header_t *) udp0;
 352
 353           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 354           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 355                                    sw_if_index0);
 356
 357           vnet_get_config_data (&cm->config_main,
 358                                 &b0->current_config_index,
 359                                 &next0,
 360                                 0 /* sizeof config data */);
 361           proto0 = ~0;
 362           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 363             ? SNAT_PROTOCOL_UDP : proto0;
 364           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 365             ? SNAT_PROTOCOL_TCP : proto0;
 366           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 367             ? SNAT_PROTOCOL_ICMP : proto0;
 368
 369           if (PREDICT_FALSE (proto0 == ~0))
 370               goto trace0;
 371
 372           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 373             {
 374               next0 = icmp_out2in_slow_path
 375                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 376                  next0, now);
 377               goto trace0;
 378             }
 379
 380           key0.addr = ip0->dst_address;
 381           key0.port = udp0->dst_port;
 382           key0.protocol = proto0;
 383           key0.fib_index = rx_fib_index0;
 384
 385           kv0.key = key0.as_u64;
 386
 387           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 388             {
 389               /* Try to match static mapping by external address and port,
 390                  destination address and port in packet */
 391               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 392                 {
 393                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 394                   goto trace0;
 395                 }
 396
 397               /* Create session initiated by host from external network */
 398               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node);
 399               if (!s0)
 400                 goto trace0;
 401             }
 402           else
 403             s0 = pool_elt_at_index (sm->sessions, value0.value);
 404
 405           old_addr0 = ip0->dst_address.as_u32;
 406           ip0->dst_address = s0->in2out.addr;
 407           new_addr0 = ip0->dst_address.as_u32;
 408           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 409
 410           sum0 = ip0->checksum;
 411           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 412                                  ip4_header_t,
 413                                  dst_address /* changed member */);
 414           ip0->checksum = ip_csum_fold (sum0);
 415
 416           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 417             {
 418               old_port0 = tcp0->ports.dst;
 419               tcp0->ports.dst = s0->in2out.port;
 420               new_port0 = tcp0->ports.dst;
 421
 422               sum0 = tcp0->checksum;
 423               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 424                                      ip4_header_t,
 425                                      dst_address /* changed member */);
 426
 427               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 428                                      ip4_header_t /* cheat */,
 429                                      length /* changed member */);
 430               tcp0->checksum = ip_csum_fold(sum0);
 431             }
 432           else
 433             {
 434               old_port0 = udp0->dst_port;
 435               udp0->dst_port = s0->in2out.port;
 436               udp0->checksum = 0;
 437             }
 438
 439           /* Accounting */
 440           s0->last_heard = now;
 441           s0->total_pkts++;
 442           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 443           /* Per-user LRU list maintenance for dynamic translation */
 444           if (!snat_is_session_static (s0))
 445             {
 446               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 447               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 448                                   s0->per_user_index);
 449             }
 450         trace0:
 451
 452           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 453                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 454             {
 455               snat_out2in_trace_t *t =
 456                  vlib_add_trace (vm, node, b0, sizeof (*t));
 457               t->sw_if_index = sw_if_index0;
 458               t->next_index = next0;
 459               t->session_index = ~0;
 460               if (s0)
 461                   t->session_index = s0 - sm->sessions;
 462             }
 463
 464           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 465
 466
 467           ip1 = vlib_buffer_get_current (b1);
 468           udp1 = ip4_next_header (ip1);
 469           tcp1 = (tcp_header_t *) udp1;
 470           icmp1 = (icmp46_header_t *) udp1;
 471
 472           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 473           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 474                                    sw_if_index1);
 475
 476           vnet_get_config_data (&cm->config_main,
 477                                 &b1->current_config_index,
 478                                 &next1,
 479                                 0 /* sizeof config data */);
 480           proto1 = ~0;
 481           proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
 482             ? SNAT_PROTOCOL_UDP : proto1;
 483           proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
 484             ? SNAT_PROTOCOL_TCP : proto1;
 485           proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
 486             ? SNAT_PROTOCOL_ICMP : proto1;
 487
 488           if (PREDICT_FALSE (proto1 == ~0))
 489               goto trace1;
 490
 491           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 492             {
 493               next1 = icmp_out2in_slow_path
 494                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 495                  next1, now);
 496               goto trace1;
 497             }
 498
 499           key1.addr = ip1->dst_address;
 500           key1.port = udp1->dst_port;
 501           key1.protocol = proto1;
 502           key1.fib_index = rx_fib_index1;
 503
 504           kv1.key = key1.as_u64;
 505
 506           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 507             {
 508               /* Try to match static mapping by external address and port,
 509                  destination address and port in packet */
 510               if (snat_static_mapping_match(sm, key1, &sm1, 1))
 511                 {
 512                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 513                   goto trace1;
 514                 }
 515
 516               /* Create session initiated by host from external network */
 517               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node);
 518               if (!s1)
 519                 goto trace1;
 520             }
 521           else
 522             s1 = pool_elt_at_index (sm->sessions, value1.value);
 523
 524           old_addr1 = ip1->dst_address.as_u32;
 525           ip1->dst_address = s1->in2out.addr;
 526           new_addr1 = ip1->dst_address.as_u32;
 527           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 528
 529           sum1 = ip1->checksum;
 530           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 531                                  ip4_header_t,
 532                                  dst_address /* changed member */);
 533           ip1->checksum = ip_csum_fold (sum1);
 534
 535           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 536             {
 537               old_port1 = tcp1->ports.dst;
 538               tcp1->ports.dst = s1->in2out.port;
 539               new_port1 = tcp1->ports.dst;
 540
 541               sum1 = tcp1->checksum;
 542               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 543                                      ip4_header_t,
 544                                      dst_address /* changed member */);
 545
 546               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 547                                      ip4_header_t /* cheat */,
 548                                      length /* changed member */);
 549               tcp1->checksum = ip_csum_fold(sum1);
 550             }
 551           else
 552             {
 553               old_port1 = udp1->dst_port;
 554               udp1->dst_port = s1->in2out.port;
 555               udp1->checksum = 0;
 556             }
 557
 558           /* Accounting */
 559           s1->last_heard = now;
 560           s1->total_pkts++;
 561           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 562           /* Per-user LRU list maintenance for dynamic translation */
 563           if (!snat_is_session_static (s1))
 564             {
 565               clib_dlist_remove (sm->list_pool, s1->per_user_index);
 566               clib_dlist_addtail (sm->list_pool, s1->per_user_list_head_index,
 567                                   s1->per_user_index);
 568             }
 569         trace1:
 570
 571           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 572                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 573             {
 574               snat_out2in_trace_t *t =
 575                  vlib_add_trace (vm, node, b1, sizeof (*t));
 576               t->sw_if_index = sw_if_index1;
 577               t->next_index = next1;
 578               t->session_index = ~0;
 579               if (s1)
 580                   t->session_index = s1 - sm->sessions;
 581             }
 582
 583           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 584           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 585
 586           /* verify speculative enqueues, maybe switch current next frame */
 587           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 588                                            to_next, n_left_to_next,
 589                                            bi0, bi1, next0, next1);
 590         }
 591
 592       while (n_left_from > 0 && n_left_to_next > 0)
 593         {
 594           u32 bi0;
 595           vlib_buffer_t * b0;
 596           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
 597           u32 sw_if_index0;
 598           ip4_header_t * ip0;
 599           ip_csum_t sum0;
 600           u32 new_addr0, old_addr0;
 601           u16 new_port0, old_port0;
 602           udp_header_t * udp0;
 603           tcp_header_t * tcp0;
 604           icmp46_header_t * icmp0;
 605           snat_session_key_t key0, sm0;
 606           u32 rx_fib_index0;
 607           u32 proto0;
 608           snat_session_t * s0 = 0;
 609           clib_bihash_kv_8_8_t kv0, value0;
 610
 611           /* speculatively enqueue b0 to the current next frame */
 612           bi0 = from[0];
 613           to_next[0] = bi0;
 614           from += 1;
 615           to_next += 1;
 616           n_left_from -= 1;
 617           n_left_to_next -= 1;
 618
 619           b0 = vlib_get_buffer (vm, bi0);
 620
 621           ip0 = vlib_buffer_get_current (b0);
 622           udp0 = ip4_next_header (ip0);
 623           tcp0 = (tcp_header_t *) udp0;
 624           icmp0 = (icmp46_header_t *) udp0;
 625
 626           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 627           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 628                                    sw_if_index0);
 629
 630           vnet_get_config_data (&cm->config_main,
 631                                 &b0->current_config_index,
 632                                 &next0,
 633                                 0 /* sizeof config data */);
 634           proto0 = ~0;
 635           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 636             ? SNAT_PROTOCOL_UDP : proto0;
 637           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 638             ? SNAT_PROTOCOL_TCP : proto0;
 639           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 640             ? SNAT_PROTOCOL_ICMP : proto0;
 641
 642           if (PREDICT_FALSE (proto0 == ~0))
 643               goto trace00;
 644
 645           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 646             {
 647               next0 = icmp_out2in_slow_path
 648                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 649                  next0, now);
 650               goto trace00;
 651             }
 652
 653           key0.addr = ip0->dst_address;
 654           key0.port = udp0->dst_port;
 655           key0.protocol = proto0;
 656           key0.fib_index = rx_fib_index0;
 657
 658           kv0.key = key0.as_u64;
 659
 660           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 661             {
 662               /* Try to match static mapping by external address and port,
 663                  destination address and port in packet */
 664               if (snat_static_mapping_match(sm, key0, &sm0, 1))
 665                 {
 666                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 667                   goto trace00;
 668                 }
 669
 670               /* Create session initiated by host from external network */
 671               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node);
 672               if (!s0)
 673                 goto trace00;
 674             }
 675           else
 676             s0 = pool_elt_at_index (sm->sessions, value0.value);
 677
 678           old_addr0 = ip0->dst_address.as_u32;
 679           ip0->dst_address = s0->in2out.addr;
 680           new_addr0 = ip0->dst_address.as_u32;
 681           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 682
 683           sum0 = ip0->checksum;
 684           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 685                                  ip4_header_t,
 686                                  dst_address /* changed member */);
 687           ip0->checksum = ip_csum_fold (sum0);
 688
 689           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 690             {
 691               old_port0 = tcp0->ports.dst;
 692               tcp0->ports.dst = s0->in2out.port;
 693               new_port0 = tcp0->ports.dst;
 694
 695               sum0 = tcp0->checksum;
 696               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 697                                      ip4_header_t,
 698                                      dst_address /* changed member */);
 699
 700               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 701                                      ip4_header_t /* cheat */,
 702                                      length /* changed member */);
 703               tcp0->checksum = ip_csum_fold(sum0);
 704             }
 705           else
 706             {
 707               old_port0 = udp0->dst_port;
 708               udp0->dst_port = s0->in2out.port;
 709               udp0->checksum = 0;
 710             }
 711
 712           /* Accounting */
 713           s0->last_heard = now;
 714           s0->total_pkts++;
 715           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 716           /* Per-user LRU list maintenance for dynamic translation */
 717           if (!snat_is_session_static (s0))
 718             {
 719               clib_dlist_remove (sm->list_pool, s0->per_user_index);
 720               clib_dlist_addtail (sm->list_pool, s0->per_user_list_head_index,
 721                                   s0->per_user_index);
 722             }
 723         trace00:
 724
 725           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 726                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 727             {
 728               snat_out2in_trace_t *t =
 729                  vlib_add_trace (vm, node, b0, sizeof (*t));
 730               t->sw_if_index = sw_if_index0;
 731               t->next_index = next0;
 732               t->session_index = ~0;
 733               if (s0)
 734                   t->session_index = s0 - sm->sessions;
 735             }
 736
 737           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 738
 739           /* verify speculative enqueue, maybe switch current next frame */
 740           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 741                                            to_next, n_left_to_next,
 742                                            bi0, next0);
 743         }
 744
 745       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 746     }
 747
 748   vlib_node_increment_counter (vm, snat_out2in_node.index,
 749                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
 750                                pkts_processed);
 751   return frame->n_vectors;
 752 }
 753
 754 VLIB_REGISTER_NODE (snat_out2in_node) = {
 755   .function = snat_out2in_node_fn,
 756   .name = "snat-out2in",
 757   .vector_size = sizeof (u32),
 758   .format_trace = format_snat_out2in_trace,
 759   .type = VLIB_NODE_TYPE_INTERNAL,
 760
 761   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
 762   .error_strings = snat_out2in_error_strings,
 763
 764   .runtime_data_bytes = sizeof (snat_runtime_t),
 765
 766   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
 767
 768   /* edit / add dispositions here */
 769   .next_nodes = {
 770     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
 771   },
 772 };
 773 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
 774
 775 static inline u32 icmp_out2in_fast (snat_main_t *sm,
 776                                     vlib_buffer_t * b0,
 777                                     ip4_header_t * ip0,
 778                                     icmp46_header_t * icmp0,
 779                                     u32 sw_if_index0,
 780                                     vlib_node_runtime_t * node,
 781                                     u32 next0)
 782 {
 783   snat_session_key_t key0, sm0;
 784   icmp_echo_header_t *echo0;
 785   u32 new_addr0, old_addr0;
 786   u16 old_id0, new_id0;
 787   ip_csum_t sum0;
 788   snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
 789
 790   echo0 = (icmp_echo_header_t *)(icmp0+1);
 791
 792   key0.addr = ip0->dst_address;
 793   key0.port = echo0->identifier;
 794
 795   if (snat_static_mapping_match(sm, key0, &sm0, 1))
 796     {
 797       ip4_address_t * first_int_addr;
 798
 799       if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
 800         {
 801           first_int_addr =
 802             ip4_interface_first_address (sm->ip4_main, sw_if_index0,
 803                                          0 /* just want the address */);
 804           rt->cached_sw_if_index = sw_if_index0;
 805           rt->cached_ip4_address = first_int_addr->as_u32;
 806         }
 807
 808       /* Don't NAT packet aimed at the intfc address */
 809       if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
 810                         rt->cached_ip4_address))
 811         return next0;
 812
 813       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 814       return SNAT_OUT2IN_NEXT_DROP;
 815     }
 816
 817   new_addr0 = sm0.addr.as_u32;
 818   new_id0 = sm0.port;
 819   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 820
 821   old_addr0 = ip0->dst_address.as_u32;
 822   ip0->dst_address.as_u32 = new_addr0;
 823
 824   sum0 = ip0->checksum;
 825   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 826                          ip4_header_t,
 827                          dst_address /* changed member */);
 828   ip0->checksum = ip_csum_fold (sum0);
 829
 830   if (PREDICT_FALSE(new_id0 != echo0->identifier))
 831     {
 832       old_id0 = echo0->identifier;
 833       echo0->identifier = new_id0;
 834
 835       sum0 = icmp0->checksum;
 836       sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 837                              identifier);
 838       icmp0->checksum = ip_csum_fold (sum0);
 839     }
 840
 841   return next0;
 842 }
 843
 844 static uword
 845 snat_out2in_fast_node_fn (vlib_main_t * vm,
 846                           vlib_node_runtime_t * node,
 847                           vlib_frame_t * frame)
 848 {
 849   u32 n_left_from, * from, * to_next;
 850   snat_out2in_next_t next_index;
 851   u32 pkts_processed = 0;
 852   snat_main_t * sm = &snat_main;
 853   ip_lookup_main_t * lm = sm->ip4_lookup_main;
 854   ip_config_main_t * cm = &lm->feature_config_mains[VNET_IP_RX_UNICAST_FEAT];
 855
 856   from = vlib_frame_vector_args (frame);
 857   n_left_from = frame->n_vectors;
 858   next_index = node->cached_next_index;
 859
 860   while (n_left_from > 0)
 861     {
 862       u32 n_left_to_next;
 863
 864       vlib_get_next_frame (vm, node, next_index,
 865                            to_next, n_left_to_next);
 866
 867       while (n_left_from > 0 && n_left_to_next > 0)
 868         {
 869           u32 bi0;
 870           vlib_buffer_t * b0;
 871           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
 872           u32 sw_if_index0;
 873           ip4_header_t * ip0;
 874           ip_csum_t sum0;
 875           u32 new_addr0, old_addr0;
 876           u16 new_port0, old_port0;
 877           udp_header_t * udp0;
 878           tcp_header_t * tcp0;
 879           icmp46_header_t * icmp0;
 880           snat_session_key_t key0, sm0;
 881           u32 proto0;
 882
 883           /* speculatively enqueue b0 to the current next frame */
 884           bi0 = from[0];
 885           to_next[0] = bi0;
 886           from += 1;
 887           to_next += 1;
 888           n_left_from -= 1;
 889           n_left_to_next -= 1;
 890
 891           b0 = vlib_get_buffer (vm, bi0);
 892
 893           ip0 = vlib_buffer_get_current (b0);
 894           udp0 = ip4_next_header (ip0);
 895           tcp0 = (tcp_header_t *) udp0;
 896           icmp0 = (icmp46_header_t *) udp0;
 897
 898           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 899
 900           vnet_get_config_data (&cm->config_main,
 901                                 &b0->current_config_index,
 902                                 &next0,
 903                                 0 /* sizeof config data */);
 904           proto0 = ~0;
 905           proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
 906             ? SNAT_PROTOCOL_UDP : proto0;
 907           proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
 908             ? SNAT_PROTOCOL_TCP : proto0;
 909           proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
 910             ? SNAT_PROTOCOL_ICMP : proto0;
 911
 912           if (PREDICT_FALSE (proto0 == ~0))
 913               goto trace00;
 914
 915           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 916             {
 917               next0 = icmp_out2in_fast
 918                 (sm, b0, ip0, icmp0, sw_if_index0, node, next0);
 919               goto trace00;
 920             }
 921
 922           key0.addr = ip0->dst_address;
 923           key0.port = udp0->dst_port;
 924
 925           if (snat_static_mapping_match(sm, key0, &sm0, 1))
 926             {
 927               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 928               goto trace00;
 929             }
 930
 931           new_addr0 = sm0.addr.as_u32;
 932           new_port0 = sm0.port;
 933           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 934           old_addr0 = ip0->dst_address.as_u32;
 935           ip0->dst_address.as_u32 = new_addr0;
 936
 937           sum0 = ip0->checksum;
 938           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 939                                  ip4_header_t,
 940                                  dst_address /* changed member */);
 941           ip0->checksum = ip_csum_fold (sum0);
 942
 943           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
 944             {
 945                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 946                 {
 947                   old_port0 = tcp0->ports.dst;
 948                   tcp0->ports.dst = new_port0;
 949
 950                   sum0 = tcp0->checksum;
 951                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 952                                          ip4_header_t,
 953                                          dst_address /* changed member */);
 954
 955                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
 956                                          ip4_header_t /* cheat */,
 957                                          length /* changed member */);
 958                   tcp0->checksum = ip_csum_fold(sum0);
 959                 }
 960               else
 961                 {
 962                   old_port0 = udp0->dst_port;
 963                   udp0->dst_port = new_port0;
 964                   udp0->checksum = 0;
 965                 }
 966             }
 967           else
 968             {
 969               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 970                 {
 971                   sum0 = tcp0->checksum;
 972                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 973                                          ip4_header_t,
 974                                          dst_address /* changed member */);
 975
 976                   tcp0->checksum = ip_csum_fold(sum0);
 977                 }
 978             }
 979
 980         trace00:
 981
 982           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 983                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 984             {
 985               snat_out2in_trace_t *t =
 986                  vlib_add_trace (vm, node, b0, sizeof (*t));
 987               t->sw_if_index = sw_if_index0;
 988               t->next_index = next0;
 989             }
 990
 991           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 992
 993           /* verify speculative enqueue, maybe switch current next frame */
 994           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 995                                            to_next, n_left_to_next,
 996                                            bi0, next0);
 997         }
 998
 999       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1000     }
1001
1002   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1003                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1004                                pkts_processed);
1005   return frame->n_vectors;
1006 }
1007
1008 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1009   .function = snat_out2in_fast_node_fn,
1010   .name = "snat-out2in-fast",
1011   .vector_size = sizeof (u32),
1012   .format_trace = format_snat_out2in_fast_trace,
1013   .type = VLIB_NODE_TYPE_INTERNAL,
1014
1015   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1016   .error_strings = snat_out2in_error_strings,
1017
1018   .runtime_data_bytes = sizeof (snat_runtime_t),
1019
1020   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1021
1022   /* edit / add dispositions here */
1023   .next_nodes = {
1024     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1025   },
1026 };
1027 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);