src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")                     \
  91 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")
  92
  93 typedef enum {
  94 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  95   foreach_snat_out2in_error
  96 #undef _
  97   SNAT_OUT2IN_N_ERROR,
  98 } snat_out2in_error_t;
  99
 100 static char * snat_out2in_error_strings[] = {
 101 #define _(sym,string) string,
 102   foreach_snat_out2in_error
 103 #undef _
 104 };
 105
 106 typedef enum {
 107   SNAT_OUT2IN_NEXT_DROP,
 108   SNAT_OUT2IN_NEXT_LOOKUP,
 109   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 110   SNAT_OUT2IN_N_NEXT,
 111 } snat_out2in_next_t;
 112
 113 /**
 114  * @brief Create session for static mapping.
 115  *
 116  * Create NAT session initiated by host from external network with static
 117  * mapping.
 118  *
 119  * @param sm     NAT main.
 120  * @param b0     Vlib buffer.
 121  * @param in2out In2out NAT44 session key.
 122  * @param out2in Out2in NAT44 session key.
 123  * @param node   Vlib node.
 124  *
 125  * @returns SNAT session if successfully created otherwise 0.
 126  */
 127 static inline snat_session_t *
 128 create_session_for_static_mapping (snat_main_t *sm,
 129                                    vlib_buffer_t *b0,
 130                                    snat_session_key_t in2out,
 131                                    snat_session_key_t out2in,
 132                                    vlib_node_runtime_t * node,
 133                                    u32 thread_index)
 134 {
 135   snat_user_t *u;
 136   snat_user_key_t user_key;
 137   snat_session_t *s;
 138   clib_bihash_kv_8_8_t kv0, value0;
 139   dlist_elt_t * per_user_translation_list_elt;
 140   dlist_elt_t * per_user_list_head_elt;
 141   ip4_header_t *ip0;
 142
 143   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 144     {
 145       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 146       return 0;
 147     }
 148
 149   ip0 = vlib_buffer_get_current (b0);
 150
 151   user_key.addr = in2out.addr;
 152   user_key.fib_index = in2out.fib_index;
 153   kv0.key = user_key.as_u64;
 154
 155   /* Ever heard of the "user" = inside ip4 address before? */
 156   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 157                               &kv0, &value0))
 158     {
 159       /* no, make a new one */
 160       pool_get (sm->per_thread_data[thread_index].users, u);
 161       memset (u, 0, sizeof (*u));
 162       u->addr = in2out.addr;
 163       u->fib_index = in2out.fib_index;
 164
 165       pool_get (sm->per_thread_data[thread_index].list_pool,
 166                 per_user_list_head_elt);
 167
 168       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 169         sm->per_thread_data[thread_index].list_pool;
 170
 171       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 172                        u->sessions_per_user_list_head_index);
 173
 174       kv0.value = u - sm->per_thread_data[thread_index].users;
 175
 176       /* add user */
 177       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 178                                &kv0, 1 /* is_add */);
 179
 180       /* add non-traslated packets worker lookup */
 181       kv0.value = thread_index;
 182       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 183     }
 184   else
 185     {
 186       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 187                              value0.value);
 188     }
 189
 190   pool_get (sm->per_thread_data[thread_index].sessions, s);
 191   memset (s, 0, sizeof (*s));
 192
 193   s->outside_address_index = ~0;
 194   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 195   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 196   u->nstaticsessions++;
 197
 198   /* Create list elts */
 199   pool_get (sm->per_thread_data[thread_index].list_pool,
 200             per_user_translation_list_elt);
 201   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 202                    per_user_translation_list_elt -
 203                    sm->per_thread_data[thread_index].list_pool);
 204
 205   per_user_translation_list_elt->value =
 206     s - sm->per_thread_data[thread_index].sessions;
 207   s->per_user_index =
 208     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 209   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 210
 211   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 212                       s->per_user_list_head_index,
 213                       per_user_translation_list_elt -
 214                       sm->per_thread_data[thread_index].list_pool);
 215
 216   s->in2out = in2out;
 217   s->out2in = out2in;
 218   s->in2out.protocol = out2in.protocol;
 219
 220   /* Add to translation hashes */
 221   kv0.key = s->in2out.as_u64;
 222   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 223   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 224                                1 /* is_add */))
 225       clib_warning ("in2out key add failed");
 226
 227   kv0.key = s->out2in.as_u64;
 228   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 229
 230   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 231                                1 /* is_add */))
 232       clib_warning ("out2in key add failed");
 233
 234   /* log NAT event */
 235   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 236                                       s->out2in.addr.as_u32,
 237                                       s->in2out.protocol,
 238                                       s->in2out.port,
 239                                       s->out2in.port,
 240                                       s->in2out.fib_index);
 241    return s;
 242 }
 243
 244 static_always_inline
 245 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 246                                  snat_session_key_t *p_key0)
 247 {
 248   icmp46_header_t *icmp0;
 249   snat_session_key_t key0;
 250   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 251   ip4_header_t *inner_ip0;
 252   void *l4_header = 0;
 253   icmp46_header_t *inner_icmp0;
 254
 255   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 256   echo0 = (icmp_echo_header_t *)(icmp0+1);
 257
 258   if (!icmp_is_error_message (icmp0))
 259     {
 260       key0.protocol = SNAT_PROTOCOL_ICMP;
 261       key0.addr = ip0->dst_address;
 262       key0.port = echo0->identifier;
 263     }
 264   else
 265     {
 266       inner_ip0 = (ip4_header_t *)(echo0+1);
 267       l4_header = ip4_next_header (inner_ip0);
 268       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 269       key0.addr = inner_ip0->src_address;
 270       switch (key0.protocol)
 271         {
 272         case SNAT_PROTOCOL_ICMP:
 273           inner_icmp0 = (icmp46_header_t*)l4_header;
 274           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 275           key0.port = inner_echo0->identifier;
 276           break;
 277         case SNAT_PROTOCOL_UDP:
 278         case SNAT_PROTOCOL_TCP:
 279           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 280           break;
 281         default:
 282           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 283         }
 284     }
 285   *p_key0 = key0;
 286   return -1; /* success */
 287 }
 288
 289 /**
 290  * Get address and port values to be used for ICMP packet translation
 291  * and create session if needed
 292  *
 293  * @param[in,out] sm             NAT main
 294  * @param[in,out] node           NAT node runtime
 295  * @param[in] thread_index       thread index
 296  * @param[in,out] b0             buffer containing packet to be translated
 297  * @param[out] p_proto           protocol used for matching
 298  * @param[out] p_value           address and port after NAT translation
 299  * @param[out] p_dont_translate  if packet should not be translated
 300  * @param d                      optional parameter
 301  * @param e                      optional parameter
 302  */
 303 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 304                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 305                            snat_session_key_t *p_value,
 306                            u8 *p_dont_translate, void *d, void *e)
 307 {
 308   ip4_header_t *ip0;
 309   icmp46_header_t *icmp0;
 310   u32 sw_if_index0;
 311   u32 rx_fib_index0;
 312   snat_session_key_t key0;
 313   snat_session_key_t sm0;
 314   snat_session_t *s0 = 0;
 315   u8 dont_translate = 0;
 316   clib_bihash_kv_8_8_t kv0, value0;
 317   u8 is_addr_only;
 318   u32 next0 = ~0;
 319   int err;
 320
 321   ip0 = vlib_buffer_get_current (b0);
 322   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 323   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 324   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 325
 326   key0.protocol = 0;
 327
 328   err = icmp_get_key (ip0, &key0);
 329   if (err != -1)
 330     {
 331       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 332       next0 = SNAT_OUT2IN_NEXT_DROP;
 333       goto out;
 334     }
 335   key0.fib_index = rx_fib_index0;
 336
 337   kv0.key = key0.as_u64;
 338
 339   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 340                               &value0))
 341     {
 342       /* Try to match static mapping by external address and port,
 343          destination address and port in packet */
 344       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 345         {
 346           /* Don't NAT packet aimed at the intfc address */
 347           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 348                                               ip0->dst_address.as_u32)))
 349             {
 350               dont_translate = 1;
 351               goto out;
 352             }
 353           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 354           next0 = SNAT_OUT2IN_NEXT_DROP;
 355           goto out;
 356         }
 357
 358       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 359                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 360         {
 361           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 362           next0 = SNAT_OUT2IN_NEXT_DROP;
 363           goto out;
 364         }
 365
 366       /* Create session initiated by host from external network */
 367       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 368                                              node, thread_index);
 369
 370       if (!s0)
 371         {
 372           next0 = SNAT_OUT2IN_NEXT_DROP;
 373           goto out;
 374         }
 375     }
 376   else
 377     {
 378       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 379                         icmp0->type != ICMP4_echo_request &&
 380                         !icmp_is_error_message (icmp0)))
 381         {
 382           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 383           next0 = SNAT_OUT2IN_NEXT_DROP;
 384           goto out;
 385         }
 386
 387       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 388                               value0.value);
 389     }
 390
 391 out:
 392   *p_proto = key0.protocol;
 393   if (s0)
 394     *p_value = s0->in2out;
 395   *p_dont_translate = dont_translate;
 396   if (d)
 397     *(snat_session_t**)d = s0;
 398   return next0;
 399 }
 400
 401 /**
 402  * Get address and port values to be used for ICMP packet translation
 403  *
 404  * @param[in] sm                 NAT main
 405  * @param[in,out] node           NAT node runtime
 406  * @param[in] thread_index       thread index
 407  * @param[in,out] b0             buffer containing packet to be translated
 408  * @param[out] p_proto           protocol used for matching
 409  * @param[out] p_value           address and port after NAT translation
 410  * @param[out] p_dont_translate  if packet should not be translated
 411  * @param d                      optional parameter
 412  * @param e                      optional parameter
 413  */
 414 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 415                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 416                            snat_session_key_t *p_value,
 417                            u8 *p_dont_translate, void *d, void *e)
 418 {
 419   ip4_header_t *ip0;
 420   icmp46_header_t *icmp0;
 421   u32 sw_if_index0;
 422   u32 rx_fib_index0;
 423   snat_session_key_t key0;
 424   snat_session_key_t sm0;
 425   u8 dont_translate = 0;
 426   u8 is_addr_only;
 427   u32 next0 = ~0;
 428   int err;
 429
 430   ip0 = vlib_buffer_get_current (b0);
 431   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 432   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 433   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 434
 435   err = icmp_get_key (ip0, &key0);
 436   if (err != -1)
 437     {
 438       b0->error = node->errors[err];
 439       next0 = SNAT_OUT2IN_NEXT_DROP;
 440       goto out2;
 441     }
 442   key0.fib_index = rx_fib_index0;
 443
 444   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 445     {
 446       /* Don't NAT packet aimed at the intfc address */
 447       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 448         {
 449           dont_translate = 1;
 450           goto out;
 451         }
 452       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 453       next0 = SNAT_OUT2IN_NEXT_DROP;
 454       goto out;
 455     }
 456
 457   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 458                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 459                     !icmp_is_error_message (icmp0)))
 460     {
 461       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 462       next0 = SNAT_OUT2IN_NEXT_DROP;
 463       goto out;
 464     }
 465
 466 out:
 467   *p_value = sm0;
 468 out2:
 469   *p_proto = key0.protocol;
 470   *p_dont_translate = dont_translate;
 471   return next0;
 472 }
 473
 474 static inline u32 icmp_out2in (snat_main_t *sm,
 475                                vlib_buffer_t * b0,
 476                                ip4_header_t * ip0,
 477                                icmp46_header_t * icmp0,
 478                                u32 sw_if_index0,
 479                                u32 rx_fib_index0,
 480                                vlib_node_runtime_t * node,
 481                                u32 next0,
 482                                u32 thread_index,
 483                                void *d,
 484                                void *e)
 485 {
 486   snat_session_key_t sm0;
 487   u8 protocol;
 488   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 489   ip4_header_t *inner_ip0 = 0;
 490   void *l4_header = 0;
 491   icmp46_header_t *inner_icmp0;
 492   u8 dont_translate;
 493   u32 new_addr0, old_addr0;
 494   u16 old_id0, new_id0;
 495   ip_csum_t sum0;
 496   u16 checksum0;
 497   u32 next0_tmp;
 498
 499   echo0 = (icmp_echo_header_t *)(icmp0+1);
 500
 501   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 502                                        &protocol, &sm0, &dont_translate, d, e);
 503   if (next0_tmp != ~0)
 504     next0 = next0_tmp;
 505   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 506     goto out;
 507
 508   sum0 = ip_incremental_checksum (0, icmp0,
 509                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 510   checksum0 = ~ip_csum_fold (sum0);
 511   if (checksum0 != 0 && checksum0 != 0xffff)
 512     {
 513       next0 = SNAT_OUT2IN_NEXT_DROP;
 514       goto out;
 515     }
 516
 517   old_addr0 = ip0->dst_address.as_u32;
 518   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 519   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 520
 521   sum0 = ip0->checksum;
 522   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 523                          dst_address /* changed member */);
 524   ip0->checksum = ip_csum_fold (sum0);
 525
 526   if (!icmp_is_error_message (icmp0))
 527     {
 528       new_id0 = sm0.port;
 529       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 530         {
 531           old_id0 = echo0->identifier;
 532           new_id0 = sm0.port;
 533           echo0->identifier = new_id0;
 534
 535           sum0 = icmp0->checksum;
 536           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 537                                  identifier /* changed member */);
 538           icmp0->checksum = ip_csum_fold (sum0);
 539         }
 540     }
 541   else
 542     {
 543       inner_ip0 = (ip4_header_t *)(echo0+1);
 544       l4_header = ip4_next_header (inner_ip0);
 545
 546       if (!ip4_header_checksum_is_valid (inner_ip0))
 547         {
 548           next0 = SNAT_OUT2IN_NEXT_DROP;
 549           goto out;
 550         }
 551
 552       old_addr0 = inner_ip0->src_address.as_u32;
 553       inner_ip0->src_address = sm0.addr;
 554       new_addr0 = inner_ip0->src_address.as_u32;
 555
 556       sum0 = icmp0->checksum;
 557       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 558                              src_address /* changed member */);
 559       icmp0->checksum = ip_csum_fold (sum0);
 560
 561       switch (protocol)
 562         {
 563         case SNAT_PROTOCOL_ICMP:
 564           inner_icmp0 = (icmp46_header_t*)l4_header;
 565           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 566
 567           old_id0 = inner_echo0->identifier;
 568           new_id0 = sm0.port;
 569           inner_echo0->identifier = new_id0;
 570
 571           sum0 = icmp0->checksum;
 572           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 573                                  identifier);
 574           icmp0->checksum = ip_csum_fold (sum0);
 575           break;
 576         case SNAT_PROTOCOL_UDP:
 577         case SNAT_PROTOCOL_TCP:
 578           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 579           new_id0 = sm0.port;
 580           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 581
 582           sum0 = icmp0->checksum;
 583           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 584                                  src_port);
 585           icmp0->checksum = ip_csum_fold (sum0);
 586           break;
 587         default:
 588           ASSERT(0);
 589         }
 590     }
 591
 592 out:
 593   return next0;
 594 }
 595
 596
 597 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 598                                          vlib_buffer_t * b0,
 599                                          ip4_header_t * ip0,
 600                                          icmp46_header_t * icmp0,
 601                                          u32 sw_if_index0,
 602                                          u32 rx_fib_index0,
 603                                          vlib_node_runtime_t * node,
 604                                          u32 next0, f64 now,
 605                                          u32 thread_index,
 606                                          snat_session_t ** p_s0)
 607 {
 608   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 609                       next0, thread_index, p_s0, 0);
 610   snat_session_t * s0 = *p_s0;
 611   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 612     {
 613       /* Accounting */
 614       s0->last_heard = now;
 615       s0->total_pkts++;
 616       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 617       /* Per-user LRU list maintenance for dynamic translation */
 618       if (!snat_is_session_static (s0))
 619         {
 620           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 621                              s0->per_user_index);
 622           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 623                               s0->per_user_list_head_index,
 624                               s0->per_user_index);
 625         }
 626     }
 627   return next0;
 628 }
 629
 630 static snat_session_t *
 631 snat_out2in_unknown_proto (snat_main_t *sm,
 632                            vlib_buffer_t * b,
 633                            ip4_header_t * ip,
 634                            u32 rx_fib_index,
 635                            u32 thread_index,
 636                            f64 now,
 637                            vlib_main_t * vm,
 638                            vlib_node_runtime_t * node)
 639 {
 640   clib_bihash_kv_8_8_t kv, value;
 641   clib_bihash_kv_16_8_t s_kv, s_value;
 642   snat_static_mapping_t *m;
 643   snat_session_key_t m_key;
 644   u32 old_addr, new_addr;
 645   ip_csum_t sum;
 646   nat_ed_ses_key_t key;
 647   snat_session_t * s;
 648   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 649   snat_user_key_t u_key;
 650   snat_user_t *u;
 651   dlist_elt_t *head, *elt;
 652
 653   old_addr = ip->dst_address.as_u32;
 654
 655   key.l_addr = ip->dst_address;
 656   key.r_addr = ip->src_address;
 657   key.fib_index = rx_fib_index;
 658   key.proto = ip->protocol;
 659   key.rsvd = 0;
 660   key.l_port = 0;
 661   s_kv.key[0] = key.as_u64[0];
 662   s_kv.key[1] = key.as_u64[1];
 663
 664   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 665     {
 666       s = pool_elt_at_index (tsm->sessions, s_value.value);
 667       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 668     }
 669   else
 670     {
 671       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 672         {
 673           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 674           return 0;
 675         }
 676
 677       m_key.addr = ip->dst_address;
 678       m_key.port = 0;
 679       m_key.protocol = 0;
 680       m_key.fib_index = rx_fib_index;
 681       kv.key = m_key.as_u64;
 682       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 683         {
 684           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 685           return 0;
 686         }
 687
 688       m = pool_elt_at_index (sm->static_mappings, value.value);
 689
 690       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 691
 692       u_key.addr = ip->src_address;
 693       u_key.fib_index = m->fib_index;
 694       kv.key = u_key.as_u64;
 695
 696       /* Ever heard of the "user" = src ip4 address before? */
 697       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 698         {
 699           /* no, make a new one */
 700           pool_get (tsm->users, u);
 701           memset (u, 0, sizeof (*u));
 702           u->addr = ip->src_address;
 703           u->fib_index = rx_fib_index;
 704
 705           pool_get (tsm->list_pool, head);
 706           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 707
 708           clib_dlist_init (tsm->list_pool,
 709                            u->sessions_per_user_list_head_index);
 710
 711           kv.value = u - tsm->users;
 712
 713           /* add user */
 714           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
 715         }
 716       else
 717         {
 718           u = pool_elt_at_index (tsm->users, value.value);
 719         }
 720
 721       /* Create a new session */
 722       pool_get (tsm->sessions, s);
 723       memset (s, 0, sizeof (*s));
 724
 725       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 726       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 727       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 728       s->outside_address_index = ~0;
 729       s->out2in.addr.as_u32 = old_addr;
 730       s->out2in.fib_index = rx_fib_index;
 731       s->in2out.addr.as_u32 = new_addr;
 732       s->in2out.fib_index = m->fib_index;
 733       s->in2out.port = s->out2in.port = ip->protocol;
 734       u->nstaticsessions++;
 735
 736       /* Create list elts */
 737       pool_get (tsm->list_pool, elt);
 738       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 739       elt->value = s - tsm->sessions;
 740       s->per_user_index = elt - tsm->list_pool;
 741       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 742       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 743                           s->per_user_index);
 744
 745       /* Add to lookup tables */
 746       s_kv.value = s - tsm->sessions;
 747       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 748         clib_warning ("out2in key add failed");
 749
 750       key.l_addr = ip->dst_address;
 751       key.fib_index = m->fib_index;
 752       s_kv.key[0] = key.as_u64[0];
 753       s_kv.key[1] = key.as_u64[1];
 754       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 755         clib_warning ("in2out key add failed");
 756    }
 757
 758   /* Update IP checksum */
 759   sum = ip->checksum;
 760   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 761   ip->checksum = ip_csum_fold (sum);
 762
 763   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 764
 765   /* Accounting */
 766   s->last_heard = now;
 767   s->total_pkts++;
 768   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 769   /* Per-user LRU list maintenance */
 770   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 771   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 772                       s->per_user_index);
 773
 774   return s;
 775 }
 776
 777 static snat_session_t *
 778 snat_out2in_lb (snat_main_t *sm,
 779                 vlib_buffer_t * b,
 780                 ip4_header_t * ip,
 781                 u32 rx_fib_index,
 782                 u32 thread_index,
 783                 f64 now,
 784                 vlib_main_t * vm,
 785                 vlib_node_runtime_t * node)
 786 {
 787   nat_ed_ses_key_t key;
 788   clib_bihash_kv_16_8_t s_kv, s_value;
 789   udp_header_t *udp = ip4_next_header (ip);
 790   tcp_header_t *tcp = (tcp_header_t *) udp;
 791   snat_session_t *s = 0;
 792   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 793   snat_session_key_t e_key, l_key;
 794   clib_bihash_kv_8_8_t kv, value;
 795   u32 old_addr, new_addr;
 796   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 797   u16 new_port, old_port;
 798   ip_csum_t sum;
 799   snat_user_key_t u_key;
 800   snat_user_t *u;
 801   dlist_elt_t *head, *elt;
 802
 803   old_addr = ip->dst_address.as_u32;
 804
 805   key.l_addr = ip->dst_address;
 806   key.r_addr = ip->src_address;
 807   key.fib_index = rx_fib_index;
 808   key.proto = ip->protocol;
 809   key.rsvd = 0;
 810   key.l_port = udp->dst_port;
 811   s_kv.key[0] = key.as_u64[0];
 812   s_kv.key[1] = key.as_u64[1];
 813
 814   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 815     {
 816       s = pool_elt_at_index (tsm->sessions, s_value.value);
 817     }
 818   else
 819     {
 820       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 821         {
 822           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 823           return 0;
 824         }
 825
 826       e_key.addr = ip->dst_address;
 827       e_key.port = udp->dst_port;
 828       e_key.protocol = proto;
 829       e_key.fib_index = rx_fib_index;
 830       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
 831         return 0;
 832
 833       u_key.addr = l_key.addr;
 834       u_key.fib_index = l_key.fib_index;
 835       kv.key = u_key.as_u64;
 836
 837       /* Ever heard of the "user" = src ip4 address before? */
 838       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 839         {
 840           /* no, make a new one */
 841           pool_get (tsm->users, u);
 842           memset (u, 0, sizeof (*u));
 843           u->addr = l_key.addr;
 844           u->fib_index = l_key.fib_index;
 845
 846           pool_get (tsm->list_pool, head);
 847           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 848
 849           clib_dlist_init (tsm->list_pool,
 850                            u->sessions_per_user_list_head_index);
 851
 852           kv.value = u - tsm->users;
 853
 854           /* add user */
 855           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 856             clib_warning ("user key add failed");
 857         }
 858       else
 859         {
 860           u = pool_elt_at_index (tsm->users, value.value);
 861         }
 862
 863       /* Create a new session */
 864       pool_get (tsm->sessions, s);
 865       memset (s, 0, sizeof (*s));
 866
 867       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 868       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 869       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 870       s->outside_address_index = ~0;
 871       s->out2in = e_key;
 872       s->in2out = l_key;
 873       u->nstaticsessions++;
 874
 875       /* Create list elts */
 876       pool_get (tsm->list_pool, elt);
 877       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 878       elt->value = s - tsm->sessions;
 879       s->per_user_index = elt - tsm->list_pool;
 880       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 881       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 882                           s->per_user_index);
 883
 884       /* Add to lookup tables */
 885       s_kv.value = s - tsm->sessions;
 886       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 887         clib_warning ("out2in-ed key add failed");
 888
 889       key.l_addr = l_key.addr;
 890       key.fib_index = l_key.fib_index;
 891       key.l_port = l_key.port;
 892       s_kv.key[0] = key.as_u64[0];
 893       s_kv.key[1] = key.as_u64[1];
 894       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 895         clib_warning ("in2out-ed key add failed");
 896     }
 897
 898   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 899
 900   /* Update IP checksum */
 901   sum = ip->checksum;
 902   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 903   ip->checksum = ip_csum_fold (sum);
 904
 905   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 906     {
 907       old_port = tcp->dst_port;
 908       tcp->dst_port = s->in2out.port;
 909       new_port = tcp->dst_port;
 910
 911       sum = tcp->checksum;
 912       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 913       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 914       tcp->checksum = ip_csum_fold(sum);
 915     }
 916   else
 917     {
 918       udp->dst_port = s->in2out.port;
 919       udp->checksum = 0;
 920     }
 921
 922   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 923
 924   /* Accounting */
 925   s->last_heard = now;
 926   s->total_pkts++;
 927   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 928   return s;
 929 }
 930
 931 static uword
 932 snat_out2in_node_fn (vlib_main_t * vm,
 933                   vlib_node_runtime_t * node,
 934                   vlib_frame_t * frame)
 935 {
 936   u32 n_left_from, * from, * to_next;
 937   snat_out2in_next_t next_index;
 938   u32 pkts_processed = 0;
 939   snat_main_t * sm = &snat_main;
 940   f64 now = vlib_time_now (vm);
 941   u32 thread_index = vlib_get_thread_index ();
 942
 943   from = vlib_frame_vector_args (frame);
 944   n_left_from = frame->n_vectors;
 945   next_index = node->cached_next_index;
 946
 947   while (n_left_from > 0)
 948     {
 949       u32 n_left_to_next;
 950
 951       vlib_get_next_frame (vm, node, next_index,
 952                            to_next, n_left_to_next);
 953
 954       while (n_left_from >= 4 && n_left_to_next >= 2)
 955         {
 956           u32 bi0, bi1;
 957           vlib_buffer_t * b0, * b1;
 958           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 959           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 960           u32 sw_if_index0, sw_if_index1;
 961           ip4_header_t * ip0, *ip1;
 962           ip_csum_t sum0, sum1;
 963           u32 new_addr0, old_addr0;
 964           u16 new_port0, old_port0;
 965           u32 new_addr1, old_addr1;
 966           u16 new_port1, old_port1;
 967           udp_header_t * udp0, * udp1;
 968           tcp_header_t * tcp0, * tcp1;
 969           icmp46_header_t * icmp0, * icmp1;
 970           snat_session_key_t key0, key1, sm0, sm1;
 971           u32 rx_fib_index0, rx_fib_index1;
 972           u32 proto0, proto1;
 973           snat_session_t * s0 = 0, * s1 = 0;
 974           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 975
 976           /* Prefetch next iteration. */
 977           {
 978             vlib_buffer_t * p2, * p3;
 979
 980             p2 = vlib_get_buffer (vm, from[2]);
 981             p3 = vlib_get_buffer (vm, from[3]);
 982
 983             vlib_prefetch_buffer_header (p2, LOAD);
 984             vlib_prefetch_buffer_header (p3, LOAD);
 985
 986             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 987             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 988           }
 989
 990           /* speculatively enqueue b0 and b1 to the current next frame */
 991           to_next[0] = bi0 = from[0];
 992           to_next[1] = bi1 = from[1];
 993           from += 2;
 994           to_next += 2;
 995           n_left_from -= 2;
 996           n_left_to_next -= 2;
 997
 998           b0 = vlib_get_buffer (vm, bi0);
 999           b1 = vlib_get_buffer (vm, bi1);
1000
1001           vnet_buffer (b0)->snat.flags = 0;
1002           vnet_buffer (b1)->snat.flags = 0;
1003
1004           ip0 = vlib_buffer_get_current (b0);
1005           udp0 = ip4_next_header (ip0);
1006           tcp0 = (tcp_header_t *) udp0;
1007           icmp0 = (icmp46_header_t *) udp0;
1008
1009           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1010           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1011                                    sw_if_index0);
1012
1013           if (PREDICT_FALSE(ip0->ttl == 1))
1014             {
1015               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1016               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1017                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1018                                            0);
1019               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1020               goto trace0;
1021             }
1022
1023           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1024
1025           if (PREDICT_FALSE (proto0 == ~0))
1026             {
1027               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1028                                              thread_index, now, vm, node);
1029               if (!s0)
1030                 next0 = SNAT_OUT2IN_NEXT_DROP;
1031               goto trace0;
1032             }
1033
1034           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1035             {
1036               next0 = icmp_out2in_slow_path
1037                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1038                  next0, now, thread_index, &s0);
1039               goto trace0;
1040             }
1041
1042           key0.addr = ip0->dst_address;
1043           key0.port = udp0->dst_port;
1044           key0.protocol = proto0;
1045           key0.fib_index = rx_fib_index0;
1046
1047           kv0.key = key0.as_u64;
1048
1049           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1050                                       &kv0, &value0))
1051             {
1052               /* Try to match static mapping by external address and port,
1053                  destination address and port in packet */
1054               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1055                 {
1056                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1057                   /*
1058                    * Send DHCP packets to the ipv4 stack, or we won't
1059                    * be able to use dhcp client on the outside interface
1060                    */
1061                   if (proto0 != SNAT_PROTOCOL_UDP
1062                       || (udp0->dst_port
1063                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1064                     next0 = SNAT_OUT2IN_NEXT_DROP;
1065                   goto trace0;
1066                 }
1067
1068               /* Create session initiated by host from external network */
1069               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1070                                                      thread_index);
1071               if (!s0)
1072                 {
1073                   next0 = SNAT_OUT2IN_NEXT_DROP;
1074                   goto trace0;
1075                 }
1076             }
1077           else
1078             {
1079               if (PREDICT_FALSE (value0.value == ~0ULL))
1080                 {
1081                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1082                                       now, vm, node);
1083                   if (!s0)
1084                     next0 = SNAT_OUT2IN_NEXT_DROP;
1085                   goto trace0;
1086                 }
1087               else
1088                 {
1089                   s0 = pool_elt_at_index (
1090                     sm->per_thread_data[thread_index].sessions,
1091                     value0.value);
1092                 }
1093             }
1094
1095           old_addr0 = ip0->dst_address.as_u32;
1096           ip0->dst_address = s0->in2out.addr;
1097           new_addr0 = ip0->dst_address.as_u32;
1098           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1099
1100           sum0 = ip0->checksum;
1101           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1102                                  ip4_header_t,
1103                                  dst_address /* changed member */);
1104           ip0->checksum = ip_csum_fold (sum0);
1105
1106           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1107             {
1108               old_port0 = tcp0->dst_port;
1109               tcp0->dst_port = s0->in2out.port;
1110               new_port0 = tcp0->dst_port;
1111
1112               sum0 = tcp0->checksum;
1113               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1114                                      ip4_header_t,
1115                                      dst_address /* changed member */);
1116
1117               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1118                                      ip4_header_t /* cheat */,
1119                                      length /* changed member */);
1120               tcp0->checksum = ip_csum_fold(sum0);
1121             }
1122           else
1123             {
1124               old_port0 = udp0->dst_port;
1125               udp0->dst_port = s0->in2out.port;
1126               udp0->checksum = 0;
1127             }
1128
1129           /* Accounting */
1130           s0->last_heard = now;
1131           s0->total_pkts++;
1132           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1133           /* Per-user LRU list maintenance for dynamic translation */
1134           if (!snat_is_session_static (s0))
1135             {
1136               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1137                                  s0->per_user_index);
1138               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1139                                   s0->per_user_list_head_index,
1140                                   s0->per_user_index);
1141             }
1142         trace0:
1143
1144           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1145                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1146             {
1147               snat_out2in_trace_t *t =
1148                  vlib_add_trace (vm, node, b0, sizeof (*t));
1149               t->sw_if_index = sw_if_index0;
1150               t->next_index = next0;
1151               t->session_index = ~0;
1152               if (s0)
1153                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1154             }
1155
1156           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1157
1158
1159           ip1 = vlib_buffer_get_current (b1);
1160           udp1 = ip4_next_header (ip1);
1161           tcp1 = (tcp_header_t *) udp1;
1162           icmp1 = (icmp46_header_t *) udp1;
1163
1164           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1165           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1166                                    sw_if_index1);
1167
1168           if (PREDICT_FALSE(ip1->ttl == 1))
1169             {
1170               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1171               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1172                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1173                                            0);
1174               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1175               goto trace1;
1176             }
1177
1178           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1179
1180           if (PREDICT_FALSE (proto1 == ~0))
1181             {
1182               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1183                                              thread_index, now, vm, node);
1184               if (!s1)
1185                 next1 = SNAT_OUT2IN_NEXT_DROP;
1186               goto trace1;
1187             }
1188
1189           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1190             {
1191               next1 = icmp_out2in_slow_path
1192                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1193                  next1, now, thread_index, &s1);
1194               goto trace1;
1195             }
1196
1197           key1.addr = ip1->dst_address;
1198           key1.port = udp1->dst_port;
1199           key1.protocol = proto1;
1200           key1.fib_index = rx_fib_index1;
1201
1202           kv1.key = key1.as_u64;
1203
1204           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1205                                       &kv1, &value1))
1206             {
1207               /* Try to match static mapping by external address and port,
1208                  destination address and port in packet */
1209               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1210                 {
1211                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1212                   /*
1213                    * Send DHCP packets to the ipv4 stack, or we won't
1214                    * be able to use dhcp client on the outside interface
1215                    */
1216                   if (proto1 != SNAT_PROTOCOL_UDP
1217                       || (udp1->dst_port
1218                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1219                     next1 = SNAT_OUT2IN_NEXT_DROP;
1220                   goto trace1;
1221                 }
1222
1223               /* Create session initiated by host from external network */
1224               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1225                                                      thread_index);
1226               if (!s1)
1227                 {
1228                   next1 = SNAT_OUT2IN_NEXT_DROP;
1229                   goto trace1;
1230                 }
1231             }
1232           else
1233             {
1234               if (PREDICT_FALSE (value1.value == ~0ULL))
1235                 {
1236                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1237                                       now, vm, node);
1238                   if (!s1)
1239                     next1 = SNAT_OUT2IN_NEXT_DROP;
1240                   goto trace1;
1241                 }
1242               else
1243                 {
1244                   s1 = pool_elt_at_index (
1245                     sm->per_thread_data[thread_index].sessions,
1246                     value1.value);
1247                 }
1248             }
1249
1250           old_addr1 = ip1->dst_address.as_u32;
1251           ip1->dst_address = s1->in2out.addr;
1252           new_addr1 = ip1->dst_address.as_u32;
1253           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1254
1255           sum1 = ip1->checksum;
1256           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1257                                  ip4_header_t,
1258                                  dst_address /* changed member */);
1259           ip1->checksum = ip_csum_fold (sum1);
1260
1261           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1262             {
1263               old_port1 = tcp1->dst_port;
1264               tcp1->dst_port = s1->in2out.port;
1265               new_port1 = tcp1->dst_port;
1266
1267               sum1 = tcp1->checksum;
1268               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1269                                      ip4_header_t,
1270                                      dst_address /* changed member */);
1271
1272               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1273                                      ip4_header_t /* cheat */,
1274                                      length /* changed member */);
1275               tcp1->checksum = ip_csum_fold(sum1);
1276             }
1277           else
1278             {
1279               old_port1 = udp1->dst_port;
1280               udp1->dst_port = s1->in2out.port;
1281               udp1->checksum = 0;
1282             }
1283
1284           /* Accounting */
1285           s1->last_heard = now;
1286           s1->total_pkts++;
1287           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1288           /* Per-user LRU list maintenance for dynamic translation */
1289           if (!snat_is_session_static (s1))
1290             {
1291               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1292                                  s1->per_user_index);
1293               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1294                                   s1->per_user_list_head_index,
1295                                   s1->per_user_index);
1296             }
1297         trace1:
1298
1299           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1300                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1301             {
1302               snat_out2in_trace_t *t =
1303                  vlib_add_trace (vm, node, b1, sizeof (*t));
1304               t->sw_if_index = sw_if_index1;
1305               t->next_index = next1;
1306               t->session_index = ~0;
1307               if (s1)
1308                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1309             }
1310
1311           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1312
1313           /* verify speculative enqueues, maybe switch current next frame */
1314           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1315                                            to_next, n_left_to_next,
1316                                            bi0, bi1, next0, next1);
1317         }
1318
1319       while (n_left_from > 0 && n_left_to_next > 0)
1320         {
1321           u32 bi0;
1322           vlib_buffer_t * b0;
1323           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1324           u32 sw_if_index0;
1325           ip4_header_t * ip0;
1326           ip_csum_t sum0;
1327           u32 new_addr0, old_addr0;
1328           u16 new_port0, old_port0;
1329           udp_header_t * udp0;
1330           tcp_header_t * tcp0;
1331           icmp46_header_t * icmp0;
1332           snat_session_key_t key0, sm0;
1333           u32 rx_fib_index0;
1334           u32 proto0;
1335           snat_session_t * s0 = 0;
1336           clib_bihash_kv_8_8_t kv0, value0;
1337
1338           /* speculatively enqueue b0 to the current next frame */
1339           bi0 = from[0];
1340           to_next[0] = bi0;
1341           from += 1;
1342           to_next += 1;
1343           n_left_from -= 1;
1344           n_left_to_next -= 1;
1345
1346           b0 = vlib_get_buffer (vm, bi0);
1347
1348           vnet_buffer (b0)->snat.flags = 0;
1349
1350           ip0 = vlib_buffer_get_current (b0);
1351           udp0 = ip4_next_header (ip0);
1352           tcp0 = (tcp_header_t *) udp0;
1353           icmp0 = (icmp46_header_t *) udp0;
1354
1355           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1356           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1357                                    sw_if_index0);
1358
1359           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1360
1361           if (PREDICT_FALSE (proto0 == ~0))
1362             {
1363               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1364                                              thread_index, now, vm, node);
1365               if (!s0)
1366                 next0 = SNAT_OUT2IN_NEXT_DROP;
1367               goto trace00;
1368             }
1369
1370           if (PREDICT_FALSE(ip0->ttl == 1))
1371             {
1372               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1373               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1374                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1375                                            0);
1376               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1377               goto trace00;
1378             }
1379
1380           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1381             {
1382               next0 = icmp_out2in_slow_path
1383                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1384                  next0, now, thread_index, &s0);
1385               goto trace00;
1386             }
1387
1388           key0.addr = ip0->dst_address;
1389           key0.port = udp0->dst_port;
1390           key0.protocol = proto0;
1391           key0.fib_index = rx_fib_index0;
1392
1393           kv0.key = key0.as_u64;
1394
1395           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1396                                       &kv0, &value0))
1397             {
1398               /* Try to match static mapping by external address and port,
1399                  destination address and port in packet */
1400               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1401                 {
1402                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1403                   /*
1404                    * Send DHCP packets to the ipv4 stack, or we won't
1405                    * be able to use dhcp client on the outside interface
1406                    */
1407                   if (proto0 != SNAT_PROTOCOL_UDP
1408                       || (udp0->dst_port
1409                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1410
1411                     next0 = SNAT_OUT2IN_NEXT_DROP;
1412                   goto trace00;
1413                 }
1414
1415               /* Create session initiated by host from external network */
1416               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1417                                                      thread_index);
1418               if (!s0)
1419                 {
1420                   next0 = SNAT_OUT2IN_NEXT_DROP;
1421                   goto trace00;
1422                 }
1423             }
1424           else
1425             {
1426               if (PREDICT_FALSE (value0.value == ~0ULL))
1427                 {
1428                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1429                                       now, vm, node);
1430                   if (!s0)
1431                     next0 = SNAT_OUT2IN_NEXT_DROP;
1432                   goto trace00;
1433                 }
1434               else
1435                 {
1436                   s0 = pool_elt_at_index (
1437                     sm->per_thread_data[thread_index].sessions,
1438                     value0.value);
1439                 }
1440             }
1441
1442           old_addr0 = ip0->dst_address.as_u32;
1443           ip0->dst_address = s0->in2out.addr;
1444           new_addr0 = ip0->dst_address.as_u32;
1445           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1446
1447           sum0 = ip0->checksum;
1448           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1449                                  ip4_header_t,
1450                                  dst_address /* changed member */);
1451           ip0->checksum = ip_csum_fold (sum0);
1452
1453           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1454             {
1455               old_port0 = tcp0->dst_port;
1456               tcp0->dst_port = s0->in2out.port;
1457               new_port0 = tcp0->dst_port;
1458
1459               sum0 = tcp0->checksum;
1460               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1461                                      ip4_header_t,
1462                                      dst_address /* changed member */);
1463
1464               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1465                                      ip4_header_t /* cheat */,
1466                                      length /* changed member */);
1467               tcp0->checksum = ip_csum_fold(sum0);
1468             }
1469           else
1470             {
1471               old_port0 = udp0->dst_port;
1472               udp0->dst_port = s0->in2out.port;
1473               udp0->checksum = 0;
1474             }
1475
1476           /* Accounting */
1477           s0->last_heard = now;
1478           s0->total_pkts++;
1479           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1480           /* Per-user LRU list maintenance for dynamic translation */
1481           if (!snat_is_session_static (s0))
1482             {
1483               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1484                                  s0->per_user_index);
1485               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1486                                   s0->per_user_list_head_index,
1487                                   s0->per_user_index);
1488             }
1489         trace00:
1490
1491           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1492                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1493             {
1494               snat_out2in_trace_t *t =
1495                  vlib_add_trace (vm, node, b0, sizeof (*t));
1496               t->sw_if_index = sw_if_index0;
1497               t->next_index = next0;
1498               t->session_index = ~0;
1499               if (s0)
1500                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1501             }
1502
1503           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1504
1505           /* verify speculative enqueue, maybe switch current next frame */
1506           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1507                                            to_next, n_left_to_next,
1508                                            bi0, next0);
1509         }
1510
1511       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1512     }
1513
1514   vlib_node_increment_counter (vm, snat_out2in_node.index,
1515                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1516                                pkts_processed);
1517   return frame->n_vectors;
1518 }
1519
1520 VLIB_REGISTER_NODE (snat_out2in_node) = {
1521   .function = snat_out2in_node_fn,
1522   .name = "nat44-out2in",
1523   .vector_size = sizeof (u32),
1524   .format_trace = format_snat_out2in_trace,
1525   .type = VLIB_NODE_TYPE_INTERNAL,
1526
1527   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1528   .error_strings = snat_out2in_error_strings,
1529
1530   .runtime_data_bytes = sizeof (snat_runtime_t),
1531
1532   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1533
1534   /* edit / add dispositions here */
1535   .next_nodes = {
1536     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1537     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1538     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1539   },
1540 };
1541 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1542
1543 /**************************/
1544 /*** deterministic mode ***/
1545 /**************************/
1546 static uword
1547 snat_det_out2in_node_fn (vlib_main_t * vm,
1548                          vlib_node_runtime_t * node,
1549                          vlib_frame_t * frame)
1550 {
1551   u32 n_left_from, * from, * to_next;
1552   snat_out2in_next_t next_index;
1553   u32 pkts_processed = 0;
1554   snat_main_t * sm = &snat_main;
1555   u32 thread_index = vlib_get_thread_index ();
1556
1557   from = vlib_frame_vector_args (frame);
1558   n_left_from = frame->n_vectors;
1559   next_index = node->cached_next_index;
1560
1561   while (n_left_from > 0)
1562     {
1563       u32 n_left_to_next;
1564
1565       vlib_get_next_frame (vm, node, next_index,
1566                            to_next, n_left_to_next);
1567
1568       while (n_left_from >= 4 && n_left_to_next >= 2)
1569         {
1570           u32 bi0, bi1;
1571           vlib_buffer_t * b0, * b1;
1572           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1573           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1574           u32 sw_if_index0, sw_if_index1;
1575           ip4_header_t * ip0, * ip1;
1576           ip_csum_t sum0, sum1;
1577           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1578           u16 new_port0, old_port0, old_port1, new_port1;
1579           udp_header_t * udp0, * udp1;
1580           tcp_header_t * tcp0, * tcp1;
1581           u32 proto0, proto1;
1582           snat_det_out_key_t key0, key1;
1583           snat_det_map_t * dm0, * dm1;
1584           snat_det_session_t * ses0 = 0, * ses1 = 0;
1585           u32 rx_fib_index0, rx_fib_index1;
1586           icmp46_header_t * icmp0, * icmp1;
1587
1588           /* Prefetch next iteration. */
1589           {
1590             vlib_buffer_t * p2, * p3;
1591
1592             p2 = vlib_get_buffer (vm, from[2]);
1593             p3 = vlib_get_buffer (vm, from[3]);
1594
1595             vlib_prefetch_buffer_header (p2, LOAD);
1596             vlib_prefetch_buffer_header (p3, LOAD);
1597
1598             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1599             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1600           }
1601
1602           /* speculatively enqueue b0 and b1 to the current next frame */
1603           to_next[0] = bi0 = from[0];
1604           to_next[1] = bi1 = from[1];
1605           from += 2;
1606           to_next += 2;
1607           n_left_from -= 2;
1608           n_left_to_next -= 2;
1609
1610           b0 = vlib_get_buffer (vm, bi0);
1611           b1 = vlib_get_buffer (vm, bi1);
1612
1613           ip0 = vlib_buffer_get_current (b0);
1614           udp0 = ip4_next_header (ip0);
1615           tcp0 = (tcp_header_t *) udp0;
1616
1617           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1618
1619           if (PREDICT_FALSE(ip0->ttl == 1))
1620             {
1621               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1622               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1623                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1624                                            0);
1625               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1626               goto trace0;
1627             }
1628
1629           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1630
1631           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1632             {
1633               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1634               icmp0 = (icmp46_header_t *) udp0;
1635
1636               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1637                                   rx_fib_index0, node, next0, thread_index,
1638                                   &ses0, &dm0);
1639               goto trace0;
1640             }
1641
1642           key0.ext_host_addr = ip0->src_address;
1643           key0.ext_host_port = tcp0->src;
1644           key0.out_port = tcp0->dst;
1645
1646           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1647           if (PREDICT_FALSE(!dm0))
1648             {
1649               clib_warning("unknown dst address:  %U",
1650                            format_ip4_address, &ip0->dst_address);
1651               next0 = SNAT_OUT2IN_NEXT_DROP;
1652               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1653               goto trace0;
1654             }
1655
1656           snat_det_reverse(dm0, &ip0->dst_address,
1657                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1658
1659           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1660           if (PREDICT_FALSE(!ses0))
1661             {
1662               clib_warning("no match src %U:%d dst %U:%d for user %U",
1663                            format_ip4_address, &ip0->src_address,
1664                            clib_net_to_host_u16 (tcp0->src),
1665                            format_ip4_address, &ip0->dst_address,
1666                            clib_net_to_host_u16 (tcp0->dst),
1667                            format_ip4_address, &new_addr0);
1668               next0 = SNAT_OUT2IN_NEXT_DROP;
1669               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1670               goto trace0;
1671             }
1672           new_port0 = ses0->in_port;
1673
1674           old_addr0 = ip0->dst_address;
1675           ip0->dst_address = new_addr0;
1676           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1677
1678           sum0 = ip0->checksum;
1679           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1680                                  ip4_header_t,
1681                                  dst_address /* changed member */);
1682           ip0->checksum = ip_csum_fold (sum0);
1683
1684           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1685             {
1686               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1687                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1688               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1689                 snat_det_ses_close(dm0, ses0);
1690
1691               old_port0 = tcp0->dst;
1692               tcp0->dst = new_port0;
1693
1694               sum0 = tcp0->checksum;
1695               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1696                                      ip4_header_t,
1697                                      dst_address /* changed member */);
1698
1699               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1700                                      ip4_header_t /* cheat */,
1701                                      length /* changed member */);
1702               tcp0->checksum = ip_csum_fold(sum0);
1703             }
1704           else
1705             {
1706               old_port0 = udp0->dst_port;
1707               udp0->dst_port = new_port0;
1708               udp0->checksum = 0;
1709             }
1710
1711         trace0:
1712
1713           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1714                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1715             {
1716               snat_out2in_trace_t *t =
1717                  vlib_add_trace (vm, node, b0, sizeof (*t));
1718               t->sw_if_index = sw_if_index0;
1719               t->next_index = next0;
1720               t->session_index = ~0;
1721               if (ses0)
1722                 t->session_index = ses0 - dm0->sessions;
1723             }
1724
1725           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1726
1727           b1 = vlib_get_buffer (vm, bi1);
1728
1729           ip1 = vlib_buffer_get_current (b1);
1730           udp1 = ip4_next_header (ip1);
1731           tcp1 = (tcp_header_t *) udp1;
1732
1733           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1734
1735           if (PREDICT_FALSE(ip1->ttl == 1))
1736             {
1737               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1738               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1739                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1740                                            0);
1741               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1742               goto trace1;
1743             }
1744
1745           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1746
1747           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1748             {
1749               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1750               icmp1 = (icmp46_header_t *) udp1;
1751
1752               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1753                                   rx_fib_index1, node, next1, thread_index,
1754                                   &ses1, &dm1);
1755               goto trace1;
1756             }
1757
1758           key1.ext_host_addr = ip1->src_address;
1759           key1.ext_host_port = tcp1->src;
1760           key1.out_port = tcp1->dst;
1761
1762           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1763           if (PREDICT_FALSE(!dm1))
1764             {
1765               clib_warning("unknown dst address:  %U",
1766                            format_ip4_address, &ip1->dst_address);
1767               next1 = SNAT_OUT2IN_NEXT_DROP;
1768               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1769               goto trace1;
1770             }
1771
1772           snat_det_reverse(dm1, &ip1->dst_address,
1773                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1774
1775           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1776           if (PREDICT_FALSE(!ses1))
1777             {
1778               clib_warning("no match src %U:%d dst %U:%d for user %U",
1779                            format_ip4_address, &ip1->src_address,
1780                            clib_net_to_host_u16 (tcp1->src),
1781                            format_ip4_address, &ip1->dst_address,
1782                            clib_net_to_host_u16 (tcp1->dst),
1783                            format_ip4_address, &new_addr1);
1784               next1 = SNAT_OUT2IN_NEXT_DROP;
1785               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1786               goto trace1;
1787             }
1788           new_port1 = ses1->in_port;
1789
1790           old_addr1 = ip1->dst_address;
1791           ip1->dst_address = new_addr1;
1792           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1793
1794           sum1 = ip1->checksum;
1795           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1796                                  ip4_header_t,
1797                                  dst_address /* changed member */);
1798           ip1->checksum = ip_csum_fold (sum1);
1799
1800           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1801             {
1802               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1803                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1804               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1805                 snat_det_ses_close(dm1, ses1);
1806
1807               old_port1 = tcp1->dst;
1808               tcp1->dst = new_port1;
1809
1810               sum1 = tcp1->checksum;
1811               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1812                                      ip4_header_t,
1813                                      dst_address /* changed member */);
1814
1815               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1816                                      ip4_header_t /* cheat */,
1817                                      length /* changed member */);
1818               tcp1->checksum = ip_csum_fold(sum1);
1819             }
1820           else
1821             {
1822               old_port1 = udp1->dst_port;
1823               udp1->dst_port = new_port1;
1824               udp1->checksum = 0;
1825             }
1826
1827         trace1:
1828
1829           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1830                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1831             {
1832               snat_out2in_trace_t *t =
1833                  vlib_add_trace (vm, node, b1, sizeof (*t));
1834               t->sw_if_index = sw_if_index1;
1835               t->next_index = next1;
1836               t->session_index = ~0;
1837               if (ses1)
1838                 t->session_index = ses1 - dm1->sessions;
1839             }
1840
1841           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1842
1843           /* verify speculative enqueues, maybe switch current next frame */
1844           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1845                                            to_next, n_left_to_next,
1846                                            bi0, bi1, next0, next1);
1847          }
1848
1849       while (n_left_from > 0 && n_left_to_next > 0)
1850         {
1851           u32 bi0;
1852           vlib_buffer_t * b0;
1853           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1854           u32 sw_if_index0;
1855           ip4_header_t * ip0;
1856           ip_csum_t sum0;
1857           ip4_address_t new_addr0, old_addr0;
1858           u16 new_port0, old_port0;
1859           udp_header_t * udp0;
1860           tcp_header_t * tcp0;
1861           u32 proto0;
1862           snat_det_out_key_t key0;
1863           snat_det_map_t * dm0;
1864           snat_det_session_t * ses0 = 0;
1865           u32 rx_fib_index0;
1866           icmp46_header_t * icmp0;
1867
1868           /* speculatively enqueue b0 to the current next frame */
1869           bi0 = from[0];
1870           to_next[0] = bi0;
1871           from += 1;
1872           to_next += 1;
1873           n_left_from -= 1;
1874           n_left_to_next -= 1;
1875
1876           b0 = vlib_get_buffer (vm, bi0);
1877
1878           ip0 = vlib_buffer_get_current (b0);
1879           udp0 = ip4_next_header (ip0);
1880           tcp0 = (tcp_header_t *) udp0;
1881
1882           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1883
1884           if (PREDICT_FALSE(ip0->ttl == 1))
1885             {
1886               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1887               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1888                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1889                                            0);
1890               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1891               goto trace00;
1892             }
1893
1894           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1895
1896           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1897             {
1898               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1899               icmp0 = (icmp46_header_t *) udp0;
1900
1901               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1902                                   rx_fib_index0, node, next0, thread_index,
1903                                   &ses0, &dm0);
1904               goto trace00;
1905             }
1906
1907           key0.ext_host_addr = ip0->src_address;
1908           key0.ext_host_port = tcp0->src;
1909           key0.out_port = tcp0->dst;
1910
1911           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1912           if (PREDICT_FALSE(!dm0))
1913             {
1914               clib_warning("unknown dst address:  %U",
1915                            format_ip4_address, &ip0->dst_address);
1916               next0 = SNAT_OUT2IN_NEXT_DROP;
1917               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1918               goto trace00;
1919             }
1920
1921           snat_det_reverse(dm0, &ip0->dst_address,
1922                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1923
1924           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1925           if (PREDICT_FALSE(!ses0))
1926             {
1927               clib_warning("no match src %U:%d dst %U:%d for user %U",
1928                            format_ip4_address, &ip0->src_address,
1929                            clib_net_to_host_u16 (tcp0->src),
1930                            format_ip4_address, &ip0->dst_address,
1931                            clib_net_to_host_u16 (tcp0->dst),
1932                            format_ip4_address, &new_addr0);
1933               next0 = SNAT_OUT2IN_NEXT_DROP;
1934               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1935               goto trace00;
1936             }
1937           new_port0 = ses0->in_port;
1938
1939           old_addr0 = ip0->dst_address;
1940           ip0->dst_address = new_addr0;
1941           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1942
1943           sum0 = ip0->checksum;
1944           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1945                                  ip4_header_t,
1946                                  dst_address /* changed member */);
1947           ip0->checksum = ip_csum_fold (sum0);
1948
1949           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1950             {
1951               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1952                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1953               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1954                 snat_det_ses_close(dm0, ses0);
1955
1956               old_port0 = tcp0->dst;
1957               tcp0->dst = new_port0;
1958
1959               sum0 = tcp0->checksum;
1960               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1961                                      ip4_header_t,
1962                                      dst_address /* changed member */);
1963
1964               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1965                                      ip4_header_t /* cheat */,
1966                                      length /* changed member */);
1967               tcp0->checksum = ip_csum_fold(sum0);
1968             }
1969           else
1970             {
1971               old_port0 = udp0->dst_port;
1972               udp0->dst_port = new_port0;
1973               udp0->checksum = 0;
1974             }
1975
1976         trace00:
1977
1978           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1979                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1980             {
1981               snat_out2in_trace_t *t =
1982                  vlib_add_trace (vm, node, b0, sizeof (*t));
1983               t->sw_if_index = sw_if_index0;
1984               t->next_index = next0;
1985               t->session_index = ~0;
1986               if (ses0)
1987                 t->session_index = ses0 - dm0->sessions;
1988             }
1989
1990           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1991
1992           /* verify speculative enqueue, maybe switch current next frame */
1993           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1994                                            to_next, n_left_to_next,
1995                                            bi0, next0);
1996         }
1997
1998       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1999     }
2000
2001   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2002                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2003                                pkts_processed);
2004   return frame->n_vectors;
2005 }
2006
2007 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2008   .function = snat_det_out2in_node_fn,
2009   .name = "nat44-det-out2in",
2010   .vector_size = sizeof (u32),
2011   .format_trace = format_snat_out2in_trace,
2012   .type = VLIB_NODE_TYPE_INTERNAL,
2013
2014   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2015   .error_strings = snat_out2in_error_strings,
2016
2017   .runtime_data_bytes = sizeof (snat_runtime_t),
2018
2019   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2020
2021   /* edit / add dispositions here */
2022   .next_nodes = {
2023     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2024     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2025     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2026   },
2027 };
2028 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2029
2030 /**
2031  * Get address and port values to be used for ICMP packet translation
2032  * and create session if needed
2033  *
2034  * @param[in,out] sm             NAT main
2035  * @param[in,out] node           NAT node runtime
2036  * @param[in] thread_index       thread index
2037  * @param[in,out] b0             buffer containing packet to be translated
2038  * @param[out] p_proto           protocol used for matching
2039  * @param[out] p_value           address and port after NAT translation
2040  * @param[out] p_dont_translate  if packet should not be translated
2041  * @param d                      optional parameter
2042  * @param e                      optional parameter
2043  */
2044 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2045                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2046                           snat_session_key_t *p_value,
2047                           u8 *p_dont_translate, void *d, void *e)
2048 {
2049   ip4_header_t *ip0;
2050   icmp46_header_t *icmp0;
2051   u32 sw_if_index0;
2052   u8 protocol;
2053   snat_det_out_key_t key0;
2054   u8 dont_translate = 0;
2055   u32 next0 = ~0;
2056   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2057   ip4_header_t *inner_ip0;
2058   void *l4_header = 0;
2059   icmp46_header_t *inner_icmp0;
2060   snat_det_map_t * dm0 = 0;
2061   ip4_address_t new_addr0 = {{0}};
2062   snat_det_session_t * ses0 = 0;
2063   ip4_address_t out_addr;
2064
2065   ip0 = vlib_buffer_get_current (b0);
2066   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2067   echo0 = (icmp_echo_header_t *)(icmp0+1);
2068   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2069
2070   if (!icmp_is_error_message (icmp0))
2071     {
2072       protocol = SNAT_PROTOCOL_ICMP;
2073       key0.ext_host_addr = ip0->src_address;
2074       key0.ext_host_port = 0;
2075       key0.out_port = echo0->identifier;
2076       out_addr = ip0->dst_address;
2077     }
2078   else
2079     {
2080       inner_ip0 = (ip4_header_t *)(echo0+1);
2081       l4_header = ip4_next_header (inner_ip0);
2082       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2083       key0.ext_host_addr = inner_ip0->dst_address;
2084       out_addr = inner_ip0->src_address;
2085       switch (protocol)
2086         {
2087         case SNAT_PROTOCOL_ICMP:
2088           inner_icmp0 = (icmp46_header_t*)l4_header;
2089           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2090           key0.ext_host_port = 0;
2091           key0.out_port = inner_echo0->identifier;
2092           break;
2093         case SNAT_PROTOCOL_UDP:
2094         case SNAT_PROTOCOL_TCP:
2095           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2096           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2097           break;
2098         default:
2099           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2100           next0 = SNAT_OUT2IN_NEXT_DROP;
2101           goto out;
2102         }
2103     }
2104
2105   dm0 = snat_det_map_by_out(sm, &out_addr);
2106   if (PREDICT_FALSE(!dm0))
2107     {
2108       /* Don't NAT packet aimed at the intfc address */
2109       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2110                                           ip0->dst_address.as_u32)))
2111         {
2112           dont_translate = 1;
2113           goto out;
2114         }
2115       clib_warning("unknown dst address:  %U",
2116                    format_ip4_address, &ip0->dst_address);
2117       goto out;
2118     }
2119
2120   snat_det_reverse(dm0, &ip0->dst_address,
2121                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2122
2123   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2124   if (PREDICT_FALSE(!ses0))
2125     {
2126       /* Don't NAT packet aimed at the intfc address */
2127       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2128                                           ip0->dst_address.as_u32)))
2129         {
2130           dont_translate = 1;
2131           goto out;
2132         }
2133       clib_warning("no match src %U:%d dst %U:%d for user %U",
2134                    format_ip4_address, &key0.ext_host_addr,
2135                    clib_net_to_host_u16 (key0.ext_host_port),
2136                    format_ip4_address, &out_addr,
2137                    clib_net_to_host_u16 (key0.out_port),
2138                    format_ip4_address, &new_addr0);
2139       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2140       next0 = SNAT_OUT2IN_NEXT_DROP;
2141       goto out;
2142     }
2143
2144   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2145                     !icmp_is_error_message (icmp0)))
2146     {
2147       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2148       next0 = SNAT_OUT2IN_NEXT_DROP;
2149       goto out;
2150     }
2151
2152   goto out;
2153
2154 out:
2155   *p_proto = protocol;
2156   if (ses0)
2157     {
2158       p_value->addr = new_addr0;
2159       p_value->fib_index = sm->inside_fib_index;
2160       p_value->port = ses0->in_port;
2161     }
2162   *p_dont_translate = dont_translate;
2163   if (d)
2164     *(snat_det_session_t**)d = ses0;
2165   if (e)
2166     *(snat_det_map_t**)e = dm0;
2167   return next0;
2168 }
2169
2170 /**********************/
2171 /*** worker handoff ***/
2172 /**********************/
2173 static uword
2174 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2175                                vlib_node_runtime_t * node,
2176                                vlib_frame_t * frame)
2177 {
2178   snat_main_t *sm = &snat_main;
2179   vlib_thread_main_t *tm = vlib_get_thread_main ();
2180   u32 n_left_from, *from, *to_next = 0;
2181   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2182   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2183     = 0;
2184   vlib_frame_queue_elt_t *hf = 0;
2185   vlib_frame_t *f = 0;
2186   int i;
2187   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2188   u32 next_worker_index = 0;
2189   u32 current_worker_index = ~0;
2190   u32 thread_index = vlib_get_thread_index ();
2191
2192   ASSERT (vec_len (sm->workers));
2193
2194   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2195     {
2196       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2197
2198       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2199                                sm->first_worker_index + sm->num_workers - 1,
2200                                (vlib_frame_queue_t *) (~0));
2201     }
2202
2203   from = vlib_frame_vector_args (frame);
2204   n_left_from = frame->n_vectors;
2205
2206   while (n_left_from > 0)
2207     {
2208       u32 bi0;
2209       vlib_buffer_t *b0;
2210       u32 sw_if_index0;
2211       u32 rx_fib_index0;
2212       ip4_header_t * ip0;
2213       u8 do_handoff;
2214
2215       bi0 = from[0];
2216       from += 1;
2217       n_left_from -= 1;
2218
2219       b0 = vlib_get_buffer (vm, bi0);
2220
2221       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2222       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2223
2224       ip0 = vlib_buffer_get_current (b0);
2225
2226       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2227
2228       if (PREDICT_FALSE (next_worker_index != thread_index))
2229         {
2230           do_handoff = 1;
2231
2232           if (next_worker_index != current_worker_index)
2233             {
2234               if (hf)
2235                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2236
2237               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2238                                                       next_worker_index,
2239                                                       handoff_queue_elt_by_worker_index);
2240
2241               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2242               to_next_worker = &hf->buffer_index[hf->n_vectors];
2243               current_worker_index = next_worker_index;
2244             }
2245
2246           /* enqueue to correct worker thread */
2247           to_next_worker[0] = bi0;
2248           to_next_worker++;
2249           n_left_to_next_worker--;
2250
2251           if (n_left_to_next_worker == 0)
2252             {
2253               hf->n_vectors = VLIB_FRAME_SIZE;
2254               vlib_put_frame_queue_elt (hf);
2255               current_worker_index = ~0;
2256               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2257               hf = 0;
2258             }
2259         }
2260       else
2261         {
2262           do_handoff = 0;
2263           /* if this is 1st frame */
2264           if (!f)
2265             {
2266               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2267               to_next = vlib_frame_vector_args (f);
2268             }
2269
2270           to_next[0] = bi0;
2271           to_next += 1;
2272           f->n_vectors++;
2273         }
2274
2275       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2276                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2277         {
2278           snat_out2in_worker_handoff_trace_t *t =
2279             vlib_add_trace (vm, node, b0, sizeof (*t));
2280           t->next_worker_index = next_worker_index;
2281           t->do_handoff = do_handoff;
2282         }
2283     }
2284
2285   if (f)
2286     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2287
2288   if (hf)
2289     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2290
2291   /* Ship frames to the worker nodes */
2292   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2293     {
2294       if (handoff_queue_elt_by_worker_index[i])
2295         {
2296           hf = handoff_queue_elt_by_worker_index[i];
2297           /*
2298            * It works better to let the handoff node
2299            * rate-adapt, always ship the handoff queue element.
2300            */
2301           if (1 || hf->n_vectors == hf->last_n_vectors)
2302             {
2303               vlib_put_frame_queue_elt (hf);
2304               handoff_queue_elt_by_worker_index[i] = 0;
2305             }
2306           else
2307             hf->last_n_vectors = hf->n_vectors;
2308         }
2309       congested_handoff_queue_by_worker_index[i] =
2310         (vlib_frame_queue_t *) (~0);
2311     }
2312   hf = 0;
2313   current_worker_index = ~0;
2314   return frame->n_vectors;
2315 }
2316
2317 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2318   .function = snat_out2in_worker_handoff_fn,
2319   .name = "nat44-out2in-worker-handoff",
2320   .vector_size = sizeof (u32),
2321   .format_trace = format_snat_out2in_worker_handoff_trace,
2322   .type = VLIB_NODE_TYPE_INTERNAL,
2323
2324   .n_next_nodes = 1,
2325
2326   .next_nodes = {
2327     [0] = "error-drop",
2328   },
2329 };
2330
2331 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2332
2333 static uword
2334 snat_out2in_fast_node_fn (vlib_main_t * vm,
2335                           vlib_node_runtime_t * node,
2336                           vlib_frame_t * frame)
2337 {
2338   u32 n_left_from, * from, * to_next;
2339   snat_out2in_next_t next_index;
2340   u32 pkts_processed = 0;
2341   snat_main_t * sm = &snat_main;
2342
2343   from = vlib_frame_vector_args (frame);
2344   n_left_from = frame->n_vectors;
2345   next_index = node->cached_next_index;
2346
2347   while (n_left_from > 0)
2348     {
2349       u32 n_left_to_next;
2350
2351       vlib_get_next_frame (vm, node, next_index,
2352                            to_next, n_left_to_next);
2353
2354       while (n_left_from > 0 && n_left_to_next > 0)
2355         {
2356           u32 bi0;
2357           vlib_buffer_t * b0;
2358           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2359           u32 sw_if_index0;
2360           ip4_header_t * ip0;
2361           ip_csum_t sum0;
2362           u32 new_addr0, old_addr0;
2363           u16 new_port0, old_port0;
2364           udp_header_t * udp0;
2365           tcp_header_t * tcp0;
2366           icmp46_header_t * icmp0;
2367           snat_session_key_t key0, sm0;
2368           u32 proto0;
2369           u32 rx_fib_index0;
2370
2371           /* speculatively enqueue b0 to the current next frame */
2372           bi0 = from[0];
2373           to_next[0] = bi0;
2374           from += 1;
2375           to_next += 1;
2376           n_left_from -= 1;
2377           n_left_to_next -= 1;
2378
2379           b0 = vlib_get_buffer (vm, bi0);
2380
2381           ip0 = vlib_buffer_get_current (b0);
2382           udp0 = ip4_next_header (ip0);
2383           tcp0 = (tcp_header_t *) udp0;
2384           icmp0 = (icmp46_header_t *) udp0;
2385
2386           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2387           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2388
2389           vnet_feature_next (sw_if_index0, &next0, b0);
2390
2391           if (PREDICT_FALSE(ip0->ttl == 1))
2392             {
2393               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2394               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2395                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2396                                            0);
2397               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2398               goto trace00;
2399             }
2400
2401           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2402
2403           if (PREDICT_FALSE (proto0 == ~0))
2404               goto trace00;
2405
2406           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2407             {
2408               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2409                                   rx_fib_index0, node, next0, ~0, 0, 0);
2410               goto trace00;
2411             }
2412
2413           key0.addr = ip0->dst_address;
2414           key0.port = udp0->dst_port;
2415           key0.fib_index = rx_fib_index0;
2416
2417           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2418             {
2419               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2420               goto trace00;
2421             }
2422
2423           new_addr0 = sm0.addr.as_u32;
2424           new_port0 = sm0.port;
2425           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2426           old_addr0 = ip0->dst_address.as_u32;
2427           ip0->dst_address.as_u32 = new_addr0;
2428
2429           sum0 = ip0->checksum;
2430           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2431                                  ip4_header_t,
2432                                  dst_address /* changed member */);
2433           ip0->checksum = ip_csum_fold (sum0);
2434
2435           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2436             {
2437                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2438                 {
2439                   old_port0 = tcp0->dst_port;
2440                   tcp0->dst_port = new_port0;
2441
2442                   sum0 = tcp0->checksum;
2443                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2444                                          ip4_header_t,
2445                                          dst_address /* changed member */);
2446
2447                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2448                                          ip4_header_t /* cheat */,
2449                                          length /* changed member */);
2450                   tcp0->checksum = ip_csum_fold(sum0);
2451                 }
2452               else
2453                 {
2454                   old_port0 = udp0->dst_port;
2455                   udp0->dst_port = new_port0;
2456                   udp0->checksum = 0;
2457                 }
2458             }
2459           else
2460             {
2461               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2462                 {
2463                   sum0 = tcp0->checksum;
2464                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2465                                          ip4_header_t,
2466                                          dst_address /* changed member */);
2467
2468                   tcp0->checksum = ip_csum_fold(sum0);
2469                 }
2470             }
2471
2472         trace00:
2473
2474           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2475                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2476             {
2477               snat_out2in_trace_t *t =
2478                  vlib_add_trace (vm, node, b0, sizeof (*t));
2479               t->sw_if_index = sw_if_index0;
2480               t->next_index = next0;
2481             }
2482
2483           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2484
2485           /* verify speculative enqueue, maybe switch current next frame */
2486           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2487                                            to_next, n_left_to_next,
2488                                            bi0, next0);
2489         }
2490
2491       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2492     }
2493
2494   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2495                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2496                                pkts_processed);
2497   return frame->n_vectors;
2498 }
2499
2500 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2501   .function = snat_out2in_fast_node_fn,
2502   .name = "nat44-out2in-fast",
2503   .vector_size = sizeof (u32),
2504   .format_trace = format_snat_out2in_fast_trace,
2505   .type = VLIB_NODE_TYPE_INTERNAL,
2506
2507   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2508   .error_strings = snat_out2in_error_strings,
2509
2510   .runtime_data_bytes = sizeof (snat_runtime_t),
2511
2512   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2513
2514   /* edit / add dispositions here */
2515   .next_nodes = {
2516     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2517     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2518     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2519   },
2520 };
2521 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);