src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")                     \
  91 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")
  92
  93 typedef enum {
  94 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  95   foreach_snat_out2in_error
  96 #undef _
  97   SNAT_OUT2IN_N_ERROR,
  98 } snat_out2in_error_t;
  99
 100 static char * snat_out2in_error_strings[] = {
 101 #define _(sym,string) string,
 102   foreach_snat_out2in_error
 103 #undef _
 104 };
 105
 106 typedef enum {
 107   SNAT_OUT2IN_NEXT_DROP,
 108   SNAT_OUT2IN_NEXT_LOOKUP,
 109   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 110   SNAT_OUT2IN_N_NEXT,
 111 } snat_out2in_next_t;
 112
 113 /**
 114  * @brief Create session for static mapping.
 115  *
 116  * Create NAT session initiated by host from external network with static
 117  * mapping.
 118  *
 119  * @param sm     NAT main.
 120  * @param b0     Vlib buffer.
 121  * @param in2out In2out NAT44 session key.
 122  * @param out2in Out2in NAT44 session key.
 123  * @param node   Vlib node.
 124  *
 125  * @returns SNAT session if successfully created otherwise 0.
 126  */
 127 static inline snat_session_t *
 128 create_session_for_static_mapping (snat_main_t *sm,
 129                                    vlib_buffer_t *b0,
 130                                    snat_session_key_t in2out,
 131                                    snat_session_key_t out2in,
 132                                    vlib_node_runtime_t * node,
 133                                    u32 thread_index)
 134 {
 135   snat_user_t *u;
 136   snat_user_key_t user_key;
 137   snat_session_t *s;
 138   clib_bihash_kv_8_8_t kv0, value0;
 139   dlist_elt_t * per_user_translation_list_elt;
 140   dlist_elt_t * per_user_list_head_elt;
 141   ip4_header_t *ip0;
 142
 143   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 144     {
 145       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 146       return 0;
 147     }
 148
 149   ip0 = vlib_buffer_get_current (b0);
 150
 151   user_key.addr = in2out.addr;
 152   user_key.fib_index = in2out.fib_index;
 153   kv0.key = user_key.as_u64;
 154
 155   /* Ever heard of the "user" = inside ip4 address before? */
 156   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 157                               &kv0, &value0))
 158     {
 159       /* no, make a new one */
 160       pool_get (sm->per_thread_data[thread_index].users, u);
 161       memset (u, 0, sizeof (*u));
 162       u->addr = in2out.addr;
 163       u->fib_index = in2out.fib_index;
 164
 165       pool_get (sm->per_thread_data[thread_index].list_pool,
 166                 per_user_list_head_elt);
 167
 168       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 169         sm->per_thread_data[thread_index].list_pool;
 170
 171       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 172                        u->sessions_per_user_list_head_index);
 173
 174       kv0.value = u - sm->per_thread_data[thread_index].users;
 175
 176       /* add user */
 177       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 178                                &kv0, 1 /* is_add */);
 179     }
 180   else
 181     {
 182       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 183                              value0.value);
 184     }
 185
 186   pool_get (sm->per_thread_data[thread_index].sessions, s);
 187   memset (s, 0, sizeof (*s));
 188
 189   s->outside_address_index = ~0;
 190   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 191   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 192   u->nstaticsessions++;
 193
 194   /* Create list elts */
 195   pool_get (sm->per_thread_data[thread_index].list_pool,
 196             per_user_translation_list_elt);
 197   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 198                    per_user_translation_list_elt -
 199                    sm->per_thread_data[thread_index].list_pool);
 200
 201   per_user_translation_list_elt->value =
 202     s - sm->per_thread_data[thread_index].sessions;
 203   s->per_user_index =
 204     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 205   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 206
 207   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 208                       s->per_user_list_head_index,
 209                       per_user_translation_list_elt -
 210                       sm->per_thread_data[thread_index].list_pool);
 211
 212   s->in2out = in2out;
 213   s->out2in = out2in;
 214   s->in2out.protocol = out2in.protocol;
 215
 216   /* Add to translation hashes */
 217   kv0.key = s->in2out.as_u64;
 218   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 219   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 220                                1 /* is_add */))
 221       clib_warning ("in2out key add failed");
 222
 223   kv0.key = s->out2in.as_u64;
 224   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 225
 226   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 227                                1 /* is_add */))
 228       clib_warning ("out2in key add failed");
 229
 230   /* log NAT event */
 231   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 232                                       s->out2in.addr.as_u32,
 233                                       s->in2out.protocol,
 234                                       s->in2out.port,
 235                                       s->out2in.port,
 236                                       s->in2out.fib_index);
 237    return s;
 238 }
 239
 240 static_always_inline
 241 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 242                                  snat_session_key_t *p_key0)
 243 {
 244   icmp46_header_t *icmp0;
 245   snat_session_key_t key0;
 246   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 247   ip4_header_t *inner_ip0;
 248   void *l4_header = 0;
 249   icmp46_header_t *inner_icmp0;
 250
 251   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 252   echo0 = (icmp_echo_header_t *)(icmp0+1);
 253
 254   if (!icmp_is_error_message (icmp0))
 255     {
 256       key0.protocol = SNAT_PROTOCOL_ICMP;
 257       key0.addr = ip0->dst_address;
 258       key0.port = echo0->identifier;
 259     }
 260   else
 261     {
 262       inner_ip0 = (ip4_header_t *)(echo0+1);
 263       l4_header = ip4_next_header (inner_ip0);
 264       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 265       key0.addr = inner_ip0->src_address;
 266       switch (key0.protocol)
 267         {
 268         case SNAT_PROTOCOL_ICMP:
 269           inner_icmp0 = (icmp46_header_t*)l4_header;
 270           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 271           key0.port = inner_echo0->identifier;
 272           break;
 273         case SNAT_PROTOCOL_UDP:
 274         case SNAT_PROTOCOL_TCP:
 275           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 276           break;
 277         default:
 278           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 279         }
 280     }
 281   *p_key0 = key0;
 282   return -1; /* success */
 283 }
 284
 285 /**
 286  * Get address and port values to be used for ICMP packet translation
 287  * and create session if needed
 288  *
 289  * @param[in,out] sm             NAT main
 290  * @param[in,out] node           NAT node runtime
 291  * @param[in] thread_index       thread index
 292  * @param[in,out] b0             buffer containing packet to be translated
 293  * @param[out] p_proto           protocol used for matching
 294  * @param[out] p_value           address and port after NAT translation
 295  * @param[out] p_dont_translate  if packet should not be translated
 296  * @param d                      optional parameter
 297  * @param e                      optional parameter
 298  */
 299 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 300                            u32 thread_index, vlib_buffer_t *b0,
 301                            ip4_header_t *ip0, u8 *p_proto,
 302                            snat_session_key_t *p_value,
 303                            u8 *p_dont_translate, void *d, void *e)
 304 {
 305   icmp46_header_t *icmp0;
 306   u32 sw_if_index0;
 307   u32 rx_fib_index0;
 308   snat_session_key_t key0;
 309   snat_session_key_t sm0;
 310   snat_session_t *s0 = 0;
 311   u8 dont_translate = 0;
 312   clib_bihash_kv_8_8_t kv0, value0;
 313   u8 is_addr_only;
 314   u32 next0 = ~0;
 315   int err;
 316
 317   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 318   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 319   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 320
 321   key0.protocol = 0;
 322
 323   err = icmp_get_key (ip0, &key0);
 324   if (err != -1)
 325     {
 326       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 327       next0 = SNAT_OUT2IN_NEXT_DROP;
 328       goto out;
 329     }
 330   key0.fib_index = rx_fib_index0;
 331
 332   kv0.key = key0.as_u64;
 333
 334   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 335                               &value0))
 336     {
 337       /* Try to match static mapping by external address and port,
 338          destination address and port in packet */
 339       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 340         {
 341           /* Don't NAT packet aimed at the intfc address */
 342           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 343                                               ip0->dst_address.as_u32)))
 344             {
 345               dont_translate = 1;
 346               goto out;
 347             }
 348           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 349           next0 = SNAT_OUT2IN_NEXT_DROP;
 350           goto out;
 351         }
 352
 353       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 354                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 355         {
 356           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 357           next0 = SNAT_OUT2IN_NEXT_DROP;
 358           goto out;
 359         }
 360
 361       /* Create session initiated by host from external network */
 362       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 363                                              node, thread_index);
 364
 365       if (!s0)
 366         {
 367           next0 = SNAT_OUT2IN_NEXT_DROP;
 368           goto out;
 369         }
 370     }
 371   else
 372     {
 373       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 374                         icmp0->type != ICMP4_echo_request &&
 375                         !icmp_is_error_message (icmp0)))
 376         {
 377           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 378           next0 = SNAT_OUT2IN_NEXT_DROP;
 379           goto out;
 380         }
 381
 382       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 383                               value0.value);
 384     }
 385
 386 out:
 387   *p_proto = key0.protocol;
 388   if (s0)
 389     *p_value = s0->in2out;
 390   *p_dont_translate = dont_translate;
 391   if (d)
 392     *(snat_session_t**)d = s0;
 393   return next0;
 394 }
 395
 396 /**
 397  * Get address and port values to be used for ICMP packet translation
 398  *
 399  * @param[in] sm                 NAT main
 400  * @param[in,out] node           NAT node runtime
 401  * @param[in] thread_index       thread index
 402  * @param[in,out] b0             buffer containing packet to be translated
 403  * @param[out] p_proto           protocol used for matching
 404  * @param[out] p_value           address and port after NAT translation
 405  * @param[out] p_dont_translate  if packet should not be translated
 406  * @param d                      optional parameter
 407  * @param e                      optional parameter
 408  */
 409 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 410                            u32 thread_index, vlib_buffer_t *b0,
 411                            ip4_header_t *ip0, u8 *p_proto,
 412                            snat_session_key_t *p_value,
 413                            u8 *p_dont_translate, void *d, void *e)
 414 {
 415   icmp46_header_t *icmp0;
 416   u32 sw_if_index0;
 417   u32 rx_fib_index0;
 418   snat_session_key_t key0;
 419   snat_session_key_t sm0;
 420   u8 dont_translate = 0;
 421   u8 is_addr_only;
 422   u32 next0 = ~0;
 423   int err;
 424
 425   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 426   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 427   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 428
 429   err = icmp_get_key (ip0, &key0);
 430   if (err != -1)
 431     {
 432       b0->error = node->errors[err];
 433       next0 = SNAT_OUT2IN_NEXT_DROP;
 434       goto out2;
 435     }
 436   key0.fib_index = rx_fib_index0;
 437
 438   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 439     {
 440       /* Don't NAT packet aimed at the intfc address */
 441       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 442         {
 443           dont_translate = 1;
 444           goto out;
 445         }
 446       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 447       next0 = SNAT_OUT2IN_NEXT_DROP;
 448       goto out;
 449     }
 450
 451   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 452                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 453                     !icmp_is_error_message (icmp0)))
 454     {
 455       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 456       next0 = SNAT_OUT2IN_NEXT_DROP;
 457       goto out;
 458     }
 459
 460 out:
 461   *p_value = sm0;
 462 out2:
 463   *p_proto = key0.protocol;
 464   *p_dont_translate = dont_translate;
 465   return next0;
 466 }
 467
 468 static inline u32 icmp_out2in (snat_main_t *sm,
 469                                vlib_buffer_t * b0,
 470                                ip4_header_t * ip0,
 471                                icmp46_header_t * icmp0,
 472                                u32 sw_if_index0,
 473                                u32 rx_fib_index0,
 474                                vlib_node_runtime_t * node,
 475                                u32 next0,
 476                                u32 thread_index,
 477                                void *d,
 478                                void *e)
 479 {
 480   snat_session_key_t sm0;
 481   u8 protocol;
 482   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 483   ip4_header_t *inner_ip0 = 0;
 484   void *l4_header = 0;
 485   icmp46_header_t *inner_icmp0;
 486   u8 dont_translate;
 487   u32 new_addr0, old_addr0;
 488   u16 old_id0, new_id0;
 489   ip_csum_t sum0;
 490   u16 checksum0;
 491   u32 next0_tmp;
 492
 493   echo0 = (icmp_echo_header_t *)(icmp0+1);
 494
 495   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
 496                                        &protocol, &sm0, &dont_translate, d, e);
 497   if (next0_tmp != ~0)
 498     next0 = next0_tmp;
 499   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 500     goto out;
 501
 502   sum0 = ip_incremental_checksum (0, icmp0,
 503                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 504   checksum0 = ~ip_csum_fold (sum0);
 505   if (checksum0 != 0 && checksum0 != 0xffff)
 506     {
 507       next0 = SNAT_OUT2IN_NEXT_DROP;
 508       goto out;
 509     }
 510
 511   old_addr0 = ip0->dst_address.as_u32;
 512   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 513   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 514
 515   sum0 = ip0->checksum;
 516   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 517                          dst_address /* changed member */);
 518   ip0->checksum = ip_csum_fold (sum0);
 519
 520   if (!icmp_is_error_message (icmp0))
 521     {
 522       new_id0 = sm0.port;
 523       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 524         {
 525           old_id0 = echo0->identifier;
 526           new_id0 = sm0.port;
 527           echo0->identifier = new_id0;
 528
 529           sum0 = icmp0->checksum;
 530           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 531                                  identifier /* changed member */);
 532           icmp0->checksum = ip_csum_fold (sum0);
 533         }
 534     }
 535   else
 536     {
 537       inner_ip0 = (ip4_header_t *)(echo0+1);
 538       l4_header = ip4_next_header (inner_ip0);
 539
 540       if (!ip4_header_checksum_is_valid (inner_ip0))
 541         {
 542           next0 = SNAT_OUT2IN_NEXT_DROP;
 543           goto out;
 544         }
 545
 546       old_addr0 = inner_ip0->src_address.as_u32;
 547       inner_ip0->src_address = sm0.addr;
 548       new_addr0 = inner_ip0->src_address.as_u32;
 549
 550       sum0 = icmp0->checksum;
 551       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 552                              src_address /* changed member */);
 553       icmp0->checksum = ip_csum_fold (sum0);
 554
 555       switch (protocol)
 556         {
 557         case SNAT_PROTOCOL_ICMP:
 558           inner_icmp0 = (icmp46_header_t*)l4_header;
 559           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 560
 561           old_id0 = inner_echo0->identifier;
 562           new_id0 = sm0.port;
 563           inner_echo0->identifier = new_id0;
 564
 565           sum0 = icmp0->checksum;
 566           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 567                                  identifier);
 568           icmp0->checksum = ip_csum_fold (sum0);
 569           break;
 570         case SNAT_PROTOCOL_UDP:
 571         case SNAT_PROTOCOL_TCP:
 572           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 573           new_id0 = sm0.port;
 574           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 575
 576           sum0 = icmp0->checksum;
 577           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 578                                  src_port);
 579           icmp0->checksum = ip_csum_fold (sum0);
 580           break;
 581         default:
 582           ASSERT(0);
 583         }
 584     }
 585
 586 out:
 587   return next0;
 588 }
 589
 590
 591 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 592                                          vlib_buffer_t * b0,
 593                                          ip4_header_t * ip0,
 594                                          icmp46_header_t * icmp0,
 595                                          u32 sw_if_index0,
 596                                          u32 rx_fib_index0,
 597                                          vlib_node_runtime_t * node,
 598                                          u32 next0, f64 now,
 599                                          u32 thread_index,
 600                                          snat_session_t ** p_s0)
 601 {
 602   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 603                       next0, thread_index, p_s0, 0);
 604   snat_session_t * s0 = *p_s0;
 605   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 606     {
 607       /* Accounting */
 608       s0->last_heard = now;
 609       s0->total_pkts++;
 610       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 611       /* Per-user LRU list maintenance for dynamic translation */
 612       if (!snat_is_session_static (s0))
 613         {
 614           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 615                              s0->per_user_index);
 616           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 617                               s0->per_user_list_head_index,
 618                               s0->per_user_index);
 619         }
 620     }
 621   return next0;
 622 }
 623
 624 static snat_session_t *
 625 snat_out2in_unknown_proto (snat_main_t *sm,
 626                            vlib_buffer_t * b,
 627                            ip4_header_t * ip,
 628                            u32 rx_fib_index,
 629                            u32 thread_index,
 630                            f64 now,
 631                            vlib_main_t * vm,
 632                            vlib_node_runtime_t * node)
 633 {
 634   clib_bihash_kv_8_8_t kv, value;
 635   clib_bihash_kv_16_8_t s_kv, s_value;
 636   snat_static_mapping_t *m;
 637   snat_session_key_t m_key;
 638   u32 old_addr, new_addr;
 639   ip_csum_t sum;
 640   nat_ed_ses_key_t key;
 641   snat_session_t * s;
 642   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 643   snat_user_key_t u_key;
 644   snat_user_t *u;
 645   dlist_elt_t *head, *elt;
 646
 647   old_addr = ip->dst_address.as_u32;
 648
 649   key.l_addr = ip->dst_address;
 650   key.r_addr = ip->src_address;
 651   key.fib_index = rx_fib_index;
 652   key.proto = ip->protocol;
 653   key.rsvd = 0;
 654   key.l_port = 0;
 655   s_kv.key[0] = key.as_u64[0];
 656   s_kv.key[1] = key.as_u64[1];
 657
 658   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 659     {
 660       s = pool_elt_at_index (tsm->sessions, s_value.value);
 661       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 662     }
 663   else
 664     {
 665       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 666         {
 667           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 668           return 0;
 669         }
 670
 671       m_key.addr = ip->dst_address;
 672       m_key.port = 0;
 673       m_key.protocol = 0;
 674       m_key.fib_index = rx_fib_index;
 675       kv.key = m_key.as_u64;
 676       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 677         {
 678           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 679           return 0;
 680         }
 681
 682       m = pool_elt_at_index (sm->static_mappings, value.value);
 683
 684       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 685
 686       u_key.addr = ip->src_address;
 687       u_key.fib_index = m->fib_index;
 688       kv.key = u_key.as_u64;
 689
 690       /* Ever heard of the "user" = src ip4 address before? */
 691       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 692         {
 693           /* no, make a new one */
 694           pool_get (tsm->users, u);
 695           memset (u, 0, sizeof (*u));
 696           u->addr = ip->src_address;
 697           u->fib_index = rx_fib_index;
 698
 699           pool_get (tsm->list_pool, head);
 700           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 701
 702           clib_dlist_init (tsm->list_pool,
 703                            u->sessions_per_user_list_head_index);
 704
 705           kv.value = u - tsm->users;
 706
 707           /* add user */
 708           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
 709         }
 710       else
 711         {
 712           u = pool_elt_at_index (tsm->users, value.value);
 713         }
 714
 715       /* Create a new session */
 716       pool_get (tsm->sessions, s);
 717       memset (s, 0, sizeof (*s));
 718
 719       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 720       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 721       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 722       s->outside_address_index = ~0;
 723       s->out2in.addr.as_u32 = old_addr;
 724       s->out2in.fib_index = rx_fib_index;
 725       s->in2out.addr.as_u32 = new_addr;
 726       s->in2out.fib_index = m->fib_index;
 727       s->in2out.port = s->out2in.port = ip->protocol;
 728       u->nstaticsessions++;
 729
 730       /* Create list elts */
 731       pool_get (tsm->list_pool, elt);
 732       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 733       elt->value = s - tsm->sessions;
 734       s->per_user_index = elt - tsm->list_pool;
 735       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 736       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 737                           s->per_user_index);
 738
 739       /* Add to lookup tables */
 740       s_kv.value = s - tsm->sessions;
 741       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 742         clib_warning ("out2in key add failed");
 743
 744       key.l_addr = ip->dst_address;
 745       key.fib_index = m->fib_index;
 746       s_kv.key[0] = key.as_u64[0];
 747       s_kv.key[1] = key.as_u64[1];
 748       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 749         clib_warning ("in2out key add failed");
 750    }
 751
 752   /* Update IP checksum */
 753   sum = ip->checksum;
 754   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 755   ip->checksum = ip_csum_fold (sum);
 756
 757   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 758
 759   /* Accounting */
 760   s->last_heard = now;
 761   s->total_pkts++;
 762   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 763   /* Per-user LRU list maintenance */
 764   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 765   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 766                       s->per_user_index);
 767
 768   return s;
 769 }
 770
 771 static snat_session_t *
 772 snat_out2in_lb (snat_main_t *sm,
 773                 vlib_buffer_t * b,
 774                 ip4_header_t * ip,
 775                 u32 rx_fib_index,
 776                 u32 thread_index,
 777                 f64 now,
 778                 vlib_main_t * vm,
 779                 vlib_node_runtime_t * node)
 780 {
 781   nat_ed_ses_key_t key;
 782   clib_bihash_kv_16_8_t s_kv, s_value;
 783   udp_header_t *udp = ip4_next_header (ip);
 784   tcp_header_t *tcp = (tcp_header_t *) udp;
 785   snat_session_t *s = 0;
 786   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 787   snat_session_key_t e_key, l_key;
 788   clib_bihash_kv_8_8_t kv, value;
 789   u32 old_addr, new_addr;
 790   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 791   u16 new_port, old_port;
 792   ip_csum_t sum;
 793   snat_user_key_t u_key;
 794   snat_user_t *u;
 795   dlist_elt_t *head, *elt;
 796
 797   old_addr = ip->dst_address.as_u32;
 798
 799   key.l_addr = ip->dst_address;
 800   key.r_addr = ip->src_address;
 801   key.fib_index = rx_fib_index;
 802   key.proto = ip->protocol;
 803   key.rsvd = 0;
 804   key.l_port = udp->dst_port;
 805   s_kv.key[0] = key.as_u64[0];
 806   s_kv.key[1] = key.as_u64[1];
 807
 808   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 809     {
 810       s = pool_elt_at_index (tsm->sessions, s_value.value);
 811     }
 812   else
 813     {
 814       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 815         {
 816           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 817           return 0;
 818         }
 819
 820       e_key.addr = ip->dst_address;
 821       e_key.port = udp->dst_port;
 822       e_key.protocol = proto;
 823       e_key.fib_index = rx_fib_index;
 824       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
 825         return 0;
 826
 827       u_key.addr = l_key.addr;
 828       u_key.fib_index = l_key.fib_index;
 829       kv.key = u_key.as_u64;
 830
 831       /* Ever heard of the "user" = src ip4 address before? */
 832       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 833         {
 834           /* no, make a new one */
 835           pool_get (tsm->users, u);
 836           memset (u, 0, sizeof (*u));
 837           u->addr = l_key.addr;
 838           u->fib_index = l_key.fib_index;
 839
 840           pool_get (tsm->list_pool, head);
 841           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 842
 843           clib_dlist_init (tsm->list_pool,
 844                            u->sessions_per_user_list_head_index);
 845
 846           kv.value = u - tsm->users;
 847
 848           /* add user */
 849           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 850             clib_warning ("user key add failed");
 851         }
 852       else
 853         {
 854           u = pool_elt_at_index (tsm->users, value.value);
 855         }
 856
 857       /* Create a new session */
 858       pool_get (tsm->sessions, s);
 859       memset (s, 0, sizeof (*s));
 860
 861       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 862       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 863       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 864       s->outside_address_index = ~0;
 865       s->out2in = e_key;
 866       s->in2out = l_key;
 867       u->nstaticsessions++;
 868
 869       /* Create list elts */
 870       pool_get (tsm->list_pool, elt);
 871       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 872       elt->value = s - tsm->sessions;
 873       s->per_user_index = elt - tsm->list_pool;
 874       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 875       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 876                           s->per_user_index);
 877
 878       /* Add to lookup tables */
 879       s_kv.value = s - tsm->sessions;
 880       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 881         clib_warning ("out2in-ed key add failed");
 882
 883       key.l_addr = l_key.addr;
 884       key.fib_index = l_key.fib_index;
 885       key.l_port = l_key.port;
 886       s_kv.key[0] = key.as_u64[0];
 887       s_kv.key[1] = key.as_u64[1];
 888       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 889         clib_warning ("in2out-ed key add failed");
 890     }
 891
 892   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 893
 894   /* Update IP checksum */
 895   sum = ip->checksum;
 896   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 897   ip->checksum = ip_csum_fold (sum);
 898
 899   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 900     {
 901       old_port = tcp->dst_port;
 902       tcp->dst_port = s->in2out.port;
 903       new_port = tcp->dst_port;
 904
 905       sum = tcp->checksum;
 906       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 907       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 908       tcp->checksum = ip_csum_fold(sum);
 909     }
 910   else
 911     {
 912       udp->dst_port = s->in2out.port;
 913       udp->checksum = 0;
 914     }
 915
 916   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 917
 918   /* Accounting */
 919   s->last_heard = now;
 920   s->total_pkts++;
 921   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 922   return s;
 923 }
 924
 925 static uword
 926 snat_out2in_node_fn (vlib_main_t * vm,
 927                   vlib_node_runtime_t * node,
 928                   vlib_frame_t * frame)
 929 {
 930   u32 n_left_from, * from, * to_next;
 931   snat_out2in_next_t next_index;
 932   u32 pkts_processed = 0;
 933   snat_main_t * sm = &snat_main;
 934   f64 now = vlib_time_now (vm);
 935   u32 thread_index = vlib_get_thread_index ();
 936
 937   from = vlib_frame_vector_args (frame);
 938   n_left_from = frame->n_vectors;
 939   next_index = node->cached_next_index;
 940
 941   while (n_left_from > 0)
 942     {
 943       u32 n_left_to_next;
 944
 945       vlib_get_next_frame (vm, node, next_index,
 946                            to_next, n_left_to_next);
 947
 948       while (n_left_from >= 4 && n_left_to_next >= 2)
 949         {
 950           u32 bi0, bi1;
 951           vlib_buffer_t * b0, * b1;
 952           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 953           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 954           u32 sw_if_index0, sw_if_index1;
 955           ip4_header_t * ip0, *ip1;
 956           ip_csum_t sum0, sum1;
 957           u32 new_addr0, old_addr0;
 958           u16 new_port0, old_port0;
 959           u32 new_addr1, old_addr1;
 960           u16 new_port1, old_port1;
 961           udp_header_t * udp0, * udp1;
 962           tcp_header_t * tcp0, * tcp1;
 963           icmp46_header_t * icmp0, * icmp1;
 964           snat_session_key_t key0, key1, sm0, sm1;
 965           u32 rx_fib_index0, rx_fib_index1;
 966           u32 proto0, proto1;
 967           snat_session_t * s0 = 0, * s1 = 0;
 968           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 969
 970           /* Prefetch next iteration. */
 971           {
 972             vlib_buffer_t * p2, * p3;
 973
 974             p2 = vlib_get_buffer (vm, from[2]);
 975             p3 = vlib_get_buffer (vm, from[3]);
 976
 977             vlib_prefetch_buffer_header (p2, LOAD);
 978             vlib_prefetch_buffer_header (p3, LOAD);
 979
 980             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 981             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 982           }
 983
 984           /* speculatively enqueue b0 and b1 to the current next frame */
 985           to_next[0] = bi0 = from[0];
 986           to_next[1] = bi1 = from[1];
 987           from += 2;
 988           to_next += 2;
 989           n_left_from -= 2;
 990           n_left_to_next -= 2;
 991
 992           b0 = vlib_get_buffer (vm, bi0);
 993           b1 = vlib_get_buffer (vm, bi1);
 994
 995           vnet_buffer (b0)->snat.flags = 0;
 996           vnet_buffer (b1)->snat.flags = 0;
 997
 998           ip0 = vlib_buffer_get_current (b0);
 999           udp0 = ip4_next_header (ip0);
1000           tcp0 = (tcp_header_t *) udp0;
1001           icmp0 = (icmp46_header_t *) udp0;
1002
1003           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1004           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1005                                    sw_if_index0);
1006
1007           if (PREDICT_FALSE(ip0->ttl == 1))
1008             {
1009               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1010               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1011                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1012                                            0);
1013               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1014               goto trace0;
1015             }
1016
1017           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1018
1019           if (PREDICT_FALSE (proto0 == ~0))
1020             {
1021               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1022                                              thread_index, now, vm, node);
1023               if (!s0)
1024                 next0 = SNAT_OUT2IN_NEXT_DROP;
1025               goto trace0;
1026             }
1027
1028           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1029             {
1030               next0 = icmp_out2in_slow_path
1031                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1032                  next0, now, thread_index, &s0);
1033               goto trace0;
1034             }
1035
1036           key0.addr = ip0->dst_address;
1037           key0.port = udp0->dst_port;
1038           key0.protocol = proto0;
1039           key0.fib_index = rx_fib_index0;
1040
1041           kv0.key = key0.as_u64;
1042
1043           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1044                                       &kv0, &value0))
1045             {
1046               /* Try to match static mapping by external address and port,
1047                  destination address and port in packet */
1048               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1049                 {
1050                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1051                   /*
1052                    * Send DHCP packets to the ipv4 stack, or we won't
1053                    * be able to use dhcp client on the outside interface
1054                    */
1055                   if (proto0 != SNAT_PROTOCOL_UDP
1056                       || (udp0->dst_port
1057                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1058                     next0 = SNAT_OUT2IN_NEXT_DROP;
1059                   goto trace0;
1060                 }
1061
1062               /* Create session initiated by host from external network */
1063               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1064                                                      thread_index);
1065               if (!s0)
1066                 {
1067                   next0 = SNAT_OUT2IN_NEXT_DROP;
1068                   goto trace0;
1069                 }
1070             }
1071           else
1072             {
1073               if (PREDICT_FALSE (value0.value == ~0ULL))
1074                 {
1075                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1076                                       now, vm, node);
1077                   if (!s0)
1078                     next0 = SNAT_OUT2IN_NEXT_DROP;
1079                   goto trace0;
1080                 }
1081               else
1082                 {
1083                   s0 = pool_elt_at_index (
1084                     sm->per_thread_data[thread_index].sessions,
1085                     value0.value);
1086                 }
1087             }
1088
1089           old_addr0 = ip0->dst_address.as_u32;
1090           ip0->dst_address = s0->in2out.addr;
1091           new_addr0 = ip0->dst_address.as_u32;
1092           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1093
1094           sum0 = ip0->checksum;
1095           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1096                                  ip4_header_t,
1097                                  dst_address /* changed member */);
1098           ip0->checksum = ip_csum_fold (sum0);
1099
1100           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1101             {
1102               old_port0 = tcp0->dst_port;
1103               tcp0->dst_port = s0->in2out.port;
1104               new_port0 = tcp0->dst_port;
1105
1106               sum0 = tcp0->checksum;
1107               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1108                                      ip4_header_t,
1109                                      dst_address /* changed member */);
1110
1111               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1112                                      ip4_header_t /* cheat */,
1113                                      length /* changed member */);
1114               tcp0->checksum = ip_csum_fold(sum0);
1115             }
1116           else
1117             {
1118               old_port0 = udp0->dst_port;
1119               udp0->dst_port = s0->in2out.port;
1120               udp0->checksum = 0;
1121             }
1122
1123           /* Accounting */
1124           s0->last_heard = now;
1125           s0->total_pkts++;
1126           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1127           /* Per-user LRU list maintenance for dynamic translation */
1128           if (!snat_is_session_static (s0))
1129             {
1130               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1131                                  s0->per_user_index);
1132               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1133                                   s0->per_user_list_head_index,
1134                                   s0->per_user_index);
1135             }
1136         trace0:
1137
1138           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1139                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1140             {
1141               snat_out2in_trace_t *t =
1142                  vlib_add_trace (vm, node, b0, sizeof (*t));
1143               t->sw_if_index = sw_if_index0;
1144               t->next_index = next0;
1145               t->session_index = ~0;
1146               if (s0)
1147                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1148             }
1149
1150           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1151
1152
1153           ip1 = vlib_buffer_get_current (b1);
1154           udp1 = ip4_next_header (ip1);
1155           tcp1 = (tcp_header_t *) udp1;
1156           icmp1 = (icmp46_header_t *) udp1;
1157
1158           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1159           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1160                                    sw_if_index1);
1161
1162           if (PREDICT_FALSE(ip1->ttl == 1))
1163             {
1164               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1165               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1166                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1167                                            0);
1168               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1169               goto trace1;
1170             }
1171
1172           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1173
1174           if (PREDICT_FALSE (proto1 == ~0))
1175             {
1176               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1177                                              thread_index, now, vm, node);
1178               if (!s1)
1179                 next1 = SNAT_OUT2IN_NEXT_DROP;
1180               goto trace1;
1181             }
1182
1183           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1184             {
1185               next1 = icmp_out2in_slow_path
1186                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1187                  next1, now, thread_index, &s1);
1188               goto trace1;
1189             }
1190
1191           key1.addr = ip1->dst_address;
1192           key1.port = udp1->dst_port;
1193           key1.protocol = proto1;
1194           key1.fib_index = rx_fib_index1;
1195
1196           kv1.key = key1.as_u64;
1197
1198           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1199                                       &kv1, &value1))
1200             {
1201               /* Try to match static mapping by external address and port,
1202                  destination address and port in packet */
1203               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1204                 {
1205                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1206                   /*
1207                    * Send DHCP packets to the ipv4 stack, or we won't
1208                    * be able to use dhcp client on the outside interface
1209                    */
1210                   if (proto1 != SNAT_PROTOCOL_UDP
1211                       || (udp1->dst_port
1212                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1213                     next1 = SNAT_OUT2IN_NEXT_DROP;
1214                   goto trace1;
1215                 }
1216
1217               /* Create session initiated by host from external network */
1218               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1219                                                      thread_index);
1220               if (!s1)
1221                 {
1222                   next1 = SNAT_OUT2IN_NEXT_DROP;
1223                   goto trace1;
1224                 }
1225             }
1226           else
1227             {
1228               if (PREDICT_FALSE (value1.value == ~0ULL))
1229                 {
1230                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1231                                       now, vm, node);
1232                   if (!s1)
1233                     next1 = SNAT_OUT2IN_NEXT_DROP;
1234                   goto trace1;
1235                 }
1236               else
1237                 {
1238                   s1 = pool_elt_at_index (
1239                     sm->per_thread_data[thread_index].sessions,
1240                     value1.value);
1241                 }
1242             }
1243
1244           old_addr1 = ip1->dst_address.as_u32;
1245           ip1->dst_address = s1->in2out.addr;
1246           new_addr1 = ip1->dst_address.as_u32;
1247           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1248
1249           sum1 = ip1->checksum;
1250           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1251                                  ip4_header_t,
1252                                  dst_address /* changed member */);
1253           ip1->checksum = ip_csum_fold (sum1);
1254
1255           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1256             {
1257               old_port1 = tcp1->dst_port;
1258               tcp1->dst_port = s1->in2out.port;
1259               new_port1 = tcp1->dst_port;
1260
1261               sum1 = tcp1->checksum;
1262               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1263                                      ip4_header_t,
1264                                      dst_address /* changed member */);
1265
1266               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1267                                      ip4_header_t /* cheat */,
1268                                      length /* changed member */);
1269               tcp1->checksum = ip_csum_fold(sum1);
1270             }
1271           else
1272             {
1273               old_port1 = udp1->dst_port;
1274               udp1->dst_port = s1->in2out.port;
1275               udp1->checksum = 0;
1276             }
1277
1278           /* Accounting */
1279           s1->last_heard = now;
1280           s1->total_pkts++;
1281           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1282           /* Per-user LRU list maintenance for dynamic translation */
1283           if (!snat_is_session_static (s1))
1284             {
1285               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1286                                  s1->per_user_index);
1287               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1288                                   s1->per_user_list_head_index,
1289                                   s1->per_user_index);
1290             }
1291         trace1:
1292
1293           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1294                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1295             {
1296               snat_out2in_trace_t *t =
1297                  vlib_add_trace (vm, node, b1, sizeof (*t));
1298               t->sw_if_index = sw_if_index1;
1299               t->next_index = next1;
1300               t->session_index = ~0;
1301               if (s1)
1302                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1303             }
1304
1305           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1306
1307           /* verify speculative enqueues, maybe switch current next frame */
1308           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1309                                            to_next, n_left_to_next,
1310                                            bi0, bi1, next0, next1);
1311         }
1312
1313       while (n_left_from > 0 && n_left_to_next > 0)
1314         {
1315           u32 bi0;
1316           vlib_buffer_t * b0;
1317           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1318           u32 sw_if_index0;
1319           ip4_header_t * ip0;
1320           ip_csum_t sum0;
1321           u32 new_addr0, old_addr0;
1322           u16 new_port0, old_port0;
1323           udp_header_t * udp0;
1324           tcp_header_t * tcp0;
1325           icmp46_header_t * icmp0;
1326           snat_session_key_t key0, sm0;
1327           u32 rx_fib_index0;
1328           u32 proto0;
1329           snat_session_t * s0 = 0;
1330           clib_bihash_kv_8_8_t kv0, value0;
1331
1332           /* speculatively enqueue b0 to the current next frame */
1333           bi0 = from[0];
1334           to_next[0] = bi0;
1335           from += 1;
1336           to_next += 1;
1337           n_left_from -= 1;
1338           n_left_to_next -= 1;
1339
1340           b0 = vlib_get_buffer (vm, bi0);
1341
1342           vnet_buffer (b0)->snat.flags = 0;
1343
1344           ip0 = vlib_buffer_get_current (b0);
1345           udp0 = ip4_next_header (ip0);
1346           tcp0 = (tcp_header_t *) udp0;
1347           icmp0 = (icmp46_header_t *) udp0;
1348
1349           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1350           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1351                                    sw_if_index0);
1352
1353           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1354
1355           if (PREDICT_FALSE (proto0 == ~0))
1356             {
1357               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1358                                              thread_index, now, vm, node);
1359               if (!s0)
1360                 next0 = SNAT_OUT2IN_NEXT_DROP;
1361               goto trace00;
1362             }
1363
1364           if (PREDICT_FALSE(ip0->ttl == 1))
1365             {
1366               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1367               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1368                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1369                                            0);
1370               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1371               goto trace00;
1372             }
1373
1374           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1375             {
1376               next0 = icmp_out2in_slow_path
1377                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1378                  next0, now, thread_index, &s0);
1379               goto trace00;
1380             }
1381
1382           key0.addr = ip0->dst_address;
1383           key0.port = udp0->dst_port;
1384           key0.protocol = proto0;
1385           key0.fib_index = rx_fib_index0;
1386
1387           kv0.key = key0.as_u64;
1388
1389           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1390                                       &kv0, &value0))
1391             {
1392               /* Try to match static mapping by external address and port,
1393                  destination address and port in packet */
1394               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1395                 {
1396                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1397                   /*
1398                    * Send DHCP packets to the ipv4 stack, or we won't
1399                    * be able to use dhcp client on the outside interface
1400                    */
1401                   if (proto0 != SNAT_PROTOCOL_UDP
1402                       || (udp0->dst_port
1403                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1404
1405                     next0 = SNAT_OUT2IN_NEXT_DROP;
1406                   goto trace00;
1407                 }
1408
1409               /* Create session initiated by host from external network */
1410               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1411                                                      thread_index);
1412               if (!s0)
1413                 {
1414                   next0 = SNAT_OUT2IN_NEXT_DROP;
1415                   goto trace00;
1416                 }
1417             }
1418           else
1419             {
1420               if (PREDICT_FALSE (value0.value == ~0ULL))
1421                 {
1422                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1423                                       now, vm, node);
1424                   if (!s0)
1425                     next0 = SNAT_OUT2IN_NEXT_DROP;
1426                   goto trace00;
1427                 }
1428               else
1429                 {
1430                   s0 = pool_elt_at_index (
1431                     sm->per_thread_data[thread_index].sessions,
1432                     value0.value);
1433                 }
1434             }
1435
1436           old_addr0 = ip0->dst_address.as_u32;
1437           ip0->dst_address = s0->in2out.addr;
1438           new_addr0 = ip0->dst_address.as_u32;
1439           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1440
1441           sum0 = ip0->checksum;
1442           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1443                                  ip4_header_t,
1444                                  dst_address /* changed member */);
1445           ip0->checksum = ip_csum_fold (sum0);
1446
1447           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1448             {
1449               old_port0 = tcp0->dst_port;
1450               tcp0->dst_port = s0->in2out.port;
1451               new_port0 = tcp0->dst_port;
1452
1453               sum0 = tcp0->checksum;
1454               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1455                                      ip4_header_t,
1456                                      dst_address /* changed member */);
1457
1458               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1459                                      ip4_header_t /* cheat */,
1460                                      length /* changed member */);
1461               tcp0->checksum = ip_csum_fold(sum0);
1462             }
1463           else
1464             {
1465               old_port0 = udp0->dst_port;
1466               udp0->dst_port = s0->in2out.port;
1467               udp0->checksum = 0;
1468             }
1469
1470           /* Accounting */
1471           s0->last_heard = now;
1472           s0->total_pkts++;
1473           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1474           /* Per-user LRU list maintenance for dynamic translation */
1475           if (!snat_is_session_static (s0))
1476             {
1477               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1478                                  s0->per_user_index);
1479               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1480                                   s0->per_user_list_head_index,
1481                                   s0->per_user_index);
1482             }
1483         trace00:
1484
1485           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1486                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1487             {
1488               snat_out2in_trace_t *t =
1489                  vlib_add_trace (vm, node, b0, sizeof (*t));
1490               t->sw_if_index = sw_if_index0;
1491               t->next_index = next0;
1492               t->session_index = ~0;
1493               if (s0)
1494                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1495             }
1496
1497           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1498
1499           /* verify speculative enqueue, maybe switch current next frame */
1500           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1501                                            to_next, n_left_to_next,
1502                                            bi0, next0);
1503         }
1504
1505       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1506     }
1507
1508   vlib_node_increment_counter (vm, snat_out2in_node.index,
1509                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1510                                pkts_processed);
1511   return frame->n_vectors;
1512 }
1513
1514 VLIB_REGISTER_NODE (snat_out2in_node) = {
1515   .function = snat_out2in_node_fn,
1516   .name = "nat44-out2in",
1517   .vector_size = sizeof (u32),
1518   .format_trace = format_snat_out2in_trace,
1519   .type = VLIB_NODE_TYPE_INTERNAL,
1520
1521   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1522   .error_strings = snat_out2in_error_strings,
1523
1524   .runtime_data_bytes = sizeof (snat_runtime_t),
1525
1526   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1527
1528   /* edit / add dispositions here */
1529   .next_nodes = {
1530     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1531     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1532     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1533   },
1534 };
1535 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1536
1537 /**************************/
1538 /*** deterministic mode ***/
1539 /**************************/
1540 static uword
1541 snat_det_out2in_node_fn (vlib_main_t * vm,
1542                          vlib_node_runtime_t * node,
1543                          vlib_frame_t * frame)
1544 {
1545   u32 n_left_from, * from, * to_next;
1546   snat_out2in_next_t next_index;
1547   u32 pkts_processed = 0;
1548   snat_main_t * sm = &snat_main;
1549   u32 thread_index = vlib_get_thread_index ();
1550
1551   from = vlib_frame_vector_args (frame);
1552   n_left_from = frame->n_vectors;
1553   next_index = node->cached_next_index;
1554
1555   while (n_left_from > 0)
1556     {
1557       u32 n_left_to_next;
1558
1559       vlib_get_next_frame (vm, node, next_index,
1560                            to_next, n_left_to_next);
1561
1562       while (n_left_from >= 4 && n_left_to_next >= 2)
1563         {
1564           u32 bi0, bi1;
1565           vlib_buffer_t * b0, * b1;
1566           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1567           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1568           u32 sw_if_index0, sw_if_index1;
1569           ip4_header_t * ip0, * ip1;
1570           ip_csum_t sum0, sum1;
1571           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1572           u16 new_port0, old_port0, old_port1, new_port1;
1573           udp_header_t * udp0, * udp1;
1574           tcp_header_t * tcp0, * tcp1;
1575           u32 proto0, proto1;
1576           snat_det_out_key_t key0, key1;
1577           snat_det_map_t * dm0, * dm1;
1578           snat_det_session_t * ses0 = 0, * ses1 = 0;
1579           u32 rx_fib_index0, rx_fib_index1;
1580           icmp46_header_t * icmp0, * icmp1;
1581
1582           /* Prefetch next iteration. */
1583           {
1584             vlib_buffer_t * p2, * p3;
1585
1586             p2 = vlib_get_buffer (vm, from[2]);
1587             p3 = vlib_get_buffer (vm, from[3]);
1588
1589             vlib_prefetch_buffer_header (p2, LOAD);
1590             vlib_prefetch_buffer_header (p3, LOAD);
1591
1592             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1593             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1594           }
1595
1596           /* speculatively enqueue b0 and b1 to the current next frame */
1597           to_next[0] = bi0 = from[0];
1598           to_next[1] = bi1 = from[1];
1599           from += 2;
1600           to_next += 2;
1601           n_left_from -= 2;
1602           n_left_to_next -= 2;
1603
1604           b0 = vlib_get_buffer (vm, bi0);
1605           b1 = vlib_get_buffer (vm, bi1);
1606
1607           ip0 = vlib_buffer_get_current (b0);
1608           udp0 = ip4_next_header (ip0);
1609           tcp0 = (tcp_header_t *) udp0;
1610
1611           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1612
1613           if (PREDICT_FALSE(ip0->ttl == 1))
1614             {
1615               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1616               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1617                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1618                                            0);
1619               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1620               goto trace0;
1621             }
1622
1623           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1624
1625           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1626             {
1627               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1628               icmp0 = (icmp46_header_t *) udp0;
1629
1630               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1631                                   rx_fib_index0, node, next0, thread_index,
1632                                   &ses0, &dm0);
1633               goto trace0;
1634             }
1635
1636           key0.ext_host_addr = ip0->src_address;
1637           key0.ext_host_port = tcp0->src;
1638           key0.out_port = tcp0->dst;
1639
1640           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1641           if (PREDICT_FALSE(!dm0))
1642             {
1643               clib_warning("unknown dst address:  %U",
1644                            format_ip4_address, &ip0->dst_address);
1645               next0 = SNAT_OUT2IN_NEXT_DROP;
1646               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1647               goto trace0;
1648             }
1649
1650           snat_det_reverse(dm0, &ip0->dst_address,
1651                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1652
1653           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1654           if (PREDICT_FALSE(!ses0))
1655             {
1656               clib_warning("no match src %U:%d dst %U:%d for user %U",
1657                            format_ip4_address, &ip0->src_address,
1658                            clib_net_to_host_u16 (tcp0->src),
1659                            format_ip4_address, &ip0->dst_address,
1660                            clib_net_to_host_u16 (tcp0->dst),
1661                            format_ip4_address, &new_addr0);
1662               next0 = SNAT_OUT2IN_NEXT_DROP;
1663               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1664               goto trace0;
1665             }
1666           new_port0 = ses0->in_port;
1667
1668           old_addr0 = ip0->dst_address;
1669           ip0->dst_address = new_addr0;
1670           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1671
1672           sum0 = ip0->checksum;
1673           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1674                                  ip4_header_t,
1675                                  dst_address /* changed member */);
1676           ip0->checksum = ip_csum_fold (sum0);
1677
1678           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1679             {
1680               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1681                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1682               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1683                 snat_det_ses_close(dm0, ses0);
1684
1685               old_port0 = tcp0->dst;
1686               tcp0->dst = new_port0;
1687
1688               sum0 = tcp0->checksum;
1689               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1690                                      ip4_header_t,
1691                                      dst_address /* changed member */);
1692
1693               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1694                                      ip4_header_t /* cheat */,
1695                                      length /* changed member */);
1696               tcp0->checksum = ip_csum_fold(sum0);
1697             }
1698           else
1699             {
1700               old_port0 = udp0->dst_port;
1701               udp0->dst_port = new_port0;
1702               udp0->checksum = 0;
1703             }
1704
1705         trace0:
1706
1707           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1708                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1709             {
1710               snat_out2in_trace_t *t =
1711                  vlib_add_trace (vm, node, b0, sizeof (*t));
1712               t->sw_if_index = sw_if_index0;
1713               t->next_index = next0;
1714               t->session_index = ~0;
1715               if (ses0)
1716                 t->session_index = ses0 - dm0->sessions;
1717             }
1718
1719           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1720
1721           b1 = vlib_get_buffer (vm, bi1);
1722
1723           ip1 = vlib_buffer_get_current (b1);
1724           udp1 = ip4_next_header (ip1);
1725           tcp1 = (tcp_header_t *) udp1;
1726
1727           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1728
1729           if (PREDICT_FALSE(ip1->ttl == 1))
1730             {
1731               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1732               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1733                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1734                                            0);
1735               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1736               goto trace1;
1737             }
1738
1739           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1740
1741           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1742             {
1743               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1744               icmp1 = (icmp46_header_t *) udp1;
1745
1746               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1747                                   rx_fib_index1, node, next1, thread_index,
1748                                   &ses1, &dm1);
1749               goto trace1;
1750             }
1751
1752           key1.ext_host_addr = ip1->src_address;
1753           key1.ext_host_port = tcp1->src;
1754           key1.out_port = tcp1->dst;
1755
1756           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1757           if (PREDICT_FALSE(!dm1))
1758             {
1759               clib_warning("unknown dst address:  %U",
1760                            format_ip4_address, &ip1->dst_address);
1761               next1 = SNAT_OUT2IN_NEXT_DROP;
1762               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1763               goto trace1;
1764             }
1765
1766           snat_det_reverse(dm1, &ip1->dst_address,
1767                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1768
1769           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1770           if (PREDICT_FALSE(!ses1))
1771             {
1772               clib_warning("no match src %U:%d dst %U:%d for user %U",
1773                            format_ip4_address, &ip1->src_address,
1774                            clib_net_to_host_u16 (tcp1->src),
1775                            format_ip4_address, &ip1->dst_address,
1776                            clib_net_to_host_u16 (tcp1->dst),
1777                            format_ip4_address, &new_addr1);
1778               next1 = SNAT_OUT2IN_NEXT_DROP;
1779               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1780               goto trace1;
1781             }
1782           new_port1 = ses1->in_port;
1783
1784           old_addr1 = ip1->dst_address;
1785           ip1->dst_address = new_addr1;
1786           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1787
1788           sum1 = ip1->checksum;
1789           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1790                                  ip4_header_t,
1791                                  dst_address /* changed member */);
1792           ip1->checksum = ip_csum_fold (sum1);
1793
1794           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1795             {
1796               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1797                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1798               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1799                 snat_det_ses_close(dm1, ses1);
1800
1801               old_port1 = tcp1->dst;
1802               tcp1->dst = new_port1;
1803
1804               sum1 = tcp1->checksum;
1805               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1806                                      ip4_header_t,
1807                                      dst_address /* changed member */);
1808
1809               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1810                                      ip4_header_t /* cheat */,
1811                                      length /* changed member */);
1812               tcp1->checksum = ip_csum_fold(sum1);
1813             }
1814           else
1815             {
1816               old_port1 = udp1->dst_port;
1817               udp1->dst_port = new_port1;
1818               udp1->checksum = 0;
1819             }
1820
1821         trace1:
1822
1823           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1824                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1825             {
1826               snat_out2in_trace_t *t =
1827                  vlib_add_trace (vm, node, b1, sizeof (*t));
1828               t->sw_if_index = sw_if_index1;
1829               t->next_index = next1;
1830               t->session_index = ~0;
1831               if (ses1)
1832                 t->session_index = ses1 - dm1->sessions;
1833             }
1834
1835           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1836
1837           /* verify speculative enqueues, maybe switch current next frame */
1838           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1839                                            to_next, n_left_to_next,
1840                                            bi0, bi1, next0, next1);
1841          }
1842
1843       while (n_left_from > 0 && n_left_to_next > 0)
1844         {
1845           u32 bi0;
1846           vlib_buffer_t * b0;
1847           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1848           u32 sw_if_index0;
1849           ip4_header_t * ip0;
1850           ip_csum_t sum0;
1851           ip4_address_t new_addr0, old_addr0;
1852           u16 new_port0, old_port0;
1853           udp_header_t * udp0;
1854           tcp_header_t * tcp0;
1855           u32 proto0;
1856           snat_det_out_key_t key0;
1857           snat_det_map_t * dm0;
1858           snat_det_session_t * ses0 = 0;
1859           u32 rx_fib_index0;
1860           icmp46_header_t * icmp0;
1861
1862           /* speculatively enqueue b0 to the current next frame */
1863           bi0 = from[0];
1864           to_next[0] = bi0;
1865           from += 1;
1866           to_next += 1;
1867           n_left_from -= 1;
1868           n_left_to_next -= 1;
1869
1870           b0 = vlib_get_buffer (vm, bi0);
1871
1872           ip0 = vlib_buffer_get_current (b0);
1873           udp0 = ip4_next_header (ip0);
1874           tcp0 = (tcp_header_t *) udp0;
1875
1876           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1877
1878           if (PREDICT_FALSE(ip0->ttl == 1))
1879             {
1880               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1881               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1882                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1883                                            0);
1884               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1885               goto trace00;
1886             }
1887
1888           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1889
1890           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1891             {
1892               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1893               icmp0 = (icmp46_header_t *) udp0;
1894
1895               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1896                                   rx_fib_index0, node, next0, thread_index,
1897                                   &ses0, &dm0);
1898               goto trace00;
1899             }
1900
1901           key0.ext_host_addr = ip0->src_address;
1902           key0.ext_host_port = tcp0->src;
1903           key0.out_port = tcp0->dst;
1904
1905           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1906           if (PREDICT_FALSE(!dm0))
1907             {
1908               clib_warning("unknown dst address:  %U",
1909                            format_ip4_address, &ip0->dst_address);
1910               next0 = SNAT_OUT2IN_NEXT_DROP;
1911               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1912               goto trace00;
1913             }
1914
1915           snat_det_reverse(dm0, &ip0->dst_address,
1916                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1917
1918           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1919           if (PREDICT_FALSE(!ses0))
1920             {
1921               clib_warning("no match src %U:%d dst %U:%d for user %U",
1922                            format_ip4_address, &ip0->src_address,
1923                            clib_net_to_host_u16 (tcp0->src),
1924                            format_ip4_address, &ip0->dst_address,
1925                            clib_net_to_host_u16 (tcp0->dst),
1926                            format_ip4_address, &new_addr0);
1927               next0 = SNAT_OUT2IN_NEXT_DROP;
1928               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1929               goto trace00;
1930             }
1931           new_port0 = ses0->in_port;
1932
1933           old_addr0 = ip0->dst_address;
1934           ip0->dst_address = new_addr0;
1935           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1936
1937           sum0 = ip0->checksum;
1938           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1939                                  ip4_header_t,
1940                                  dst_address /* changed member */);
1941           ip0->checksum = ip_csum_fold (sum0);
1942
1943           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1944             {
1945               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1946                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1947               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1948                 snat_det_ses_close(dm0, ses0);
1949
1950               old_port0 = tcp0->dst;
1951               tcp0->dst = new_port0;
1952
1953               sum0 = tcp0->checksum;
1954               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1955                                      ip4_header_t,
1956                                      dst_address /* changed member */);
1957
1958               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1959                                      ip4_header_t /* cheat */,
1960                                      length /* changed member */);
1961               tcp0->checksum = ip_csum_fold(sum0);
1962             }
1963           else
1964             {
1965               old_port0 = udp0->dst_port;
1966               udp0->dst_port = new_port0;
1967               udp0->checksum = 0;
1968             }
1969
1970         trace00:
1971
1972           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1973                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1974             {
1975               snat_out2in_trace_t *t =
1976                  vlib_add_trace (vm, node, b0, sizeof (*t));
1977               t->sw_if_index = sw_if_index0;
1978               t->next_index = next0;
1979               t->session_index = ~0;
1980               if (ses0)
1981                 t->session_index = ses0 - dm0->sessions;
1982             }
1983
1984           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1985
1986           /* verify speculative enqueue, maybe switch current next frame */
1987           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1988                                            to_next, n_left_to_next,
1989                                            bi0, next0);
1990         }
1991
1992       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1993     }
1994
1995   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1996                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1997                                pkts_processed);
1998   return frame->n_vectors;
1999 }
2000
2001 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2002   .function = snat_det_out2in_node_fn,
2003   .name = "nat44-det-out2in",
2004   .vector_size = sizeof (u32),
2005   .format_trace = format_snat_out2in_trace,
2006   .type = VLIB_NODE_TYPE_INTERNAL,
2007
2008   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2009   .error_strings = snat_out2in_error_strings,
2010
2011   .runtime_data_bytes = sizeof (snat_runtime_t),
2012
2013   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2014
2015   /* edit / add dispositions here */
2016   .next_nodes = {
2017     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2018     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2019     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2020   },
2021 };
2022 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2023
2024 /**
2025  * Get address and port values to be used for ICMP packet translation
2026  * and create session if needed
2027  *
2028  * @param[in,out] sm             NAT main
2029  * @param[in,out] node           NAT node runtime
2030  * @param[in] thread_index       thread index
2031  * @param[in,out] b0             buffer containing packet to be translated
2032  * @param[out] p_proto           protocol used for matching
2033  * @param[out] p_value           address and port after NAT translation
2034  * @param[out] p_dont_translate  if packet should not be translated
2035  * @param d                      optional parameter
2036  * @param e                      optional parameter
2037  */
2038 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2039                           u32 thread_index, vlib_buffer_t *b0,
2040                           ip4_header_t *ip0, u8 *p_proto,
2041                           snat_session_key_t *p_value,
2042                           u8 *p_dont_translate, void *d, void *e)
2043 {
2044   icmp46_header_t *icmp0;
2045   u32 sw_if_index0;
2046   u8 protocol;
2047   snat_det_out_key_t key0;
2048   u8 dont_translate = 0;
2049   u32 next0 = ~0;
2050   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2051   ip4_header_t *inner_ip0;
2052   void *l4_header = 0;
2053   icmp46_header_t *inner_icmp0;
2054   snat_det_map_t * dm0 = 0;
2055   ip4_address_t new_addr0 = {{0}};
2056   snat_det_session_t * ses0 = 0;
2057   ip4_address_t out_addr;
2058
2059   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2060   echo0 = (icmp_echo_header_t *)(icmp0+1);
2061   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2062
2063   if (!icmp_is_error_message (icmp0))
2064     {
2065       protocol = SNAT_PROTOCOL_ICMP;
2066       key0.ext_host_addr = ip0->src_address;
2067       key0.ext_host_port = 0;
2068       key0.out_port = echo0->identifier;
2069       out_addr = ip0->dst_address;
2070     }
2071   else
2072     {
2073       inner_ip0 = (ip4_header_t *)(echo0+1);
2074       l4_header = ip4_next_header (inner_ip0);
2075       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2076       key0.ext_host_addr = inner_ip0->dst_address;
2077       out_addr = inner_ip0->src_address;
2078       switch (protocol)
2079         {
2080         case SNAT_PROTOCOL_ICMP:
2081           inner_icmp0 = (icmp46_header_t*)l4_header;
2082           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2083           key0.ext_host_port = 0;
2084           key0.out_port = inner_echo0->identifier;
2085           break;
2086         case SNAT_PROTOCOL_UDP:
2087         case SNAT_PROTOCOL_TCP:
2088           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2089           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2090           break;
2091         default:
2092           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2093           next0 = SNAT_OUT2IN_NEXT_DROP;
2094           goto out;
2095         }
2096     }
2097
2098   dm0 = snat_det_map_by_out(sm, &out_addr);
2099   if (PREDICT_FALSE(!dm0))
2100     {
2101       /* Don't NAT packet aimed at the intfc address */
2102       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2103                                           ip0->dst_address.as_u32)))
2104         {
2105           dont_translate = 1;
2106           goto out;
2107         }
2108       clib_warning("unknown dst address:  %U",
2109                    format_ip4_address, &ip0->dst_address);
2110       goto out;
2111     }
2112
2113   snat_det_reverse(dm0, &ip0->dst_address,
2114                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2115
2116   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2117   if (PREDICT_FALSE(!ses0))
2118     {
2119       /* Don't NAT packet aimed at the intfc address */
2120       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2121                                           ip0->dst_address.as_u32)))
2122         {
2123           dont_translate = 1;
2124           goto out;
2125         }
2126       clib_warning("no match src %U:%d dst %U:%d for user %U",
2127                    format_ip4_address, &key0.ext_host_addr,
2128                    clib_net_to_host_u16 (key0.ext_host_port),
2129                    format_ip4_address, &out_addr,
2130                    clib_net_to_host_u16 (key0.out_port),
2131                    format_ip4_address, &new_addr0);
2132       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2133       next0 = SNAT_OUT2IN_NEXT_DROP;
2134       goto out;
2135     }
2136
2137   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2138                     !icmp_is_error_message (icmp0)))
2139     {
2140       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2141       next0 = SNAT_OUT2IN_NEXT_DROP;
2142       goto out;
2143     }
2144
2145   goto out;
2146
2147 out:
2148   *p_proto = protocol;
2149   if (ses0)
2150     {
2151       p_value->addr = new_addr0;
2152       p_value->fib_index = sm->inside_fib_index;
2153       p_value->port = ses0->in_port;
2154     }
2155   *p_dont_translate = dont_translate;
2156   if (d)
2157     *(snat_det_session_t**)d = ses0;
2158   if (e)
2159     *(snat_det_map_t**)e = dm0;
2160   return next0;
2161 }
2162
2163 /**********************/
2164 /*** worker handoff ***/
2165 /**********************/
2166 static uword
2167 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2168                                vlib_node_runtime_t * node,
2169                                vlib_frame_t * frame)
2170 {
2171   snat_main_t *sm = &snat_main;
2172   vlib_thread_main_t *tm = vlib_get_thread_main ();
2173   u32 n_left_from, *from, *to_next = 0;
2174   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2175   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2176     = 0;
2177   vlib_frame_queue_elt_t *hf = 0;
2178   vlib_frame_t *f = 0;
2179   int i;
2180   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2181   u32 next_worker_index = 0;
2182   u32 current_worker_index = ~0;
2183   u32 thread_index = vlib_get_thread_index ();
2184
2185   ASSERT (vec_len (sm->workers));
2186
2187   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2188     {
2189       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2190
2191       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2192                                sm->first_worker_index + sm->num_workers - 1,
2193                                (vlib_frame_queue_t *) (~0));
2194     }
2195
2196   from = vlib_frame_vector_args (frame);
2197   n_left_from = frame->n_vectors;
2198
2199   while (n_left_from > 0)
2200     {
2201       u32 bi0;
2202       vlib_buffer_t *b0;
2203       u32 sw_if_index0;
2204       u32 rx_fib_index0;
2205       ip4_header_t * ip0;
2206       u8 do_handoff;
2207
2208       bi0 = from[0];
2209       from += 1;
2210       n_left_from -= 1;
2211
2212       b0 = vlib_get_buffer (vm, bi0);
2213
2214       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2215       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2216
2217       ip0 = vlib_buffer_get_current (b0);
2218
2219       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2220
2221       if (PREDICT_FALSE (next_worker_index != thread_index))
2222         {
2223           do_handoff = 1;
2224
2225           if (next_worker_index != current_worker_index)
2226             {
2227               if (hf)
2228                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2229
2230               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2231                                                       next_worker_index,
2232                                                       handoff_queue_elt_by_worker_index);
2233
2234               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2235               to_next_worker = &hf->buffer_index[hf->n_vectors];
2236               current_worker_index = next_worker_index;
2237             }
2238
2239           /* enqueue to correct worker thread */
2240           to_next_worker[0] = bi0;
2241           to_next_worker++;
2242           n_left_to_next_worker--;
2243
2244           if (n_left_to_next_worker == 0)
2245             {
2246               hf->n_vectors = VLIB_FRAME_SIZE;
2247               vlib_put_frame_queue_elt (hf);
2248               current_worker_index = ~0;
2249               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2250               hf = 0;
2251             }
2252         }
2253       else
2254         {
2255           do_handoff = 0;
2256           /* if this is 1st frame */
2257           if (!f)
2258             {
2259               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2260               to_next = vlib_frame_vector_args (f);
2261             }
2262
2263           to_next[0] = bi0;
2264           to_next += 1;
2265           f->n_vectors++;
2266         }
2267
2268       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2269                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2270         {
2271           snat_out2in_worker_handoff_trace_t *t =
2272             vlib_add_trace (vm, node, b0, sizeof (*t));
2273           t->next_worker_index = next_worker_index;
2274           t->do_handoff = do_handoff;
2275         }
2276     }
2277
2278   if (f)
2279     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2280
2281   if (hf)
2282     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2283
2284   /* Ship frames to the worker nodes */
2285   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2286     {
2287       if (handoff_queue_elt_by_worker_index[i])
2288         {
2289           hf = handoff_queue_elt_by_worker_index[i];
2290           /*
2291            * It works better to let the handoff node
2292            * rate-adapt, always ship the handoff queue element.
2293            */
2294           if (1 || hf->n_vectors == hf->last_n_vectors)
2295             {
2296               vlib_put_frame_queue_elt (hf);
2297               handoff_queue_elt_by_worker_index[i] = 0;
2298             }
2299           else
2300             hf->last_n_vectors = hf->n_vectors;
2301         }
2302       congested_handoff_queue_by_worker_index[i] =
2303         (vlib_frame_queue_t *) (~0);
2304     }
2305   hf = 0;
2306   current_worker_index = ~0;
2307   return frame->n_vectors;
2308 }
2309
2310 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2311   .function = snat_out2in_worker_handoff_fn,
2312   .name = "nat44-out2in-worker-handoff",
2313   .vector_size = sizeof (u32),
2314   .format_trace = format_snat_out2in_worker_handoff_trace,
2315   .type = VLIB_NODE_TYPE_INTERNAL,
2316
2317   .n_next_nodes = 1,
2318
2319   .next_nodes = {
2320     [0] = "error-drop",
2321   },
2322 };
2323
2324 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2325
2326 static uword
2327 snat_out2in_fast_node_fn (vlib_main_t * vm,
2328                           vlib_node_runtime_t * node,
2329                           vlib_frame_t * frame)
2330 {
2331   u32 n_left_from, * from, * to_next;
2332   snat_out2in_next_t next_index;
2333   u32 pkts_processed = 0;
2334   snat_main_t * sm = &snat_main;
2335
2336   from = vlib_frame_vector_args (frame);
2337   n_left_from = frame->n_vectors;
2338   next_index = node->cached_next_index;
2339
2340   while (n_left_from > 0)
2341     {
2342       u32 n_left_to_next;
2343
2344       vlib_get_next_frame (vm, node, next_index,
2345                            to_next, n_left_to_next);
2346
2347       while (n_left_from > 0 && n_left_to_next > 0)
2348         {
2349           u32 bi0;
2350           vlib_buffer_t * b0;
2351           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2352           u32 sw_if_index0;
2353           ip4_header_t * ip0;
2354           ip_csum_t sum0;
2355           u32 new_addr0, old_addr0;
2356           u16 new_port0, old_port0;
2357           udp_header_t * udp0;
2358           tcp_header_t * tcp0;
2359           icmp46_header_t * icmp0;
2360           snat_session_key_t key0, sm0;
2361           u32 proto0;
2362           u32 rx_fib_index0;
2363
2364           /* speculatively enqueue b0 to the current next frame */
2365           bi0 = from[0];
2366           to_next[0] = bi0;
2367           from += 1;
2368           to_next += 1;
2369           n_left_from -= 1;
2370           n_left_to_next -= 1;
2371
2372           b0 = vlib_get_buffer (vm, bi0);
2373
2374           ip0 = vlib_buffer_get_current (b0);
2375           udp0 = ip4_next_header (ip0);
2376           tcp0 = (tcp_header_t *) udp0;
2377           icmp0 = (icmp46_header_t *) udp0;
2378
2379           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2380           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2381
2382           vnet_feature_next (sw_if_index0, &next0, b0);
2383
2384           if (PREDICT_FALSE(ip0->ttl == 1))
2385             {
2386               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2387               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2388                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2389                                            0);
2390               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2391               goto trace00;
2392             }
2393
2394           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2395
2396           if (PREDICT_FALSE (proto0 == ~0))
2397               goto trace00;
2398
2399           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2400             {
2401               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2402                                   rx_fib_index0, node, next0, ~0, 0, 0);
2403               goto trace00;
2404             }
2405
2406           key0.addr = ip0->dst_address;
2407           key0.port = udp0->dst_port;
2408           key0.fib_index = rx_fib_index0;
2409
2410           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2411             {
2412               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2413               goto trace00;
2414             }
2415
2416           new_addr0 = sm0.addr.as_u32;
2417           new_port0 = sm0.port;
2418           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2419           old_addr0 = ip0->dst_address.as_u32;
2420           ip0->dst_address.as_u32 = new_addr0;
2421
2422           sum0 = ip0->checksum;
2423           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2424                                  ip4_header_t,
2425                                  dst_address /* changed member */);
2426           ip0->checksum = ip_csum_fold (sum0);
2427
2428           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2429             {
2430                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2431                 {
2432                   old_port0 = tcp0->dst_port;
2433                   tcp0->dst_port = new_port0;
2434
2435                   sum0 = tcp0->checksum;
2436                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2437                                          ip4_header_t,
2438                                          dst_address /* changed member */);
2439
2440                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2441                                          ip4_header_t /* cheat */,
2442                                          length /* changed member */);
2443                   tcp0->checksum = ip_csum_fold(sum0);
2444                 }
2445               else
2446                 {
2447                   old_port0 = udp0->dst_port;
2448                   udp0->dst_port = new_port0;
2449                   udp0->checksum = 0;
2450                 }
2451             }
2452           else
2453             {
2454               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2455                 {
2456                   sum0 = tcp0->checksum;
2457                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2458                                          ip4_header_t,
2459                                          dst_address /* changed member */);
2460
2461                   tcp0->checksum = ip_csum_fold(sum0);
2462                 }
2463             }
2464
2465         trace00:
2466
2467           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2468                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2469             {
2470               snat_out2in_trace_t *t =
2471                  vlib_add_trace (vm, node, b0, sizeof (*t));
2472               t->sw_if_index = sw_if_index0;
2473               t->next_index = next0;
2474             }
2475
2476           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2477
2478           /* verify speculative enqueue, maybe switch current next frame */
2479           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2480                                            to_next, n_left_to_next,
2481                                            bi0, next0);
2482         }
2483
2484       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2485     }
2486
2487   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2488                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2489                                pkts_processed);
2490   return frame->n_vectors;
2491 }
2492
2493 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2494   .function = snat_out2in_fast_node_fn,
2495   .name = "nat44-out2in-fast",
2496   .vector_size = sizeof (u32),
2497   .format_trace = format_snat_out2in_fast_trace,
2498   .type = VLIB_NODE_TYPE_INTERNAL,
2499
2500   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2501   .error_strings = snat_out2in_error_strings,
2502
2503   .runtime_data_bytes = sizeof (snat_runtime_t),
2504
2505   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2506
2507   /* edit / add dispositions here */
2508   .next_nodes = {
2509     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2510     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2511     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2512   },
2513 };
2514 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);