src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27 #include <snat/snat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     SNAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out SNAT session key.
 121  * @param out2in Out2in SNAT session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140
 141   user_key.addr = in2out.addr;
 142   user_key.fib_index = in2out.fib_index;
 143   kv0.key = user_key.as_u64;
 144
 145   /* Ever heard of the "user" = inside ip4 address before? */
 146   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 147     {
 148       /* no, make a new one */
 149       pool_get (sm->per_thread_data[thread_index].users, u);
 150       memset (u, 0, sizeof (*u));
 151       u->addr = in2out.addr;
 152       u->fib_index = in2out.fib_index;
 153
 154       pool_get (sm->per_thread_data[thread_index].list_pool,
 155                 per_user_list_head_elt);
 156
 157       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 158         sm->per_thread_data[thread_index].list_pool;
 159
 160       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 161                        u->sessions_per_user_list_head_index);
 162
 163       kv0.value = u - sm->per_thread_data[thread_index].users;
 164
 165       /* add user */
 166       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 167
 168       /* add non-traslated packets worker lookup */
 169       kv0.value = thread_index;
 170       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 171     }
 172   else
 173     {
 174       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 175                              value0.value);
 176     }
 177
 178   pool_get (sm->per_thread_data[thread_index].sessions, s);
 179   memset (s, 0, sizeof (*s));
 180
 181   s->outside_address_index = ~0;
 182   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 183   u->nstaticsessions++;
 184
 185   /* Create list elts */
 186   pool_get (sm->per_thread_data[thread_index].list_pool,
 187             per_user_translation_list_elt);
 188   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 189                    per_user_translation_list_elt -
 190                    sm->per_thread_data[thread_index].list_pool);
 191
 192   per_user_translation_list_elt->value =
 193     s - sm->per_thread_data[thread_index].sessions;
 194   s->per_user_index =
 195     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 196   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 197
 198   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 199                       s->per_user_list_head_index,
 200                       per_user_translation_list_elt -
 201                       sm->per_thread_data[thread_index].list_pool);
 202
 203   s->in2out = in2out;
 204   s->out2in = out2in;
 205   s->in2out.protocol = out2in.protocol;
 206
 207   /* Add to translation hashes */
 208   kv0.key = s->in2out.as_u64;
 209   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 210   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 211       clib_warning ("in2out key add failed");
 212
 213   kv0.key = s->out2in.as_u64;
 214   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 215
 216   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 217       clib_warning ("out2in key add failed");
 218
 219   /* log NAT event */
 220   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 221                                       s->out2in.addr.as_u32,
 222                                       s->in2out.protocol,
 223                                       s->in2out.port,
 224                                       s->out2in.port,
 225                                       s->in2out.fib_index);
 226    return s;
 227 }
 228
 229 static_always_inline
 230 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 231                                  snat_session_key_t *p_key0)
 232 {
 233   icmp46_header_t *icmp0;
 234   snat_session_key_t key0;
 235   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 236   ip4_header_t *inner_ip0;
 237   void *l4_header = 0;
 238   icmp46_header_t *inner_icmp0;
 239
 240   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 241   echo0 = (icmp_echo_header_t *)(icmp0+1);
 242
 243   if (!icmp_is_error_message (icmp0))
 244     {
 245       key0.protocol = SNAT_PROTOCOL_ICMP;
 246       key0.addr = ip0->dst_address;
 247       key0.port = echo0->identifier;
 248     }
 249   else
 250     {
 251       inner_ip0 = (ip4_header_t *)(echo0+1);
 252       l4_header = ip4_next_header (inner_ip0);
 253       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 254       key0.addr = inner_ip0->src_address;
 255       switch (key0.protocol)
 256         {
 257         case SNAT_PROTOCOL_ICMP:
 258           inner_icmp0 = (icmp46_header_t*)l4_header;
 259           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 260           key0.port = inner_echo0->identifier;
 261           break;
 262         case SNAT_PROTOCOL_UDP:
 263         case SNAT_PROTOCOL_TCP:
 264           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 265           break;
 266         default:
 267           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 268         }
 269     }
 270   *p_key0 = key0;
 271   return -1; /* success */
 272 }
 273
 274 /**
 275  * Get address and port values to be used for packet SNAT translation
 276  * and create session if needed
 277  *
 278  * @param[in,out] sm             SNAT main
 279  * @param[in,out] node           SNAT node runtime
 280  * @param[in] thread_index       thread index
 281  * @param[in,out] b0             buffer containing packet to be translated
 282  * @param[out] p_proto           protocol used for matching
 283  * @param[out] p_value           address and port after NAT translation
 284  * @param[out] p_dont_translate  if packet should not be translated
 285  * @param d                      optional parameter
 286  * @param e                      optional parameter
 287  */
 288 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 289                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 290                            snat_session_key_t *p_value,
 291                            u8 *p_dont_translate, void *d, void *e)
 292 {
 293   ip4_header_t *ip0;
 294   icmp46_header_t *icmp0;
 295   u32 sw_if_index0;
 296   u32 rx_fib_index0;
 297   snat_session_key_t key0;
 298   snat_session_key_t sm0;
 299   snat_session_t *s0 = 0;
 300   u8 dont_translate = 0;
 301   clib_bihash_kv_8_8_t kv0, value0;
 302   u8 is_addr_only;
 303   u32 next0 = ~0;
 304   int err;
 305
 306   ip0 = vlib_buffer_get_current (b0);
 307   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 308   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 309   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 310
 311   err = icmp_get_key (ip0, &key0);
 312   if (err != -1)
 313     {
 314       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 315       next0 = SNAT_OUT2IN_NEXT_DROP;
 316       goto out;
 317     }
 318   key0.fib_index = rx_fib_index0;
 319
 320   kv0.key = key0.as_u64;
 321
 322   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 323     {
 324       /* Try to match static mapping by external address and port,
 325          destination address and port in packet */
 326       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 327         {
 328           /* Don't NAT packet aimed at the intfc address */
 329           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 330                                               ip0->dst_address.as_u32)))
 331             {
 332               dont_translate = 1;
 333               goto out;
 334             }
 335           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 336           next0 = SNAT_OUT2IN_NEXT_DROP;
 337           goto out;
 338         }
 339
 340       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 341                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 342         {
 343           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 344           next0 = SNAT_OUT2IN_NEXT_DROP;
 345           goto out;
 346         }
 347
 348       /* Create session initiated by host from external network */
 349       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 350                                              node, thread_index);
 351
 352       if (!s0)
 353         {
 354           next0 = SNAT_OUT2IN_NEXT_DROP;
 355           goto out;
 356         }
 357     }
 358   else
 359     {
 360       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 361                         icmp0->type != ICMP4_echo_request &&
 362                         !icmp_is_error_message (icmp0)))
 363         {
 364           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 365           next0 = SNAT_OUT2IN_NEXT_DROP;
 366           goto out;
 367         }
 368
 369       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 370                               value0.value);
 371     }
 372
 373 out:
 374   *p_proto = key0.protocol;
 375   if (s0)
 376     *p_value = s0->in2out;
 377   *p_dont_translate = dont_translate;
 378   if (d)
 379     *(snat_session_t**)d = s0;
 380   return next0;
 381 }
 382
 383 /**
 384  * Get address and port values to be used for packet SNAT translation
 385  *
 386  * @param[in] sm                 SNAT main
 387  * @param[in,out] node           SNAT node runtime
 388  * @param[in] thread_index       thread index
 389  * @param[in,out] b0             buffer containing packet to be translated
 390  * @param[out] p_proto           protocol used for matching
 391  * @param[out] p_value           address and port after NAT translation
 392  * @param[out] p_dont_translate  if packet should not be translated
 393  * @param d                      optional parameter
 394  * @param e                      optional parameter
 395  */
 396 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 397                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 398                            snat_session_key_t *p_value,
 399                            u8 *p_dont_translate, void *d, void *e)
 400 {
 401   ip4_header_t *ip0;
 402   icmp46_header_t *icmp0;
 403   u32 sw_if_index0;
 404   u32 rx_fib_index0;
 405   snat_session_key_t key0;
 406   snat_session_key_t sm0;
 407   u8 dont_translate = 0;
 408   u8 is_addr_only;
 409   u32 next0 = ~0;
 410   int err;
 411
 412   ip0 = vlib_buffer_get_current (b0);
 413   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 414   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 415   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 416
 417   err = icmp_get_key (ip0, &key0);
 418   if (err != -1)
 419     {
 420       b0->error = node->errors[err];
 421       next0 = SNAT_OUT2IN_NEXT_DROP;
 422       goto out2;
 423     }
 424   key0.fib_index = rx_fib_index0;
 425
 426   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 427     {
 428       /* Don't NAT packet aimed at the intfc address */
 429       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 430         {
 431           dont_translate = 1;
 432           goto out;
 433         }
 434       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 435       next0 = SNAT_OUT2IN_NEXT_DROP;
 436       goto out;
 437     }
 438
 439   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 440                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 441                     !icmp_is_error_message (icmp0)))
 442     {
 443       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 444       next0 = SNAT_OUT2IN_NEXT_DROP;
 445       goto out;
 446     }
 447
 448 out:
 449   *p_value = sm0;
 450 out2:
 451   *p_proto = key0.protocol;
 452   *p_dont_translate = dont_translate;
 453   return next0;
 454 }
 455
 456 static inline u32 icmp_out2in (snat_main_t *sm,
 457                                vlib_buffer_t * b0,
 458                                ip4_header_t * ip0,
 459                                icmp46_header_t * icmp0,
 460                                u32 sw_if_index0,
 461                                u32 rx_fib_index0,
 462                                vlib_node_runtime_t * node,
 463                                u32 next0,
 464                                u32 thread_index,
 465                                void *d,
 466                                void *e)
 467 {
 468   snat_session_key_t sm0;
 469   u8 protocol;
 470   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 471   ip4_header_t *inner_ip0 = 0;
 472   void *l4_header = 0;
 473   icmp46_header_t *inner_icmp0;
 474   u8 dont_translate;
 475   u32 new_addr0, old_addr0;
 476   u16 old_id0, new_id0;
 477   ip_csum_t sum0;
 478   u16 checksum0;
 479   u32 next0_tmp;
 480
 481   echo0 = (icmp_echo_header_t *)(icmp0+1);
 482
 483   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 484                                        &protocol, &sm0, &dont_translate, d, e);
 485   if (next0_tmp != ~0)
 486     next0 = next0_tmp;
 487   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 488     goto out;
 489
 490   sum0 = ip_incremental_checksum (0, icmp0,
 491                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 492   checksum0 = ~ip_csum_fold (sum0);
 493   if (checksum0 != 0 && checksum0 != 0xffff)
 494     {
 495       next0 = SNAT_OUT2IN_NEXT_DROP;
 496       goto out;
 497     }
 498
 499   old_addr0 = ip0->dst_address.as_u32;
 500   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 501   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 502
 503   sum0 = ip0->checksum;
 504   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 505                          dst_address /* changed member */);
 506   ip0->checksum = ip_csum_fold (sum0);
 507
 508   if (!icmp_is_error_message (icmp0))
 509     {
 510       new_id0 = sm0.port;
 511       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 512         {
 513           old_id0 = echo0->identifier;
 514           new_id0 = sm0.port;
 515           echo0->identifier = new_id0;
 516
 517           sum0 = icmp0->checksum;
 518           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 519                                  identifier /* changed member */);
 520           icmp0->checksum = ip_csum_fold (sum0);
 521         }
 522     }
 523   else
 524     {
 525       inner_ip0 = (ip4_header_t *)(echo0+1);
 526       l4_header = ip4_next_header (inner_ip0);
 527
 528       if (!ip4_header_checksum_is_valid (inner_ip0))
 529         {
 530           next0 = SNAT_OUT2IN_NEXT_DROP;
 531           goto out;
 532         }
 533
 534       old_addr0 = inner_ip0->src_address.as_u32;
 535       inner_ip0->src_address = sm0.addr;
 536       new_addr0 = inner_ip0->src_address.as_u32;
 537
 538       sum0 = icmp0->checksum;
 539       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 540                              src_address /* changed member */);
 541       icmp0->checksum = ip_csum_fold (sum0);
 542
 543       switch (protocol)
 544         {
 545         case SNAT_PROTOCOL_ICMP:
 546           inner_icmp0 = (icmp46_header_t*)l4_header;
 547           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 548
 549           old_id0 = inner_echo0->identifier;
 550           new_id0 = sm0.port;
 551           inner_echo0->identifier = new_id0;
 552
 553           sum0 = icmp0->checksum;
 554           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 555                                  identifier);
 556           icmp0->checksum = ip_csum_fold (sum0);
 557           break;
 558         case SNAT_PROTOCOL_UDP:
 559         case SNAT_PROTOCOL_TCP:
 560           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 561           new_id0 = sm0.port;
 562           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 563
 564           sum0 = icmp0->checksum;
 565           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 566                                  src_port);
 567           icmp0->checksum = ip_csum_fold (sum0);
 568           break;
 569         default:
 570           ASSERT(0);
 571         }
 572     }
 573
 574 out:
 575   return next0;
 576 }
 577
 578
 579 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 580                                          vlib_buffer_t * b0,
 581                                          ip4_header_t * ip0,
 582                                          icmp46_header_t * icmp0,
 583                                          u32 sw_if_index0,
 584                                          u32 rx_fib_index0,
 585                                          vlib_node_runtime_t * node,
 586                                          u32 next0, f64 now,
 587                                          u32 thread_index,
 588                                          snat_session_t ** p_s0)
 589 {
 590   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 591                       next0, thread_index, p_s0, 0);
 592   snat_session_t * s0 = *p_s0;
 593   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 594     {
 595       /* Accounting */
 596       s0->last_heard = now;
 597       s0->total_pkts++;
 598       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 599       /* Per-user LRU list maintenance for dynamic translation */
 600       if (!snat_is_session_static (s0))
 601         {
 602           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 603                              s0->per_user_index);
 604           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 605                               s0->per_user_list_head_index,
 606                               s0->per_user_index);
 607         }
 608     }
 609   return next0;
 610 }
 611
 612 static uword
 613 snat_out2in_node_fn (vlib_main_t * vm,
 614                   vlib_node_runtime_t * node,
 615                   vlib_frame_t * frame)
 616 {
 617   u32 n_left_from, * from, * to_next;
 618   snat_out2in_next_t next_index;
 619   u32 pkts_processed = 0;
 620   snat_main_t * sm = &snat_main;
 621   f64 now = vlib_time_now (vm);
 622   u32 thread_index = vlib_get_thread_index ();
 623
 624   from = vlib_frame_vector_args (frame);
 625   n_left_from = frame->n_vectors;
 626   next_index = node->cached_next_index;
 627
 628   while (n_left_from > 0)
 629     {
 630       u32 n_left_to_next;
 631
 632       vlib_get_next_frame (vm, node, next_index,
 633                            to_next, n_left_to_next);
 634
 635       while (n_left_from >= 4 && n_left_to_next >= 2)
 636         {
 637           u32 bi0, bi1;
 638           vlib_buffer_t * b0, * b1;
 639           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 640           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 641           u32 sw_if_index0, sw_if_index1;
 642           ip4_header_t * ip0, *ip1;
 643           ip_csum_t sum0, sum1;
 644           u32 new_addr0, old_addr0;
 645           u16 new_port0, old_port0;
 646           u32 new_addr1, old_addr1;
 647           u16 new_port1, old_port1;
 648           udp_header_t * udp0, * udp1;
 649           tcp_header_t * tcp0, * tcp1;
 650           icmp46_header_t * icmp0, * icmp1;
 651           snat_session_key_t key0, key1, sm0, sm1;
 652           u32 rx_fib_index0, rx_fib_index1;
 653           u32 proto0, proto1;
 654           snat_session_t * s0 = 0, * s1 = 0;
 655           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 656
 657           /* Prefetch next iteration. */
 658           {
 659             vlib_buffer_t * p2, * p3;
 660
 661             p2 = vlib_get_buffer (vm, from[2]);
 662             p3 = vlib_get_buffer (vm, from[3]);
 663
 664             vlib_prefetch_buffer_header (p2, LOAD);
 665             vlib_prefetch_buffer_header (p3, LOAD);
 666
 667             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 668             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 669           }
 670
 671           /* speculatively enqueue b0 and b1 to the current next frame */
 672           to_next[0] = bi0 = from[0];
 673           to_next[1] = bi1 = from[1];
 674           from += 2;
 675           to_next += 2;
 676           n_left_from -= 2;
 677           n_left_to_next -= 2;
 678
 679           b0 = vlib_get_buffer (vm, bi0);
 680           b1 = vlib_get_buffer (vm, bi1);
 681
 682           ip0 = vlib_buffer_get_current (b0);
 683           udp0 = ip4_next_header (ip0);
 684           tcp0 = (tcp_header_t *) udp0;
 685           icmp0 = (icmp46_header_t *) udp0;
 686
 687           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 688           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 689                                    sw_if_index0);
 690
 691           if (PREDICT_FALSE(ip0->ttl == 1))
 692             {
 693               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 694               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 695                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 696                                            0);
 697               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 698               goto trace0;
 699             }
 700
 701           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 702
 703           if (PREDICT_FALSE (proto0 == ~0))
 704               goto trace0;
 705
 706           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 707             {
 708               next0 = icmp_out2in_slow_path
 709                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 710                  next0, now, thread_index, &s0);
 711               goto trace0;
 712             }
 713
 714           key0.addr = ip0->dst_address;
 715           key0.port = udp0->dst_port;
 716           key0.protocol = proto0;
 717           key0.fib_index = rx_fib_index0;
 718
 719           kv0.key = key0.as_u64;
 720
 721           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 722             {
 723               /* Try to match static mapping by external address and port,
 724                  destination address and port in packet */
 725               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 726                 {
 727                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 728                   /*
 729                    * Send DHCP packets to the ipv4 stack, or we won't
 730                    * be able to use dhcp client on the outside interface
 731                    */
 732                   if (proto0 != SNAT_PROTOCOL_UDP
 733                       || (udp0->dst_port
 734                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 735                     next0 = SNAT_OUT2IN_NEXT_DROP;
 736                   goto trace0;
 737                 }
 738
 739               /* Create session initiated by host from external network */
 740               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 741                                                      thread_index);
 742               if (!s0)
 743                 {
 744                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 745                   next0 = SNAT_OUT2IN_NEXT_DROP;
 746                   goto trace0;
 747                 }
 748             }
 749           else
 750             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 751                                     value0.value);
 752
 753           old_addr0 = ip0->dst_address.as_u32;
 754           ip0->dst_address = s0->in2out.addr;
 755           new_addr0 = ip0->dst_address.as_u32;
 756           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 757
 758           sum0 = ip0->checksum;
 759           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 760                                  ip4_header_t,
 761                                  dst_address /* changed member */);
 762           ip0->checksum = ip_csum_fold (sum0);
 763
 764           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 765             {
 766               old_port0 = tcp0->dst_port;
 767               tcp0->dst_port = s0->in2out.port;
 768               new_port0 = tcp0->dst_port;
 769
 770               sum0 = tcp0->checksum;
 771               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 772                                      ip4_header_t,
 773                                      dst_address /* changed member */);
 774
 775               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 776                                      ip4_header_t /* cheat */,
 777                                      length /* changed member */);
 778               tcp0->checksum = ip_csum_fold(sum0);
 779             }
 780           else
 781             {
 782               old_port0 = udp0->dst_port;
 783               udp0->dst_port = s0->in2out.port;
 784               udp0->checksum = 0;
 785             }
 786
 787           /* Accounting */
 788           s0->last_heard = now;
 789           s0->total_pkts++;
 790           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 791           /* Per-user LRU list maintenance for dynamic translation */
 792           if (!snat_is_session_static (s0))
 793             {
 794               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 795                                  s0->per_user_index);
 796               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 797                                   s0->per_user_list_head_index,
 798                                   s0->per_user_index);
 799             }
 800         trace0:
 801
 802           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 803                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 804             {
 805               snat_out2in_trace_t *t =
 806                  vlib_add_trace (vm, node, b0, sizeof (*t));
 807               t->sw_if_index = sw_if_index0;
 808               t->next_index = next0;
 809               t->session_index = ~0;
 810               if (s0)
 811                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 812             }
 813
 814           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 815
 816
 817           ip1 = vlib_buffer_get_current (b1);
 818           udp1 = ip4_next_header (ip1);
 819           tcp1 = (tcp_header_t *) udp1;
 820           icmp1 = (icmp46_header_t *) udp1;
 821
 822           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 823           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 824                                    sw_if_index1);
 825
 826           if (PREDICT_FALSE(ip1->ttl == 1))
 827             {
 828               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 829               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 830                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 831                                            0);
 832               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 833               goto trace1;
 834             }
 835
 836           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 837
 838           if (PREDICT_FALSE (proto1 == ~0))
 839               goto trace1;
 840
 841           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 842             {
 843               next1 = icmp_out2in_slow_path
 844                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 845                  next1, now, thread_index, &s1);
 846               goto trace1;
 847             }
 848
 849           key1.addr = ip1->dst_address;
 850           key1.port = udp1->dst_port;
 851           key1.protocol = proto1;
 852           key1.fib_index = rx_fib_index1;
 853
 854           kv1.key = key1.as_u64;
 855
 856           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 857             {
 858               /* Try to match static mapping by external address and port,
 859                  destination address and port in packet */
 860               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
 861                 {
 862                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 863                   /*
 864                    * Send DHCP packets to the ipv4 stack, or we won't
 865                    * be able to use dhcp client on the outside interface
 866                    */
 867                   if (proto1 != SNAT_PROTOCOL_UDP
 868                       || (udp1->dst_port
 869                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 870                     next1 = SNAT_OUT2IN_NEXT_DROP;
 871                   goto trace1;
 872                 }
 873
 874               /* Create session initiated by host from external network */
 875               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 876                                                      thread_index);
 877               if (!s1)
 878                 {
 879                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 880                   next1 = SNAT_OUT2IN_NEXT_DROP;
 881                   goto trace1;
 882                 }
 883             }
 884           else
 885             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 886                                     value1.value);
 887
 888           old_addr1 = ip1->dst_address.as_u32;
 889           ip1->dst_address = s1->in2out.addr;
 890           new_addr1 = ip1->dst_address.as_u32;
 891           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 892
 893           sum1 = ip1->checksum;
 894           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 895                                  ip4_header_t,
 896                                  dst_address /* changed member */);
 897           ip1->checksum = ip_csum_fold (sum1);
 898
 899           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 900             {
 901               old_port1 = tcp1->dst_port;
 902               tcp1->dst_port = s1->in2out.port;
 903               new_port1 = tcp1->dst_port;
 904
 905               sum1 = tcp1->checksum;
 906               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 907                                      ip4_header_t,
 908                                      dst_address /* changed member */);
 909
 910               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 911                                      ip4_header_t /* cheat */,
 912                                      length /* changed member */);
 913               tcp1->checksum = ip_csum_fold(sum1);
 914             }
 915           else
 916             {
 917               old_port1 = udp1->dst_port;
 918               udp1->dst_port = s1->in2out.port;
 919               udp1->checksum = 0;
 920             }
 921
 922           /* Accounting */
 923           s1->last_heard = now;
 924           s1->total_pkts++;
 925           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 926           /* Per-user LRU list maintenance for dynamic translation */
 927           if (!snat_is_session_static (s1))
 928             {
 929               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 930                                  s1->per_user_index);
 931               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 932                                   s1->per_user_list_head_index,
 933                                   s1->per_user_index);
 934             }
 935         trace1:
 936
 937           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 938                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 939             {
 940               snat_out2in_trace_t *t =
 941                  vlib_add_trace (vm, node, b1, sizeof (*t));
 942               t->sw_if_index = sw_if_index1;
 943               t->next_index = next1;
 944               t->session_index = ~0;
 945               if (s1)
 946                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
 947             }
 948
 949           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 950
 951           /* verify speculative enqueues, maybe switch current next frame */
 952           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 953                                            to_next, n_left_to_next,
 954                                            bi0, bi1, next0, next1);
 955         }
 956
 957       while (n_left_from > 0 && n_left_to_next > 0)
 958         {
 959           u32 bi0;
 960           vlib_buffer_t * b0;
 961           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 962           u32 sw_if_index0;
 963           ip4_header_t * ip0;
 964           ip_csum_t sum0;
 965           u32 new_addr0, old_addr0;
 966           u16 new_port0, old_port0;
 967           udp_header_t * udp0;
 968           tcp_header_t * tcp0;
 969           icmp46_header_t * icmp0;
 970           snat_session_key_t key0, sm0;
 971           u32 rx_fib_index0;
 972           u32 proto0;
 973           snat_session_t * s0 = 0;
 974           clib_bihash_kv_8_8_t kv0, value0;
 975
 976           /* speculatively enqueue b0 to the current next frame */
 977           bi0 = from[0];
 978           to_next[0] = bi0;
 979           from += 1;
 980           to_next += 1;
 981           n_left_from -= 1;
 982           n_left_to_next -= 1;
 983
 984           b0 = vlib_get_buffer (vm, bi0);
 985
 986           ip0 = vlib_buffer_get_current (b0);
 987           udp0 = ip4_next_header (ip0);
 988           tcp0 = (tcp_header_t *) udp0;
 989           icmp0 = (icmp46_header_t *) udp0;
 990
 991           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 992           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 993                                    sw_if_index0);
 994
 995           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 996
 997           if (PREDICT_FALSE (proto0 == ~0))
 998               goto trace00;
 999
1000           if (PREDICT_FALSE(ip0->ttl == 1))
1001             {
1002               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1003               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1004                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1005                                            0);
1006               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1007               goto trace00;
1008             }
1009
1010           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1011             {
1012               next0 = icmp_out2in_slow_path
1013                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1014                  next0, now, thread_index, &s0);
1015               goto trace00;
1016             }
1017
1018           key0.addr = ip0->dst_address;
1019           key0.port = udp0->dst_port;
1020           key0.protocol = proto0;
1021           key0.fib_index = rx_fib_index0;
1022
1023           kv0.key = key0.as_u64;
1024
1025           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1026             {
1027               /* Try to match static mapping by external address and port,
1028                  destination address and port in packet */
1029               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1030                 {
1031                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1032                   /*
1033                    * Send DHCP packets to the ipv4 stack, or we won't
1034                    * be able to use dhcp client on the outside interface
1035                    */
1036                   if (proto0 != SNAT_PROTOCOL_UDP
1037                       || (udp0->dst_port
1038                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1039
1040                     next0 = SNAT_OUT2IN_NEXT_DROP;
1041                   goto trace00;
1042                 }
1043
1044               /* Create session initiated by host from external network */
1045               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1046                                                      thread_index);
1047               if (!s0)
1048                 {
1049                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1050                     next0 = SNAT_OUT2IN_NEXT_DROP;
1051                   goto trace00;
1052                 }
1053             }
1054           else
1055             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1056                                     value0.value);
1057
1058           old_addr0 = ip0->dst_address.as_u32;
1059           ip0->dst_address = s0->in2out.addr;
1060           new_addr0 = ip0->dst_address.as_u32;
1061           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1062
1063           sum0 = ip0->checksum;
1064           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1065                                  ip4_header_t,
1066                                  dst_address /* changed member */);
1067           ip0->checksum = ip_csum_fold (sum0);
1068
1069           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1070             {
1071               old_port0 = tcp0->dst_port;
1072               tcp0->dst_port = s0->in2out.port;
1073               new_port0 = tcp0->dst_port;
1074
1075               sum0 = tcp0->checksum;
1076               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1077                                      ip4_header_t,
1078                                      dst_address /* changed member */);
1079
1080               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1081                                      ip4_header_t /* cheat */,
1082                                      length /* changed member */);
1083               tcp0->checksum = ip_csum_fold(sum0);
1084             }
1085           else
1086             {
1087               old_port0 = udp0->dst_port;
1088               udp0->dst_port = s0->in2out.port;
1089               udp0->checksum = 0;
1090             }
1091
1092           /* Accounting */
1093           s0->last_heard = now;
1094           s0->total_pkts++;
1095           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1096           /* Per-user LRU list maintenance for dynamic translation */
1097           if (!snat_is_session_static (s0))
1098             {
1099               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1100                                  s0->per_user_index);
1101               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1102                                   s0->per_user_list_head_index,
1103                                   s0->per_user_index);
1104             }
1105         trace00:
1106
1107           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1108                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1109             {
1110               snat_out2in_trace_t *t =
1111                  vlib_add_trace (vm, node, b0, sizeof (*t));
1112               t->sw_if_index = sw_if_index0;
1113               t->next_index = next0;
1114               t->session_index = ~0;
1115               if (s0)
1116                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1117             }
1118
1119           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1120
1121           /* verify speculative enqueue, maybe switch current next frame */
1122           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1123                                            to_next, n_left_to_next,
1124                                            bi0, next0);
1125         }
1126
1127       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1128     }
1129
1130   vlib_node_increment_counter (vm, snat_out2in_node.index,
1131                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1132                                pkts_processed);
1133   return frame->n_vectors;
1134 }
1135
1136 VLIB_REGISTER_NODE (snat_out2in_node) = {
1137   .function = snat_out2in_node_fn,
1138   .name = "snat-out2in",
1139   .vector_size = sizeof (u32),
1140   .format_trace = format_snat_out2in_trace,
1141   .type = VLIB_NODE_TYPE_INTERNAL,
1142
1143   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1144   .error_strings = snat_out2in_error_strings,
1145
1146   .runtime_data_bytes = sizeof (snat_runtime_t),
1147
1148   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1149
1150   /* edit / add dispositions here */
1151   .next_nodes = {
1152     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1153     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1154     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1155   },
1156 };
1157 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1158
1159 /**************************/
1160 /*** deterministic mode ***/
1161 /**************************/
1162 static uword
1163 snat_det_out2in_node_fn (vlib_main_t * vm,
1164                          vlib_node_runtime_t * node,
1165                          vlib_frame_t * frame)
1166 {
1167   u32 n_left_from, * from, * to_next;
1168   snat_out2in_next_t next_index;
1169   u32 pkts_processed = 0;
1170   snat_main_t * sm = &snat_main;
1171   u32 thread_index = os_get_cpu_number ();
1172
1173   from = vlib_frame_vector_args (frame);
1174   n_left_from = frame->n_vectors;
1175   next_index = node->cached_next_index;
1176
1177   while (n_left_from > 0)
1178     {
1179       u32 n_left_to_next;
1180
1181       vlib_get_next_frame (vm, node, next_index,
1182                            to_next, n_left_to_next);
1183
1184       while (n_left_from >= 4 && n_left_to_next >= 2)
1185         {
1186           u32 bi0, bi1;
1187           vlib_buffer_t * b0, * b1;
1188           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1189           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1190           u32 sw_if_index0, sw_if_index1;
1191           ip4_header_t * ip0, * ip1;
1192           ip_csum_t sum0, sum1;
1193           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1194           u16 new_port0, old_port0, old_port1, new_port1;
1195           udp_header_t * udp0, * udp1;
1196           tcp_header_t * tcp0, * tcp1;
1197           u32 proto0, proto1;
1198           snat_det_out_key_t key0, key1;
1199           snat_det_map_t * dm0, * dm1;
1200           snat_det_session_t * ses0 = 0, * ses1 = 0;
1201           u32 rx_fib_index0, rx_fib_index1;
1202           icmp46_header_t * icmp0, * icmp1;
1203
1204           /* Prefetch next iteration. */
1205           {
1206             vlib_buffer_t * p2, * p3;
1207
1208             p2 = vlib_get_buffer (vm, from[2]);
1209             p3 = vlib_get_buffer (vm, from[3]);
1210
1211             vlib_prefetch_buffer_header (p2, LOAD);
1212             vlib_prefetch_buffer_header (p3, LOAD);
1213
1214             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1215             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1216           }
1217
1218           /* speculatively enqueue b0 and b1 to the current next frame */
1219           to_next[0] = bi0 = from[0];
1220           to_next[1] = bi1 = from[1];
1221           from += 2;
1222           to_next += 2;
1223           n_left_from -= 2;
1224           n_left_to_next -= 2;
1225
1226           b0 = vlib_get_buffer (vm, bi0);
1227           b1 = vlib_get_buffer (vm, bi1);
1228
1229           ip0 = vlib_buffer_get_current (b0);
1230           udp0 = ip4_next_header (ip0);
1231           tcp0 = (tcp_header_t *) udp0;
1232
1233           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1234
1235           if (PREDICT_FALSE(ip0->ttl == 1))
1236             {
1237               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1238               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1239                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1240                                            0);
1241               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1242               goto trace0;
1243             }
1244
1245           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1246
1247           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1248             {
1249               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1250               icmp0 = (icmp46_header_t *) udp0;
1251
1252               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1253                                   rx_fib_index0, node, next0, thread_index,
1254                                   &ses0, &dm0);
1255               goto trace0;
1256             }
1257
1258           key0.ext_host_addr = ip0->src_address;
1259           key0.ext_host_port = tcp0->src;
1260           key0.out_port = tcp0->dst;
1261
1262           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1263           if (PREDICT_FALSE(!dm0))
1264             {
1265               clib_warning("unknown dst address:  %U",
1266                            format_ip4_address, &ip0->dst_address);
1267               next0 = SNAT_OUT2IN_NEXT_DROP;
1268               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1269               goto trace0;
1270             }
1271
1272           snat_det_reverse(dm0, &ip0->dst_address,
1273                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1274
1275           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1276           if (PREDICT_FALSE(!ses0))
1277             {
1278               clib_warning("no match src %U:%d dst %U:%d for user %U",
1279                            format_ip4_address, &ip0->src_address,
1280                            clib_net_to_host_u16 (tcp0->src),
1281                            format_ip4_address, &ip0->dst_address,
1282                            clib_net_to_host_u16 (tcp0->dst),
1283                            format_ip4_address, &new_addr0);
1284               next0 = SNAT_OUT2IN_NEXT_DROP;
1285               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1286               goto trace0;
1287             }
1288           new_port0 = ses0->in_port;
1289
1290           old_addr0 = ip0->dst_address;
1291           ip0->dst_address = new_addr0;
1292           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1293
1294           sum0 = ip0->checksum;
1295           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1296                                  ip4_header_t,
1297                                  dst_address /* changed member */);
1298           ip0->checksum = ip_csum_fold (sum0);
1299
1300           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1301             {
1302               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1303                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1304               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1305                 snat_det_ses_close(dm0, ses0);
1306
1307               old_port0 = tcp0->dst;
1308               tcp0->dst = new_port0;
1309
1310               sum0 = tcp0->checksum;
1311               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1312                                      ip4_header_t,
1313                                      dst_address /* changed member */);
1314
1315               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1316                                      ip4_header_t /* cheat */,
1317                                      length /* changed member */);
1318               tcp0->checksum = ip_csum_fold(sum0);
1319             }
1320           else
1321             {
1322               old_port0 = udp0->dst_port;
1323               udp0->dst_port = new_port0;
1324               udp0->checksum = 0;
1325             }
1326
1327         trace0:
1328
1329           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1330                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1331             {
1332               snat_out2in_trace_t *t =
1333                  vlib_add_trace (vm, node, b0, sizeof (*t));
1334               t->sw_if_index = sw_if_index0;
1335               t->next_index = next0;
1336               t->session_index = ~0;
1337               if (ses0)
1338                 t->session_index = ses0 - dm0->sessions;
1339             }
1340
1341           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1342
1343           b1 = vlib_get_buffer (vm, bi1);
1344
1345           ip1 = vlib_buffer_get_current (b1);
1346           udp1 = ip4_next_header (ip1);
1347           tcp1 = (tcp_header_t *) udp1;
1348
1349           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1350
1351           if (PREDICT_FALSE(ip1->ttl == 1))
1352             {
1353               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1354               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1355                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1356                                            0);
1357               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1358               goto trace1;
1359             }
1360
1361           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1362
1363           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1364             {
1365               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1366               icmp1 = (icmp46_header_t *) udp1;
1367
1368               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1369                                   rx_fib_index1, node, next1, thread_index,
1370                                   &ses1, &dm1);
1371               goto trace1;
1372             }
1373
1374           key1.ext_host_addr = ip1->src_address;
1375           key1.ext_host_port = tcp1->src;
1376           key1.out_port = tcp1->dst;
1377
1378           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1379           if (PREDICT_FALSE(!dm1))
1380             {
1381               clib_warning("unknown dst address:  %U",
1382                            format_ip4_address, &ip1->dst_address);
1383               next1 = SNAT_OUT2IN_NEXT_DROP;
1384               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1385               goto trace1;
1386             }
1387
1388           snat_det_reverse(dm1, &ip1->dst_address,
1389                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1390
1391           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1392           if (PREDICT_FALSE(!ses1))
1393             {
1394               clib_warning("no match src %U:%d dst %U:%d for user %U",
1395                            format_ip4_address, &ip1->src_address,
1396                            clib_net_to_host_u16 (tcp1->src),
1397                            format_ip4_address, &ip1->dst_address,
1398                            clib_net_to_host_u16 (tcp1->dst),
1399                            format_ip4_address, &new_addr1);
1400               next1 = SNAT_OUT2IN_NEXT_DROP;
1401               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1402               goto trace1;
1403             }
1404           new_port1 = ses1->in_port;
1405
1406           old_addr1 = ip1->dst_address;
1407           ip1->dst_address = new_addr1;
1408           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1409
1410           sum1 = ip1->checksum;
1411           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1412                                  ip4_header_t,
1413                                  dst_address /* changed member */);
1414           ip1->checksum = ip_csum_fold (sum1);
1415
1416           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1417             {
1418               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1419                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1420               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1421                 snat_det_ses_close(dm1, ses1);
1422
1423               old_port1 = tcp1->dst;
1424               tcp1->dst = new_port1;
1425
1426               sum1 = tcp1->checksum;
1427               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1428                                      ip4_header_t,
1429                                      dst_address /* changed member */);
1430
1431               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1432                                      ip4_header_t /* cheat */,
1433                                      length /* changed member */);
1434               tcp1->checksum = ip_csum_fold(sum1);
1435             }
1436           else
1437             {
1438               old_port1 = udp1->dst_port;
1439               udp1->dst_port = new_port1;
1440               udp1->checksum = 0;
1441             }
1442
1443         trace1:
1444
1445           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1446                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1447             {
1448               snat_out2in_trace_t *t =
1449                  vlib_add_trace (vm, node, b1, sizeof (*t));
1450               t->sw_if_index = sw_if_index1;
1451               t->next_index = next1;
1452               t->session_index = ~0;
1453               if (ses1)
1454                 t->session_index = ses1 - dm1->sessions;
1455             }
1456
1457           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1458
1459           /* verify speculative enqueues, maybe switch current next frame */
1460           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1461                                            to_next, n_left_to_next,
1462                                            bi0, bi1, next0, next1);
1463          }
1464
1465       while (n_left_from > 0 && n_left_to_next > 0)
1466         {
1467           u32 bi0;
1468           vlib_buffer_t * b0;
1469           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1470           u32 sw_if_index0;
1471           ip4_header_t * ip0;
1472           ip_csum_t sum0;
1473           ip4_address_t new_addr0, old_addr0;
1474           u16 new_port0, old_port0;
1475           udp_header_t * udp0;
1476           tcp_header_t * tcp0;
1477           u32 proto0;
1478           snat_det_out_key_t key0;
1479           snat_det_map_t * dm0;
1480           snat_det_session_t * ses0 = 0;
1481           u32 rx_fib_index0;
1482           icmp46_header_t * icmp0;
1483
1484           /* speculatively enqueue b0 to the current next frame */
1485           bi0 = from[0];
1486           to_next[0] = bi0;
1487           from += 1;
1488           to_next += 1;
1489           n_left_from -= 1;
1490           n_left_to_next -= 1;
1491
1492           b0 = vlib_get_buffer (vm, bi0);
1493
1494           ip0 = vlib_buffer_get_current (b0);
1495           udp0 = ip4_next_header (ip0);
1496           tcp0 = (tcp_header_t *) udp0;
1497
1498           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1499
1500           if (PREDICT_FALSE(ip0->ttl == 1))
1501             {
1502               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1503               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1504                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1505                                            0);
1506               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1507               goto trace00;
1508             }
1509
1510           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1511
1512           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1513             {
1514               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1515               icmp0 = (icmp46_header_t *) udp0;
1516
1517               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1518                                   rx_fib_index0, node, next0, thread_index,
1519                                   &ses0, &dm0);
1520               goto trace00;
1521             }
1522
1523           key0.ext_host_addr = ip0->src_address;
1524           key0.ext_host_port = tcp0->src;
1525           key0.out_port = tcp0->dst;
1526
1527           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1528           if (PREDICT_FALSE(!dm0))
1529             {
1530               clib_warning("unknown dst address:  %U",
1531                            format_ip4_address, &ip0->dst_address);
1532               next0 = SNAT_OUT2IN_NEXT_DROP;
1533               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1534               goto trace00;
1535             }
1536
1537           snat_det_reverse(dm0, &ip0->dst_address,
1538                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1539
1540           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1541           if (PREDICT_FALSE(!ses0))
1542             {
1543               clib_warning("no match src %U:%d dst %U:%d for user %U",
1544                            format_ip4_address, &ip0->src_address,
1545                            clib_net_to_host_u16 (tcp0->src),
1546                            format_ip4_address, &ip0->dst_address,
1547                            clib_net_to_host_u16 (tcp0->dst),
1548                            format_ip4_address, &new_addr0);
1549               next0 = SNAT_OUT2IN_NEXT_DROP;
1550               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1551               goto trace00;
1552             }
1553           new_port0 = ses0->in_port;
1554
1555           old_addr0 = ip0->dst_address;
1556           ip0->dst_address = new_addr0;
1557           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1558
1559           sum0 = ip0->checksum;
1560           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1561                                  ip4_header_t,
1562                                  dst_address /* changed member */);
1563           ip0->checksum = ip_csum_fold (sum0);
1564
1565           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1566             {
1567               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1568                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1569               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1570                 snat_det_ses_close(dm0, ses0);
1571
1572               old_port0 = tcp0->dst;
1573               tcp0->dst = new_port0;
1574
1575               sum0 = tcp0->checksum;
1576               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1577                                      ip4_header_t,
1578                                      dst_address /* changed member */);
1579
1580               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1581                                      ip4_header_t /* cheat */,
1582                                      length /* changed member */);
1583               tcp0->checksum = ip_csum_fold(sum0);
1584             }
1585           else
1586             {
1587               old_port0 = udp0->dst_port;
1588               udp0->dst_port = new_port0;
1589               udp0->checksum = 0;
1590             }
1591
1592         trace00:
1593
1594           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1595                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1596             {
1597               snat_out2in_trace_t *t =
1598                  vlib_add_trace (vm, node, b0, sizeof (*t));
1599               t->sw_if_index = sw_if_index0;
1600               t->next_index = next0;
1601               t->session_index = ~0;
1602               if (ses0)
1603                 t->session_index = ses0 - dm0->sessions;
1604             }
1605
1606           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1607
1608           /* verify speculative enqueue, maybe switch current next frame */
1609           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1610                                            to_next, n_left_to_next,
1611                                            bi0, next0);
1612         }
1613
1614       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1615     }
1616
1617   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1618                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1619                                pkts_processed);
1620   return frame->n_vectors;
1621 }
1622
1623 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1624   .function = snat_det_out2in_node_fn,
1625   .name = "snat-det-out2in",
1626   .vector_size = sizeof (u32),
1627   .format_trace = format_snat_out2in_trace,
1628   .type = VLIB_NODE_TYPE_INTERNAL,
1629
1630   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1631   .error_strings = snat_out2in_error_strings,
1632
1633   .runtime_data_bytes = sizeof (snat_runtime_t),
1634
1635   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1636
1637   /* edit / add dispositions here */
1638   .next_nodes = {
1639     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1640     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1641     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1642   },
1643 };
1644 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1645
1646 /**
1647  * Get address and port values to be used for packet SNAT translation
1648  * and create session if needed
1649  *
1650  * @param[in,out] sm             SNAT main
1651  * @param[in,out] node           SNAT node runtime
1652  * @param[in] thread_index       thread index
1653  * @param[in,out] b0             buffer containing packet to be translated
1654  * @param[out] p_proto           protocol used for matching
1655  * @param[out] p_value           address and port after NAT translation
1656  * @param[out] p_dont_translate  if packet should not be translated
1657  * @param d                      optional parameter
1658  * @param e                      optional parameter
1659  */
1660 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
1661                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
1662                           snat_session_key_t *p_value,
1663                           u8 *p_dont_translate, void *d, void *e)
1664 {
1665   ip4_header_t *ip0;
1666   icmp46_header_t *icmp0;
1667   u32 sw_if_index0;
1668   u8 protocol;
1669   snat_det_out_key_t key0;
1670   u8 dont_translate = 0;
1671   u32 next0 = ~0;
1672   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1673   ip4_header_t *inner_ip0;
1674   void *l4_header = 0;
1675   icmp46_header_t *inner_icmp0;
1676   snat_det_map_t * dm0 = 0;
1677   ip4_address_t new_addr0 = {{0}};
1678   snat_det_session_t * ses0 = 0;
1679   ip4_address_t out_addr;
1680
1681   ip0 = vlib_buffer_get_current (b0);
1682   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1683   echo0 = (icmp_echo_header_t *)(icmp0+1);
1684   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1685
1686   if (!icmp_is_error_message (icmp0))
1687     {
1688       protocol = SNAT_PROTOCOL_ICMP;
1689       key0.ext_host_addr = ip0->src_address;
1690       key0.ext_host_port = 0;
1691       key0.out_port = echo0->identifier;
1692       out_addr = ip0->dst_address;
1693     }
1694   else
1695     {
1696       inner_ip0 = (ip4_header_t *)(echo0+1);
1697       l4_header = ip4_next_header (inner_ip0);
1698       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
1699       key0.ext_host_addr = inner_ip0->dst_address;
1700       out_addr = inner_ip0->src_address;
1701       switch (protocol)
1702         {
1703         case SNAT_PROTOCOL_ICMP:
1704           inner_icmp0 = (icmp46_header_t*)l4_header;
1705           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1706           key0.ext_host_port = 0;
1707           key0.out_port = inner_echo0->identifier;
1708           break;
1709         case SNAT_PROTOCOL_UDP:
1710         case SNAT_PROTOCOL_TCP:
1711           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1712           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
1713           break;
1714         default:
1715           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1716           next0 = SNAT_OUT2IN_NEXT_DROP;
1717           goto out;
1718         }
1719     }
1720
1721   dm0 = snat_det_map_by_out(sm, &out_addr);
1722   if (PREDICT_FALSE(!dm0))
1723     {
1724       /* Don't NAT packet aimed at the intfc address */
1725       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1726                                           ip0->dst_address.as_u32)))
1727         {
1728           dont_translate = 1;
1729           goto out;
1730         }
1731       clib_warning("unknown dst address:  %U",
1732                    format_ip4_address, &ip0->dst_address);
1733       goto out;
1734     }
1735
1736   snat_det_reverse(dm0, &ip0->dst_address,
1737                    clib_net_to_host_u16(key0.out_port), &new_addr0);
1738
1739   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1740   if (PREDICT_FALSE(!ses0))
1741     {
1742       /* Don't NAT packet aimed at the intfc address */
1743       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1744                                           ip0->dst_address.as_u32)))
1745         {
1746           dont_translate = 1;
1747           goto out;
1748         }
1749       clib_warning("no match src %U:%d dst %U:%d for user %U",
1750                    format_ip4_address, &key0.ext_host_addr,
1751                    clib_net_to_host_u16 (key0.ext_host_port),
1752                    format_ip4_address, &out_addr,
1753                    clib_net_to_host_u16 (key0.out_port),
1754                    format_ip4_address, &new_addr0);
1755       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1756       next0 = SNAT_OUT2IN_NEXT_DROP;
1757       goto out;
1758     }
1759
1760   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
1761                     !icmp_is_error_message (icmp0)))
1762     {
1763       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1764       next0 = SNAT_OUT2IN_NEXT_DROP;
1765       goto out;
1766     }
1767
1768   goto out;
1769
1770 out:
1771   *p_proto = protocol;
1772   if (ses0)
1773     {
1774       p_value->addr = new_addr0;
1775       p_value->fib_index = sm->inside_fib_index;
1776       p_value->port = ses0->in_port;
1777     }
1778   *p_dont_translate = dont_translate;
1779   if (d)
1780     *(snat_det_session_t**)d = ses0;
1781   if (e)
1782     *(snat_det_map_t**)e = dm0;
1783   return next0;
1784 }
1785
1786 /**********************/
1787 /*** worker handoff ***/
1788 /**********************/
1789 static uword
1790 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1791                                vlib_node_runtime_t * node,
1792                                vlib_frame_t * frame)
1793 {
1794   snat_main_t *sm = &snat_main;
1795   vlib_thread_main_t *tm = vlib_get_thread_main ();
1796   u32 n_left_from, *from, *to_next = 0;
1797   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1798   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1799     = 0;
1800   vlib_frame_queue_elt_t *hf = 0;
1801   vlib_frame_t *f = 0;
1802   int i;
1803   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1804   u32 next_worker_index = 0;
1805   u32 current_worker_index = ~0;
1806   u32 thread_index = vlib_get_thread_index ();
1807
1808   ASSERT (vec_len (sm->workers));
1809
1810   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1811     {
1812       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1813
1814       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1815                                sm->first_worker_index + sm->num_workers - 1,
1816                                (vlib_frame_queue_t *) (~0));
1817     }
1818
1819   from = vlib_frame_vector_args (frame);
1820   n_left_from = frame->n_vectors;
1821
1822   while (n_left_from > 0)
1823     {
1824       u32 bi0;
1825       vlib_buffer_t *b0;
1826       u32 sw_if_index0;
1827       u32 rx_fib_index0;
1828       ip4_header_t * ip0;
1829       u8 do_handoff;
1830
1831       bi0 = from[0];
1832       from += 1;
1833       n_left_from -= 1;
1834
1835       b0 = vlib_get_buffer (vm, bi0);
1836
1837       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1838       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1839
1840       ip0 = vlib_buffer_get_current (b0);
1841
1842       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1843
1844       if (PREDICT_FALSE (next_worker_index != thread_index))
1845         {
1846           do_handoff = 1;
1847
1848           if (next_worker_index != current_worker_index)
1849             {
1850               if (hf)
1851                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1852
1853               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1854                                                       next_worker_index,
1855                                                       handoff_queue_elt_by_worker_index);
1856
1857               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1858               to_next_worker = &hf->buffer_index[hf->n_vectors];
1859               current_worker_index = next_worker_index;
1860             }
1861
1862           /* enqueue to correct worker thread */
1863           to_next_worker[0] = bi0;
1864           to_next_worker++;
1865           n_left_to_next_worker--;
1866
1867           if (n_left_to_next_worker == 0)
1868             {
1869               hf->n_vectors = VLIB_FRAME_SIZE;
1870               vlib_put_frame_queue_elt (hf);
1871               current_worker_index = ~0;
1872               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1873               hf = 0;
1874             }
1875         }
1876       else
1877         {
1878           do_handoff = 0;
1879           /* if this is 1st frame */
1880           if (!f)
1881             {
1882               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1883               to_next = vlib_frame_vector_args (f);
1884             }
1885
1886           to_next[0] = bi0;
1887           to_next += 1;
1888           f->n_vectors++;
1889         }
1890
1891       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1892                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1893         {
1894           snat_out2in_worker_handoff_trace_t *t =
1895             vlib_add_trace (vm, node, b0, sizeof (*t));
1896           t->next_worker_index = next_worker_index;
1897           t->do_handoff = do_handoff;
1898         }
1899     }
1900
1901   if (f)
1902     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1903
1904   if (hf)
1905     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1906
1907   /* Ship frames to the worker nodes */
1908   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1909     {
1910       if (handoff_queue_elt_by_worker_index[i])
1911         {
1912           hf = handoff_queue_elt_by_worker_index[i];
1913           /*
1914            * It works better to let the handoff node
1915            * rate-adapt, always ship the handoff queue element.
1916            */
1917           if (1 || hf->n_vectors == hf->last_n_vectors)
1918             {
1919               vlib_put_frame_queue_elt (hf);
1920               handoff_queue_elt_by_worker_index[i] = 0;
1921             }
1922           else
1923             hf->last_n_vectors = hf->n_vectors;
1924         }
1925       congested_handoff_queue_by_worker_index[i] =
1926         (vlib_frame_queue_t *) (~0);
1927     }
1928   hf = 0;
1929   current_worker_index = ~0;
1930   return frame->n_vectors;
1931 }
1932
1933 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1934   .function = snat_out2in_worker_handoff_fn,
1935   .name = "snat-out2in-worker-handoff",
1936   .vector_size = sizeof (u32),
1937   .format_trace = format_snat_out2in_worker_handoff_trace,
1938   .type = VLIB_NODE_TYPE_INTERNAL,
1939
1940   .n_next_nodes = 1,
1941
1942   .next_nodes = {
1943     [0] = "error-drop",
1944   },
1945 };
1946
1947 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1948
1949 static uword
1950 snat_out2in_fast_node_fn (vlib_main_t * vm,
1951                           vlib_node_runtime_t * node,
1952                           vlib_frame_t * frame)
1953 {
1954   u32 n_left_from, * from, * to_next;
1955   snat_out2in_next_t next_index;
1956   u32 pkts_processed = 0;
1957   snat_main_t * sm = &snat_main;
1958
1959   from = vlib_frame_vector_args (frame);
1960   n_left_from = frame->n_vectors;
1961   next_index = node->cached_next_index;
1962
1963   while (n_left_from > 0)
1964     {
1965       u32 n_left_to_next;
1966
1967       vlib_get_next_frame (vm, node, next_index,
1968                            to_next, n_left_to_next);
1969
1970       while (n_left_from > 0 && n_left_to_next > 0)
1971         {
1972           u32 bi0;
1973           vlib_buffer_t * b0;
1974           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1975           u32 sw_if_index0;
1976           ip4_header_t * ip0;
1977           ip_csum_t sum0;
1978           u32 new_addr0, old_addr0;
1979           u16 new_port0, old_port0;
1980           udp_header_t * udp0;
1981           tcp_header_t * tcp0;
1982           icmp46_header_t * icmp0;
1983           snat_session_key_t key0, sm0;
1984           u32 proto0;
1985           u32 rx_fib_index0;
1986
1987           /* speculatively enqueue b0 to the current next frame */
1988           bi0 = from[0];
1989           to_next[0] = bi0;
1990           from += 1;
1991           to_next += 1;
1992           n_left_from -= 1;
1993           n_left_to_next -= 1;
1994
1995           b0 = vlib_get_buffer (vm, bi0);
1996
1997           ip0 = vlib_buffer_get_current (b0);
1998           udp0 = ip4_next_header (ip0);
1999           tcp0 = (tcp_header_t *) udp0;
2000           icmp0 = (icmp46_header_t *) udp0;
2001
2002           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2003           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2004
2005           vnet_feature_next (sw_if_index0, &next0, b0);
2006
2007           if (PREDICT_FALSE(ip0->ttl == 1))
2008             {
2009               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2010               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2011                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2012                                            0);
2013               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2014               goto trace00;
2015             }
2016
2017           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2018
2019           if (PREDICT_FALSE (proto0 == ~0))
2020               goto trace00;
2021
2022           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2023             {
2024               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2025                                   rx_fib_index0, node, next0, ~0, 0, 0);
2026               goto trace00;
2027             }
2028
2029           key0.addr = ip0->dst_address;
2030           key0.port = udp0->dst_port;
2031           key0.fib_index = rx_fib_index0;
2032
2033           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2034             {
2035               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2036               goto trace00;
2037             }
2038
2039           new_addr0 = sm0.addr.as_u32;
2040           new_port0 = sm0.port;
2041           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2042           old_addr0 = ip0->dst_address.as_u32;
2043           ip0->dst_address.as_u32 = new_addr0;
2044
2045           sum0 = ip0->checksum;
2046           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2047                                  ip4_header_t,
2048                                  dst_address /* changed member */);
2049           ip0->checksum = ip_csum_fold (sum0);
2050
2051           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2052             {
2053                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2054                 {
2055                   old_port0 = tcp0->dst_port;
2056                   tcp0->dst_port = new_port0;
2057
2058                   sum0 = tcp0->checksum;
2059                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2060                                          ip4_header_t,
2061                                          dst_address /* changed member */);
2062
2063                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2064                                          ip4_header_t /* cheat */,
2065                                          length /* changed member */);
2066                   tcp0->checksum = ip_csum_fold(sum0);
2067                 }
2068               else
2069                 {
2070                   old_port0 = udp0->dst_port;
2071                   udp0->dst_port = new_port0;
2072                   udp0->checksum = 0;
2073                 }
2074             }
2075           else
2076             {
2077               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2078                 {
2079                   sum0 = tcp0->checksum;
2080                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2081                                          ip4_header_t,
2082                                          dst_address /* changed member */);
2083
2084                   tcp0->checksum = ip_csum_fold(sum0);
2085                 }
2086             }
2087
2088         trace00:
2089
2090           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2091                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2092             {
2093               snat_out2in_trace_t *t =
2094                  vlib_add_trace (vm, node, b0, sizeof (*t));
2095               t->sw_if_index = sw_if_index0;
2096               t->next_index = next0;
2097             }
2098
2099           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2100
2101           /* verify speculative enqueue, maybe switch current next frame */
2102           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2103                                            to_next, n_left_to_next,
2104                                            bi0, next0);
2105         }
2106
2107       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2108     }
2109
2110   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2111                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2112                                pkts_processed);
2113   return frame->n_vectors;
2114 }
2115
2116 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2117   .function = snat_out2in_fast_node_fn,
2118   .name = "snat-out2in-fast",
2119   .vector_size = sizeof (u32),
2120   .format_trace = format_snat_out2in_fast_trace,
2121   .type = VLIB_NODE_TYPE_INTERNAL,
2122
2123   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2124   .error_strings = snat_out2in_error_strings,
2125
2126   .runtime_data_bytes = sizeof (snat_runtime_t),
2127
2128   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2129
2130   /* edit / add dispositions here */
2131   .next_nodes = {
2132     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2133     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2134     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2135   },
2136 };
2137 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);