src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27 #include <snat/snat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     SNAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out SNAT session key.
 121  * @param out2in Out2in SNAT session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140
 141   user_key.addr = in2out.addr;
 142   user_key.fib_index = in2out.fib_index;
 143   kv0.key = user_key.as_u64;
 144
 145   /* Ever heard of the "user" = inside ip4 address before? */
 146   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 147     {
 148       /* no, make a new one */
 149       pool_get (sm->per_thread_data[thread_index].users, u);
 150       memset (u, 0, sizeof (*u));
 151       u->addr = in2out.addr;
 152       u->fib_index = in2out.fib_index;
 153
 154       pool_get (sm->per_thread_data[thread_index].list_pool,
 155                 per_user_list_head_elt);
 156
 157       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 158         sm->per_thread_data[thread_index].list_pool;
 159
 160       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 161                        u->sessions_per_user_list_head_index);
 162
 163       kv0.value = u - sm->per_thread_data[thread_index].users;
 164
 165       /* add user */
 166       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 167
 168       /* add non-traslated packets worker lookup */
 169       kv0.value = thread_index;
 170       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 171     }
 172   else
 173     {
 174       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 175                              value0.value);
 176     }
 177
 178   pool_get (sm->per_thread_data[thread_index].sessions, s);
 179   memset (s, 0, sizeof (*s));
 180
 181   s->outside_address_index = ~0;
 182   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 183   u->nstaticsessions++;
 184
 185   /* Create list elts */
 186   pool_get (sm->per_thread_data[thread_index].list_pool,
 187             per_user_translation_list_elt);
 188   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 189                    per_user_translation_list_elt -
 190                    sm->per_thread_data[thread_index].list_pool);
 191
 192   per_user_translation_list_elt->value =
 193     s - sm->per_thread_data[thread_index].sessions;
 194   s->per_user_index =
 195     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 196   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 197
 198   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 199                       s->per_user_list_head_index,
 200                       per_user_translation_list_elt -
 201                       sm->per_thread_data[thread_index].list_pool);
 202
 203   s->in2out = in2out;
 204   s->out2in = out2in;
 205   s->in2out.protocol = out2in.protocol;
 206
 207   /* Add to translation hashes */
 208   kv0.key = s->in2out.as_u64;
 209   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 210   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 211       clib_warning ("in2out key add failed");
 212
 213   kv0.key = s->out2in.as_u64;
 214   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 215
 216   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 217       clib_warning ("out2in key add failed");
 218
 219   /* log NAT event */
 220   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 221                                       s->out2in.addr.as_u32,
 222                                       s->in2out.protocol,
 223                                       s->in2out.port,
 224                                       s->out2in.port,
 225                                       s->in2out.fib_index);
 226    return s;
 227 }
 228
 229 static_always_inline
 230 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 231                                  snat_session_key_t *p_key0)
 232 {
 233   icmp46_header_t *icmp0;
 234   snat_session_key_t key0;
 235   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 236   ip4_header_t *inner_ip0;
 237   void *l4_header = 0;
 238   icmp46_header_t *inner_icmp0;
 239
 240   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 241   echo0 = (icmp_echo_header_t *)(icmp0+1);
 242
 243   if (!icmp_is_error_message (icmp0))
 244     {
 245       key0.protocol = SNAT_PROTOCOL_ICMP;
 246       key0.addr = ip0->dst_address;
 247       key0.port = echo0->identifier;
 248     }
 249   else
 250     {
 251       inner_ip0 = (ip4_header_t *)(echo0+1);
 252       l4_header = ip4_next_header (inner_ip0);
 253       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 254       key0.addr = inner_ip0->src_address;
 255       switch (key0.protocol)
 256         {
 257         case SNAT_PROTOCOL_ICMP:
 258           inner_icmp0 = (icmp46_header_t*)l4_header;
 259           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 260           key0.port = inner_echo0->identifier;
 261           break;
 262         case SNAT_PROTOCOL_UDP:
 263         case SNAT_PROTOCOL_TCP:
 264           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 265           break;
 266         default:
 267           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 268         }
 269     }
 270   *p_key0 = key0;
 271   return -1; /* success */
 272 }
 273
 274 /**
 275  * Get address and port values to be used for packet SNAT translation
 276  * and create session if needed
 277  *
 278  * @param[in,out] sm             SNAT main
 279  * @param[in,out] node           SNAT node runtime
 280  * @param[in] thread_index       thread index
 281  * @param[in,out] b0             buffer containing packet to be translated
 282  * @param[out] p_proto           protocol used for matching
 283  * @param[out] p_value           address and port after NAT translation
 284  * @param[out] p_dont_translate  if packet should not be translated
 285  * @param d                      optional parameter
 286  * @param e                      optional parameter
 287  */
 288 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 289                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 290                            snat_session_key_t *p_value,
 291                            u8 *p_dont_translate, void *d, void *e)
 292 {
 293   ip4_header_t *ip0;
 294   icmp46_header_t *icmp0;
 295   u32 sw_if_index0;
 296   u32 rx_fib_index0;
 297   snat_session_key_t key0;
 298   snat_session_key_t sm0;
 299   snat_session_t *s0 = 0;
 300   u8 dont_translate = 0;
 301   clib_bihash_kv_8_8_t kv0, value0;
 302   u8 is_addr_only;
 303   u32 next0 = ~0;
 304   int err;
 305
 306   ip0 = vlib_buffer_get_current (b0);
 307   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 308   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 309   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 310
 311   key0.protocol = 0;
 312
 313   err = icmp_get_key (ip0, &key0);
 314   if (err != -1)
 315     {
 316       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 317       next0 = SNAT_OUT2IN_NEXT_DROP;
 318       goto out;
 319     }
 320   key0.fib_index = rx_fib_index0;
 321
 322   kv0.key = key0.as_u64;
 323
 324   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 325     {
 326       /* Try to match static mapping by external address and port,
 327          destination address and port in packet */
 328       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 329         {
 330           /* Don't NAT packet aimed at the intfc address */
 331           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 332                                               ip0->dst_address.as_u32)))
 333             {
 334               dont_translate = 1;
 335               goto out;
 336             }
 337           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 338           next0 = SNAT_OUT2IN_NEXT_DROP;
 339           goto out;
 340         }
 341
 342       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 343                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 344         {
 345           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 346           next0 = SNAT_OUT2IN_NEXT_DROP;
 347           goto out;
 348         }
 349
 350       /* Create session initiated by host from external network */
 351       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 352                                              node, thread_index);
 353
 354       if (!s0)
 355         {
 356           next0 = SNAT_OUT2IN_NEXT_DROP;
 357           goto out;
 358         }
 359     }
 360   else
 361     {
 362       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 363                         icmp0->type != ICMP4_echo_request &&
 364                         !icmp_is_error_message (icmp0)))
 365         {
 366           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 367           next0 = SNAT_OUT2IN_NEXT_DROP;
 368           goto out;
 369         }
 370
 371       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 372                               value0.value);
 373     }
 374
 375 out:
 376   *p_proto = key0.protocol;
 377   if (s0)
 378     *p_value = s0->in2out;
 379   *p_dont_translate = dont_translate;
 380   if (d)
 381     *(snat_session_t**)d = s0;
 382   return next0;
 383 }
 384
 385 /**
 386  * Get address and port values to be used for packet SNAT translation
 387  *
 388  * @param[in] sm                 SNAT main
 389  * @param[in,out] node           SNAT node runtime
 390  * @param[in] thread_index       thread index
 391  * @param[in,out] b0             buffer containing packet to be translated
 392  * @param[out] p_proto           protocol used for matching
 393  * @param[out] p_value           address and port after NAT translation
 394  * @param[out] p_dont_translate  if packet should not be translated
 395  * @param d                      optional parameter
 396  * @param e                      optional parameter
 397  */
 398 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 399                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 400                            snat_session_key_t *p_value,
 401                            u8 *p_dont_translate, void *d, void *e)
 402 {
 403   ip4_header_t *ip0;
 404   icmp46_header_t *icmp0;
 405   u32 sw_if_index0;
 406   u32 rx_fib_index0;
 407   snat_session_key_t key0;
 408   snat_session_key_t sm0;
 409   u8 dont_translate = 0;
 410   u8 is_addr_only;
 411   u32 next0 = ~0;
 412   int err;
 413
 414   ip0 = vlib_buffer_get_current (b0);
 415   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 416   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 417   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 418
 419   err = icmp_get_key (ip0, &key0);
 420   if (err != -1)
 421     {
 422       b0->error = node->errors[err];
 423       next0 = SNAT_OUT2IN_NEXT_DROP;
 424       goto out2;
 425     }
 426   key0.fib_index = rx_fib_index0;
 427
 428   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 429     {
 430       /* Don't NAT packet aimed at the intfc address */
 431       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 432         {
 433           dont_translate = 1;
 434           goto out;
 435         }
 436       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 437       next0 = SNAT_OUT2IN_NEXT_DROP;
 438       goto out;
 439     }
 440
 441   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 442                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 443                     !icmp_is_error_message (icmp0)))
 444     {
 445       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 446       next0 = SNAT_OUT2IN_NEXT_DROP;
 447       goto out;
 448     }
 449
 450 out:
 451   *p_value = sm0;
 452 out2:
 453   *p_proto = key0.protocol;
 454   *p_dont_translate = dont_translate;
 455   return next0;
 456 }
 457
 458 static inline u32 icmp_out2in (snat_main_t *sm,
 459                                vlib_buffer_t * b0,
 460                                ip4_header_t * ip0,
 461                                icmp46_header_t * icmp0,
 462                                u32 sw_if_index0,
 463                                u32 rx_fib_index0,
 464                                vlib_node_runtime_t * node,
 465                                u32 next0,
 466                                u32 thread_index,
 467                                void *d,
 468                                void *e)
 469 {
 470   snat_session_key_t sm0;
 471   u8 protocol;
 472   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 473   ip4_header_t *inner_ip0 = 0;
 474   void *l4_header = 0;
 475   icmp46_header_t *inner_icmp0;
 476   u8 dont_translate;
 477   u32 new_addr0, old_addr0;
 478   u16 old_id0, new_id0;
 479   ip_csum_t sum0;
 480   u16 checksum0;
 481   u32 next0_tmp;
 482
 483   echo0 = (icmp_echo_header_t *)(icmp0+1);
 484
 485   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 486                                        &protocol, &sm0, &dont_translate, d, e);
 487   if (next0_tmp != ~0)
 488     next0 = next0_tmp;
 489   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 490     goto out;
 491
 492   sum0 = ip_incremental_checksum (0, icmp0,
 493                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 494   checksum0 = ~ip_csum_fold (sum0);
 495   if (checksum0 != 0 && checksum0 != 0xffff)
 496     {
 497       next0 = SNAT_OUT2IN_NEXT_DROP;
 498       goto out;
 499     }
 500
 501   old_addr0 = ip0->dst_address.as_u32;
 502   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 503   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 504
 505   sum0 = ip0->checksum;
 506   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 507                          dst_address /* changed member */);
 508   ip0->checksum = ip_csum_fold (sum0);
 509
 510   if (!icmp_is_error_message (icmp0))
 511     {
 512       new_id0 = sm0.port;
 513       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 514         {
 515           old_id0 = echo0->identifier;
 516           new_id0 = sm0.port;
 517           echo0->identifier = new_id0;
 518
 519           sum0 = icmp0->checksum;
 520           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 521                                  identifier /* changed member */);
 522           icmp0->checksum = ip_csum_fold (sum0);
 523         }
 524     }
 525   else
 526     {
 527       inner_ip0 = (ip4_header_t *)(echo0+1);
 528       l4_header = ip4_next_header (inner_ip0);
 529
 530       if (!ip4_header_checksum_is_valid (inner_ip0))
 531         {
 532           next0 = SNAT_OUT2IN_NEXT_DROP;
 533           goto out;
 534         }
 535
 536       old_addr0 = inner_ip0->src_address.as_u32;
 537       inner_ip0->src_address = sm0.addr;
 538       new_addr0 = inner_ip0->src_address.as_u32;
 539
 540       sum0 = icmp0->checksum;
 541       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 542                              src_address /* changed member */);
 543       icmp0->checksum = ip_csum_fold (sum0);
 544
 545       switch (protocol)
 546         {
 547         case SNAT_PROTOCOL_ICMP:
 548           inner_icmp0 = (icmp46_header_t*)l4_header;
 549           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 550
 551           old_id0 = inner_echo0->identifier;
 552           new_id0 = sm0.port;
 553           inner_echo0->identifier = new_id0;
 554
 555           sum0 = icmp0->checksum;
 556           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 557                                  identifier);
 558           icmp0->checksum = ip_csum_fold (sum0);
 559           break;
 560         case SNAT_PROTOCOL_UDP:
 561         case SNAT_PROTOCOL_TCP:
 562           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 563           new_id0 = sm0.port;
 564           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 565
 566           sum0 = icmp0->checksum;
 567           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 568                                  src_port);
 569           icmp0->checksum = ip_csum_fold (sum0);
 570           break;
 571         default:
 572           ASSERT(0);
 573         }
 574     }
 575
 576 out:
 577   return next0;
 578 }
 579
 580
 581 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 582                                          vlib_buffer_t * b0,
 583                                          ip4_header_t * ip0,
 584                                          icmp46_header_t * icmp0,
 585                                          u32 sw_if_index0,
 586                                          u32 rx_fib_index0,
 587                                          vlib_node_runtime_t * node,
 588                                          u32 next0, f64 now,
 589                                          u32 thread_index,
 590                                          snat_session_t ** p_s0)
 591 {
 592   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 593                       next0, thread_index, p_s0, 0);
 594   snat_session_t * s0 = *p_s0;
 595   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 596     {
 597       /* Accounting */
 598       s0->last_heard = now;
 599       s0->total_pkts++;
 600       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 601       /* Per-user LRU list maintenance for dynamic translation */
 602       if (!snat_is_session_static (s0))
 603         {
 604           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 605                              s0->per_user_index);
 606           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 607                               s0->per_user_list_head_index,
 608                               s0->per_user_index);
 609         }
 610     }
 611   return next0;
 612 }
 613
 614 static uword
 615 snat_out2in_node_fn (vlib_main_t * vm,
 616                   vlib_node_runtime_t * node,
 617                   vlib_frame_t * frame)
 618 {
 619   u32 n_left_from, * from, * to_next;
 620   snat_out2in_next_t next_index;
 621   u32 pkts_processed = 0;
 622   snat_main_t * sm = &snat_main;
 623   f64 now = vlib_time_now (vm);
 624   u32 thread_index = vlib_get_thread_index ();
 625
 626   from = vlib_frame_vector_args (frame);
 627   n_left_from = frame->n_vectors;
 628   next_index = node->cached_next_index;
 629
 630   while (n_left_from > 0)
 631     {
 632       u32 n_left_to_next;
 633
 634       vlib_get_next_frame (vm, node, next_index,
 635                            to_next, n_left_to_next);
 636
 637       while (n_left_from >= 4 && n_left_to_next >= 2)
 638         {
 639           u32 bi0, bi1;
 640           vlib_buffer_t * b0, * b1;
 641           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 642           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 643           u32 sw_if_index0, sw_if_index1;
 644           ip4_header_t * ip0, *ip1;
 645           ip_csum_t sum0, sum1;
 646           u32 new_addr0, old_addr0;
 647           u16 new_port0, old_port0;
 648           u32 new_addr1, old_addr1;
 649           u16 new_port1, old_port1;
 650           udp_header_t * udp0, * udp1;
 651           tcp_header_t * tcp0, * tcp1;
 652           icmp46_header_t * icmp0, * icmp1;
 653           snat_session_key_t key0, key1, sm0, sm1;
 654           u32 rx_fib_index0, rx_fib_index1;
 655           u32 proto0, proto1;
 656           snat_session_t * s0 = 0, * s1 = 0;
 657           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 658
 659           /* Prefetch next iteration. */
 660           {
 661             vlib_buffer_t * p2, * p3;
 662
 663             p2 = vlib_get_buffer (vm, from[2]);
 664             p3 = vlib_get_buffer (vm, from[3]);
 665
 666             vlib_prefetch_buffer_header (p2, LOAD);
 667             vlib_prefetch_buffer_header (p3, LOAD);
 668
 669             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 670             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 671           }
 672
 673           /* speculatively enqueue b0 and b1 to the current next frame */
 674           to_next[0] = bi0 = from[0];
 675           to_next[1] = bi1 = from[1];
 676           from += 2;
 677           to_next += 2;
 678           n_left_from -= 2;
 679           n_left_to_next -= 2;
 680
 681           b0 = vlib_get_buffer (vm, bi0);
 682           b1 = vlib_get_buffer (vm, bi1);
 683
 684           ip0 = vlib_buffer_get_current (b0);
 685           udp0 = ip4_next_header (ip0);
 686           tcp0 = (tcp_header_t *) udp0;
 687           icmp0 = (icmp46_header_t *) udp0;
 688
 689           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 690           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 691                                    sw_if_index0);
 692
 693           if (PREDICT_FALSE(ip0->ttl == 1))
 694             {
 695               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 696               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 697                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 698                                            0);
 699               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 700               goto trace0;
 701             }
 702
 703           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 704
 705           if (PREDICT_FALSE (proto0 == ~0))
 706               goto trace0;
 707
 708           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 709             {
 710               next0 = icmp_out2in_slow_path
 711                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 712                  next0, now, thread_index, &s0);
 713               goto trace0;
 714             }
 715
 716           key0.addr = ip0->dst_address;
 717           key0.port = udp0->dst_port;
 718           key0.protocol = proto0;
 719           key0.fib_index = rx_fib_index0;
 720
 721           kv0.key = key0.as_u64;
 722
 723           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 724             {
 725               /* Try to match static mapping by external address and port,
 726                  destination address and port in packet */
 727               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 728                 {
 729                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 730                   /*
 731                    * Send DHCP packets to the ipv4 stack, or we won't
 732                    * be able to use dhcp client on the outside interface
 733                    */
 734                   if (proto0 != SNAT_PROTOCOL_UDP
 735                       || (udp0->dst_port
 736                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 737                     next0 = SNAT_OUT2IN_NEXT_DROP;
 738                   goto trace0;
 739                 }
 740
 741               /* Create session initiated by host from external network */
 742               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 743                                                      thread_index);
 744               if (!s0)
 745                 {
 746                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 747                   next0 = SNAT_OUT2IN_NEXT_DROP;
 748                   goto trace0;
 749                 }
 750             }
 751           else
 752             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 753                                     value0.value);
 754
 755           old_addr0 = ip0->dst_address.as_u32;
 756           ip0->dst_address = s0->in2out.addr;
 757           new_addr0 = ip0->dst_address.as_u32;
 758           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 759
 760           sum0 = ip0->checksum;
 761           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 762                                  ip4_header_t,
 763                                  dst_address /* changed member */);
 764           ip0->checksum = ip_csum_fold (sum0);
 765
 766           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 767             {
 768               old_port0 = tcp0->dst_port;
 769               tcp0->dst_port = s0->in2out.port;
 770               new_port0 = tcp0->dst_port;
 771
 772               sum0 = tcp0->checksum;
 773               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 774                                      ip4_header_t,
 775                                      dst_address /* changed member */);
 776
 777               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 778                                      ip4_header_t /* cheat */,
 779                                      length /* changed member */);
 780               tcp0->checksum = ip_csum_fold(sum0);
 781             }
 782           else
 783             {
 784               old_port0 = udp0->dst_port;
 785               udp0->dst_port = s0->in2out.port;
 786               udp0->checksum = 0;
 787             }
 788
 789           /* Accounting */
 790           s0->last_heard = now;
 791           s0->total_pkts++;
 792           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 793           /* Per-user LRU list maintenance for dynamic translation */
 794           if (!snat_is_session_static (s0))
 795             {
 796               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 797                                  s0->per_user_index);
 798               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 799                                   s0->per_user_list_head_index,
 800                                   s0->per_user_index);
 801             }
 802         trace0:
 803
 804           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 805                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 806             {
 807               snat_out2in_trace_t *t =
 808                  vlib_add_trace (vm, node, b0, sizeof (*t));
 809               t->sw_if_index = sw_if_index0;
 810               t->next_index = next0;
 811               t->session_index = ~0;
 812               if (s0)
 813                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 814             }
 815
 816           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 817
 818
 819           ip1 = vlib_buffer_get_current (b1);
 820           udp1 = ip4_next_header (ip1);
 821           tcp1 = (tcp_header_t *) udp1;
 822           icmp1 = (icmp46_header_t *) udp1;
 823
 824           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 825           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 826                                    sw_if_index1);
 827
 828           if (PREDICT_FALSE(ip1->ttl == 1))
 829             {
 830               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 831               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 832                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 833                                            0);
 834               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 835               goto trace1;
 836             }
 837
 838           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 839
 840           if (PREDICT_FALSE (proto1 == ~0))
 841               goto trace1;
 842
 843           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 844             {
 845               next1 = icmp_out2in_slow_path
 846                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 847                  next1, now, thread_index, &s1);
 848               goto trace1;
 849             }
 850
 851           key1.addr = ip1->dst_address;
 852           key1.port = udp1->dst_port;
 853           key1.protocol = proto1;
 854           key1.fib_index = rx_fib_index1;
 855
 856           kv1.key = key1.as_u64;
 857
 858           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 859             {
 860               /* Try to match static mapping by external address and port,
 861                  destination address and port in packet */
 862               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
 863                 {
 864                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 865                   /*
 866                    * Send DHCP packets to the ipv4 stack, or we won't
 867                    * be able to use dhcp client on the outside interface
 868                    */
 869                   if (proto1 != SNAT_PROTOCOL_UDP
 870                       || (udp1->dst_port
 871                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 872                     next1 = SNAT_OUT2IN_NEXT_DROP;
 873                   goto trace1;
 874                 }
 875
 876               /* Create session initiated by host from external network */
 877               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 878                                                      thread_index);
 879               if (!s1)
 880                 {
 881                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 882                   next1 = SNAT_OUT2IN_NEXT_DROP;
 883                   goto trace1;
 884                 }
 885             }
 886           else
 887             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 888                                     value1.value);
 889
 890           old_addr1 = ip1->dst_address.as_u32;
 891           ip1->dst_address = s1->in2out.addr;
 892           new_addr1 = ip1->dst_address.as_u32;
 893           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 894
 895           sum1 = ip1->checksum;
 896           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 897                                  ip4_header_t,
 898                                  dst_address /* changed member */);
 899           ip1->checksum = ip_csum_fold (sum1);
 900
 901           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 902             {
 903               old_port1 = tcp1->dst_port;
 904               tcp1->dst_port = s1->in2out.port;
 905               new_port1 = tcp1->dst_port;
 906
 907               sum1 = tcp1->checksum;
 908               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 909                                      ip4_header_t,
 910                                      dst_address /* changed member */);
 911
 912               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 913                                      ip4_header_t /* cheat */,
 914                                      length /* changed member */);
 915               tcp1->checksum = ip_csum_fold(sum1);
 916             }
 917           else
 918             {
 919               old_port1 = udp1->dst_port;
 920               udp1->dst_port = s1->in2out.port;
 921               udp1->checksum = 0;
 922             }
 923
 924           /* Accounting */
 925           s1->last_heard = now;
 926           s1->total_pkts++;
 927           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 928           /* Per-user LRU list maintenance for dynamic translation */
 929           if (!snat_is_session_static (s1))
 930             {
 931               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 932                                  s1->per_user_index);
 933               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 934                                   s1->per_user_list_head_index,
 935                                   s1->per_user_index);
 936             }
 937         trace1:
 938
 939           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 940                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 941             {
 942               snat_out2in_trace_t *t =
 943                  vlib_add_trace (vm, node, b1, sizeof (*t));
 944               t->sw_if_index = sw_if_index1;
 945               t->next_index = next1;
 946               t->session_index = ~0;
 947               if (s1)
 948                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
 949             }
 950
 951           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 952
 953           /* verify speculative enqueues, maybe switch current next frame */
 954           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 955                                            to_next, n_left_to_next,
 956                                            bi0, bi1, next0, next1);
 957         }
 958
 959       while (n_left_from > 0 && n_left_to_next > 0)
 960         {
 961           u32 bi0;
 962           vlib_buffer_t * b0;
 963           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 964           u32 sw_if_index0;
 965           ip4_header_t * ip0;
 966           ip_csum_t sum0;
 967           u32 new_addr0, old_addr0;
 968           u16 new_port0, old_port0;
 969           udp_header_t * udp0;
 970           tcp_header_t * tcp0;
 971           icmp46_header_t * icmp0;
 972           snat_session_key_t key0, sm0;
 973           u32 rx_fib_index0;
 974           u32 proto0;
 975           snat_session_t * s0 = 0;
 976           clib_bihash_kv_8_8_t kv0, value0;
 977
 978           /* speculatively enqueue b0 to the current next frame */
 979           bi0 = from[0];
 980           to_next[0] = bi0;
 981           from += 1;
 982           to_next += 1;
 983           n_left_from -= 1;
 984           n_left_to_next -= 1;
 985
 986           b0 = vlib_get_buffer (vm, bi0);
 987
 988           ip0 = vlib_buffer_get_current (b0);
 989           udp0 = ip4_next_header (ip0);
 990           tcp0 = (tcp_header_t *) udp0;
 991           icmp0 = (icmp46_header_t *) udp0;
 992
 993           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 994           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 995                                    sw_if_index0);
 996
 997           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 998
 999           if (PREDICT_FALSE (proto0 == ~0))
1000               goto trace00;
1001
1002           if (PREDICT_FALSE(ip0->ttl == 1))
1003             {
1004               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1005               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1006                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1007                                            0);
1008               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1009               goto trace00;
1010             }
1011
1012           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1013             {
1014               next0 = icmp_out2in_slow_path
1015                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1016                  next0, now, thread_index, &s0);
1017               goto trace00;
1018             }
1019
1020           key0.addr = ip0->dst_address;
1021           key0.port = udp0->dst_port;
1022           key0.protocol = proto0;
1023           key0.fib_index = rx_fib_index0;
1024
1025           kv0.key = key0.as_u64;
1026
1027           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1028             {
1029               /* Try to match static mapping by external address and port,
1030                  destination address and port in packet */
1031               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1032                 {
1033                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1034                   /*
1035                    * Send DHCP packets to the ipv4 stack, or we won't
1036                    * be able to use dhcp client on the outside interface
1037                    */
1038                   if (proto0 != SNAT_PROTOCOL_UDP
1039                       || (udp0->dst_port
1040                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1041
1042                     next0 = SNAT_OUT2IN_NEXT_DROP;
1043                   goto trace00;
1044                 }
1045
1046               /* Create session initiated by host from external network */
1047               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1048                                                      thread_index);
1049               if (!s0)
1050                 {
1051                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1052                     next0 = SNAT_OUT2IN_NEXT_DROP;
1053                   goto trace00;
1054                 }
1055             }
1056           else
1057             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1058                                     value0.value);
1059
1060           old_addr0 = ip0->dst_address.as_u32;
1061           ip0->dst_address = s0->in2out.addr;
1062           new_addr0 = ip0->dst_address.as_u32;
1063           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1064
1065           sum0 = ip0->checksum;
1066           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1067                                  ip4_header_t,
1068                                  dst_address /* changed member */);
1069           ip0->checksum = ip_csum_fold (sum0);
1070
1071           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1072             {
1073               old_port0 = tcp0->dst_port;
1074               tcp0->dst_port = s0->in2out.port;
1075               new_port0 = tcp0->dst_port;
1076
1077               sum0 = tcp0->checksum;
1078               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1079                                      ip4_header_t,
1080                                      dst_address /* changed member */);
1081
1082               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1083                                      ip4_header_t /* cheat */,
1084                                      length /* changed member */);
1085               tcp0->checksum = ip_csum_fold(sum0);
1086             }
1087           else
1088             {
1089               old_port0 = udp0->dst_port;
1090               udp0->dst_port = s0->in2out.port;
1091               udp0->checksum = 0;
1092             }
1093
1094           /* Accounting */
1095           s0->last_heard = now;
1096           s0->total_pkts++;
1097           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1098           /* Per-user LRU list maintenance for dynamic translation */
1099           if (!snat_is_session_static (s0))
1100             {
1101               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1102                                  s0->per_user_index);
1103               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1104                                   s0->per_user_list_head_index,
1105                                   s0->per_user_index);
1106             }
1107         trace00:
1108
1109           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1110                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1111             {
1112               snat_out2in_trace_t *t =
1113                  vlib_add_trace (vm, node, b0, sizeof (*t));
1114               t->sw_if_index = sw_if_index0;
1115               t->next_index = next0;
1116               t->session_index = ~0;
1117               if (s0)
1118                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1119             }
1120
1121           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1122
1123           /* verify speculative enqueue, maybe switch current next frame */
1124           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1125                                            to_next, n_left_to_next,
1126                                            bi0, next0);
1127         }
1128
1129       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1130     }
1131
1132   vlib_node_increment_counter (vm, snat_out2in_node.index,
1133                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1134                                pkts_processed);
1135   return frame->n_vectors;
1136 }
1137
1138 VLIB_REGISTER_NODE (snat_out2in_node) = {
1139   .function = snat_out2in_node_fn,
1140   .name = "snat-out2in",
1141   .vector_size = sizeof (u32),
1142   .format_trace = format_snat_out2in_trace,
1143   .type = VLIB_NODE_TYPE_INTERNAL,
1144
1145   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1146   .error_strings = snat_out2in_error_strings,
1147
1148   .runtime_data_bytes = sizeof (snat_runtime_t),
1149
1150   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1151
1152   /* edit / add dispositions here */
1153   .next_nodes = {
1154     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1155     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1156     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1157   },
1158 };
1159 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1160
1161 /**************************/
1162 /*** deterministic mode ***/
1163 /**************************/
1164 static uword
1165 snat_det_out2in_node_fn (vlib_main_t * vm,
1166                          vlib_node_runtime_t * node,
1167                          vlib_frame_t * frame)
1168 {
1169   u32 n_left_from, * from, * to_next;
1170   snat_out2in_next_t next_index;
1171   u32 pkts_processed = 0;
1172   snat_main_t * sm = &snat_main;
1173   u32 thread_index = vlib_get_thread_index ();
1174
1175   from = vlib_frame_vector_args (frame);
1176   n_left_from = frame->n_vectors;
1177   next_index = node->cached_next_index;
1178
1179   while (n_left_from > 0)
1180     {
1181       u32 n_left_to_next;
1182
1183       vlib_get_next_frame (vm, node, next_index,
1184                            to_next, n_left_to_next);
1185
1186       while (n_left_from >= 4 && n_left_to_next >= 2)
1187         {
1188           u32 bi0, bi1;
1189           vlib_buffer_t * b0, * b1;
1190           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1191           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1192           u32 sw_if_index0, sw_if_index1;
1193           ip4_header_t * ip0, * ip1;
1194           ip_csum_t sum0, sum1;
1195           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1196           u16 new_port0, old_port0, old_port1, new_port1;
1197           udp_header_t * udp0, * udp1;
1198           tcp_header_t * tcp0, * tcp1;
1199           u32 proto0, proto1;
1200           snat_det_out_key_t key0, key1;
1201           snat_det_map_t * dm0, * dm1;
1202           snat_det_session_t * ses0 = 0, * ses1 = 0;
1203           u32 rx_fib_index0, rx_fib_index1;
1204           icmp46_header_t * icmp0, * icmp1;
1205
1206           /* Prefetch next iteration. */
1207           {
1208             vlib_buffer_t * p2, * p3;
1209
1210             p2 = vlib_get_buffer (vm, from[2]);
1211             p3 = vlib_get_buffer (vm, from[3]);
1212
1213             vlib_prefetch_buffer_header (p2, LOAD);
1214             vlib_prefetch_buffer_header (p3, LOAD);
1215
1216             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1217             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1218           }
1219
1220           /* speculatively enqueue b0 and b1 to the current next frame */
1221           to_next[0] = bi0 = from[0];
1222           to_next[1] = bi1 = from[1];
1223           from += 2;
1224           to_next += 2;
1225           n_left_from -= 2;
1226           n_left_to_next -= 2;
1227
1228           b0 = vlib_get_buffer (vm, bi0);
1229           b1 = vlib_get_buffer (vm, bi1);
1230
1231           ip0 = vlib_buffer_get_current (b0);
1232           udp0 = ip4_next_header (ip0);
1233           tcp0 = (tcp_header_t *) udp0;
1234
1235           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1236
1237           if (PREDICT_FALSE(ip0->ttl == 1))
1238             {
1239               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1240               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1241                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1242                                            0);
1243               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1244               goto trace0;
1245             }
1246
1247           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1248
1249           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1250             {
1251               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1252               icmp0 = (icmp46_header_t *) udp0;
1253
1254               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1255                                   rx_fib_index0, node, next0, thread_index,
1256                                   &ses0, &dm0);
1257               goto trace0;
1258             }
1259
1260           key0.ext_host_addr = ip0->src_address;
1261           key0.ext_host_port = tcp0->src;
1262           key0.out_port = tcp0->dst;
1263
1264           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1265           if (PREDICT_FALSE(!dm0))
1266             {
1267               clib_warning("unknown dst address:  %U",
1268                            format_ip4_address, &ip0->dst_address);
1269               next0 = SNAT_OUT2IN_NEXT_DROP;
1270               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1271               goto trace0;
1272             }
1273
1274           snat_det_reverse(dm0, &ip0->dst_address,
1275                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1276
1277           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1278           if (PREDICT_FALSE(!ses0))
1279             {
1280               clib_warning("no match src %U:%d dst %U:%d for user %U",
1281                            format_ip4_address, &ip0->src_address,
1282                            clib_net_to_host_u16 (tcp0->src),
1283                            format_ip4_address, &ip0->dst_address,
1284                            clib_net_to_host_u16 (tcp0->dst),
1285                            format_ip4_address, &new_addr0);
1286               next0 = SNAT_OUT2IN_NEXT_DROP;
1287               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1288               goto trace0;
1289             }
1290           new_port0 = ses0->in_port;
1291
1292           old_addr0 = ip0->dst_address;
1293           ip0->dst_address = new_addr0;
1294           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1295
1296           sum0 = ip0->checksum;
1297           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1298                                  ip4_header_t,
1299                                  dst_address /* changed member */);
1300           ip0->checksum = ip_csum_fold (sum0);
1301
1302           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1303             {
1304               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1305                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1306               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1307                 snat_det_ses_close(dm0, ses0);
1308
1309               old_port0 = tcp0->dst;
1310               tcp0->dst = new_port0;
1311
1312               sum0 = tcp0->checksum;
1313               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1314                                      ip4_header_t,
1315                                      dst_address /* changed member */);
1316
1317               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1318                                      ip4_header_t /* cheat */,
1319                                      length /* changed member */);
1320               tcp0->checksum = ip_csum_fold(sum0);
1321             }
1322           else
1323             {
1324               old_port0 = udp0->dst_port;
1325               udp0->dst_port = new_port0;
1326               udp0->checksum = 0;
1327             }
1328
1329         trace0:
1330
1331           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1332                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1333             {
1334               snat_out2in_trace_t *t =
1335                  vlib_add_trace (vm, node, b0, sizeof (*t));
1336               t->sw_if_index = sw_if_index0;
1337               t->next_index = next0;
1338               t->session_index = ~0;
1339               if (ses0)
1340                 t->session_index = ses0 - dm0->sessions;
1341             }
1342
1343           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1344
1345           b1 = vlib_get_buffer (vm, bi1);
1346
1347           ip1 = vlib_buffer_get_current (b1);
1348           udp1 = ip4_next_header (ip1);
1349           tcp1 = (tcp_header_t *) udp1;
1350
1351           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1352
1353           if (PREDICT_FALSE(ip1->ttl == 1))
1354             {
1355               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1356               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1357                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1358                                            0);
1359               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1360               goto trace1;
1361             }
1362
1363           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1364
1365           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1366             {
1367               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1368               icmp1 = (icmp46_header_t *) udp1;
1369
1370               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1371                                   rx_fib_index1, node, next1, thread_index,
1372                                   &ses1, &dm1);
1373               goto trace1;
1374             }
1375
1376           key1.ext_host_addr = ip1->src_address;
1377           key1.ext_host_port = tcp1->src;
1378           key1.out_port = tcp1->dst;
1379
1380           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1381           if (PREDICT_FALSE(!dm1))
1382             {
1383               clib_warning("unknown dst address:  %U",
1384                            format_ip4_address, &ip1->dst_address);
1385               next1 = SNAT_OUT2IN_NEXT_DROP;
1386               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1387               goto trace1;
1388             }
1389
1390           snat_det_reverse(dm1, &ip1->dst_address,
1391                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1392
1393           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1394           if (PREDICT_FALSE(!ses1))
1395             {
1396               clib_warning("no match src %U:%d dst %U:%d for user %U",
1397                            format_ip4_address, &ip1->src_address,
1398                            clib_net_to_host_u16 (tcp1->src),
1399                            format_ip4_address, &ip1->dst_address,
1400                            clib_net_to_host_u16 (tcp1->dst),
1401                            format_ip4_address, &new_addr1);
1402               next1 = SNAT_OUT2IN_NEXT_DROP;
1403               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1404               goto trace1;
1405             }
1406           new_port1 = ses1->in_port;
1407
1408           old_addr1 = ip1->dst_address;
1409           ip1->dst_address = new_addr1;
1410           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1411
1412           sum1 = ip1->checksum;
1413           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1414                                  ip4_header_t,
1415                                  dst_address /* changed member */);
1416           ip1->checksum = ip_csum_fold (sum1);
1417
1418           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1419             {
1420               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1421                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1422               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1423                 snat_det_ses_close(dm1, ses1);
1424
1425               old_port1 = tcp1->dst;
1426               tcp1->dst = new_port1;
1427
1428               sum1 = tcp1->checksum;
1429               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1430                                      ip4_header_t,
1431                                      dst_address /* changed member */);
1432
1433               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1434                                      ip4_header_t /* cheat */,
1435                                      length /* changed member */);
1436               tcp1->checksum = ip_csum_fold(sum1);
1437             }
1438           else
1439             {
1440               old_port1 = udp1->dst_port;
1441               udp1->dst_port = new_port1;
1442               udp1->checksum = 0;
1443             }
1444
1445         trace1:
1446
1447           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1448                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1449             {
1450               snat_out2in_trace_t *t =
1451                  vlib_add_trace (vm, node, b1, sizeof (*t));
1452               t->sw_if_index = sw_if_index1;
1453               t->next_index = next1;
1454               t->session_index = ~0;
1455               if (ses1)
1456                 t->session_index = ses1 - dm1->sessions;
1457             }
1458
1459           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1460
1461           /* verify speculative enqueues, maybe switch current next frame */
1462           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1463                                            to_next, n_left_to_next,
1464                                            bi0, bi1, next0, next1);
1465          }
1466
1467       while (n_left_from > 0 && n_left_to_next > 0)
1468         {
1469           u32 bi0;
1470           vlib_buffer_t * b0;
1471           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1472           u32 sw_if_index0;
1473           ip4_header_t * ip0;
1474           ip_csum_t sum0;
1475           ip4_address_t new_addr0, old_addr0;
1476           u16 new_port0, old_port0;
1477           udp_header_t * udp0;
1478           tcp_header_t * tcp0;
1479           u32 proto0;
1480           snat_det_out_key_t key0;
1481           snat_det_map_t * dm0;
1482           snat_det_session_t * ses0 = 0;
1483           u32 rx_fib_index0;
1484           icmp46_header_t * icmp0;
1485
1486           /* speculatively enqueue b0 to the current next frame */
1487           bi0 = from[0];
1488           to_next[0] = bi0;
1489           from += 1;
1490           to_next += 1;
1491           n_left_from -= 1;
1492           n_left_to_next -= 1;
1493
1494           b0 = vlib_get_buffer (vm, bi0);
1495
1496           ip0 = vlib_buffer_get_current (b0);
1497           udp0 = ip4_next_header (ip0);
1498           tcp0 = (tcp_header_t *) udp0;
1499
1500           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1501
1502           if (PREDICT_FALSE(ip0->ttl == 1))
1503             {
1504               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1505               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1506                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1507                                            0);
1508               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1509               goto trace00;
1510             }
1511
1512           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1513
1514           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1515             {
1516               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1517               icmp0 = (icmp46_header_t *) udp0;
1518
1519               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1520                                   rx_fib_index0, node, next0, thread_index,
1521                                   &ses0, &dm0);
1522               goto trace00;
1523             }
1524
1525           key0.ext_host_addr = ip0->src_address;
1526           key0.ext_host_port = tcp0->src;
1527           key0.out_port = tcp0->dst;
1528
1529           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1530           if (PREDICT_FALSE(!dm0))
1531             {
1532               clib_warning("unknown dst address:  %U",
1533                            format_ip4_address, &ip0->dst_address);
1534               next0 = SNAT_OUT2IN_NEXT_DROP;
1535               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1536               goto trace00;
1537             }
1538
1539           snat_det_reverse(dm0, &ip0->dst_address,
1540                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1541
1542           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1543           if (PREDICT_FALSE(!ses0))
1544             {
1545               clib_warning("no match src %U:%d dst %U:%d for user %U",
1546                            format_ip4_address, &ip0->src_address,
1547                            clib_net_to_host_u16 (tcp0->src),
1548                            format_ip4_address, &ip0->dst_address,
1549                            clib_net_to_host_u16 (tcp0->dst),
1550                            format_ip4_address, &new_addr0);
1551               next0 = SNAT_OUT2IN_NEXT_DROP;
1552               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1553               goto trace00;
1554             }
1555           new_port0 = ses0->in_port;
1556
1557           old_addr0 = ip0->dst_address;
1558           ip0->dst_address = new_addr0;
1559           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1560
1561           sum0 = ip0->checksum;
1562           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1563                                  ip4_header_t,
1564                                  dst_address /* changed member */);
1565           ip0->checksum = ip_csum_fold (sum0);
1566
1567           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1568             {
1569               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1570                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1571               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1572                 snat_det_ses_close(dm0, ses0);
1573
1574               old_port0 = tcp0->dst;
1575               tcp0->dst = new_port0;
1576
1577               sum0 = tcp0->checksum;
1578               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1579                                      ip4_header_t,
1580                                      dst_address /* changed member */);
1581
1582               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1583                                      ip4_header_t /* cheat */,
1584                                      length /* changed member */);
1585               tcp0->checksum = ip_csum_fold(sum0);
1586             }
1587           else
1588             {
1589               old_port0 = udp0->dst_port;
1590               udp0->dst_port = new_port0;
1591               udp0->checksum = 0;
1592             }
1593
1594         trace00:
1595
1596           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1597                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1598             {
1599               snat_out2in_trace_t *t =
1600                  vlib_add_trace (vm, node, b0, sizeof (*t));
1601               t->sw_if_index = sw_if_index0;
1602               t->next_index = next0;
1603               t->session_index = ~0;
1604               if (ses0)
1605                 t->session_index = ses0 - dm0->sessions;
1606             }
1607
1608           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1609
1610           /* verify speculative enqueue, maybe switch current next frame */
1611           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1612                                            to_next, n_left_to_next,
1613                                            bi0, next0);
1614         }
1615
1616       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1617     }
1618
1619   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1620                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1621                                pkts_processed);
1622   return frame->n_vectors;
1623 }
1624
1625 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1626   .function = snat_det_out2in_node_fn,
1627   .name = "snat-det-out2in",
1628   .vector_size = sizeof (u32),
1629   .format_trace = format_snat_out2in_trace,
1630   .type = VLIB_NODE_TYPE_INTERNAL,
1631
1632   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1633   .error_strings = snat_out2in_error_strings,
1634
1635   .runtime_data_bytes = sizeof (snat_runtime_t),
1636
1637   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1638
1639   /* edit / add dispositions here */
1640   .next_nodes = {
1641     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1642     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1643     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1644   },
1645 };
1646 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1647
1648 /**
1649  * Get address and port values to be used for packet SNAT translation
1650  * and create session if needed
1651  *
1652  * @param[in,out] sm             SNAT main
1653  * @param[in,out] node           SNAT node runtime
1654  * @param[in] thread_index       thread index
1655  * @param[in,out] b0             buffer containing packet to be translated
1656  * @param[out] p_proto           protocol used for matching
1657  * @param[out] p_value           address and port after NAT translation
1658  * @param[out] p_dont_translate  if packet should not be translated
1659  * @param d                      optional parameter
1660  * @param e                      optional parameter
1661  */
1662 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
1663                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
1664                           snat_session_key_t *p_value,
1665                           u8 *p_dont_translate, void *d, void *e)
1666 {
1667   ip4_header_t *ip0;
1668   icmp46_header_t *icmp0;
1669   u32 sw_if_index0;
1670   u8 protocol;
1671   snat_det_out_key_t key0;
1672   u8 dont_translate = 0;
1673   u32 next0 = ~0;
1674   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1675   ip4_header_t *inner_ip0;
1676   void *l4_header = 0;
1677   icmp46_header_t *inner_icmp0;
1678   snat_det_map_t * dm0 = 0;
1679   ip4_address_t new_addr0 = {{0}};
1680   snat_det_session_t * ses0 = 0;
1681   ip4_address_t out_addr;
1682
1683   ip0 = vlib_buffer_get_current (b0);
1684   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1685   echo0 = (icmp_echo_header_t *)(icmp0+1);
1686   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1687
1688   if (!icmp_is_error_message (icmp0))
1689     {
1690       protocol = SNAT_PROTOCOL_ICMP;
1691       key0.ext_host_addr = ip0->src_address;
1692       key0.ext_host_port = 0;
1693       key0.out_port = echo0->identifier;
1694       out_addr = ip0->dst_address;
1695     }
1696   else
1697     {
1698       inner_ip0 = (ip4_header_t *)(echo0+1);
1699       l4_header = ip4_next_header (inner_ip0);
1700       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
1701       key0.ext_host_addr = inner_ip0->dst_address;
1702       out_addr = inner_ip0->src_address;
1703       switch (protocol)
1704         {
1705         case SNAT_PROTOCOL_ICMP:
1706           inner_icmp0 = (icmp46_header_t*)l4_header;
1707           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1708           key0.ext_host_port = 0;
1709           key0.out_port = inner_echo0->identifier;
1710           break;
1711         case SNAT_PROTOCOL_UDP:
1712         case SNAT_PROTOCOL_TCP:
1713           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1714           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
1715           break;
1716         default:
1717           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1718           next0 = SNAT_OUT2IN_NEXT_DROP;
1719           goto out;
1720         }
1721     }
1722
1723   dm0 = snat_det_map_by_out(sm, &out_addr);
1724   if (PREDICT_FALSE(!dm0))
1725     {
1726       /* Don't NAT packet aimed at the intfc address */
1727       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1728                                           ip0->dst_address.as_u32)))
1729         {
1730           dont_translate = 1;
1731           goto out;
1732         }
1733       clib_warning("unknown dst address:  %U",
1734                    format_ip4_address, &ip0->dst_address);
1735       goto out;
1736     }
1737
1738   snat_det_reverse(dm0, &ip0->dst_address,
1739                    clib_net_to_host_u16(key0.out_port), &new_addr0);
1740
1741   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1742   if (PREDICT_FALSE(!ses0))
1743     {
1744       /* Don't NAT packet aimed at the intfc address */
1745       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1746                                           ip0->dst_address.as_u32)))
1747         {
1748           dont_translate = 1;
1749           goto out;
1750         }
1751       clib_warning("no match src %U:%d dst %U:%d for user %U",
1752                    format_ip4_address, &key0.ext_host_addr,
1753                    clib_net_to_host_u16 (key0.ext_host_port),
1754                    format_ip4_address, &out_addr,
1755                    clib_net_to_host_u16 (key0.out_port),
1756                    format_ip4_address, &new_addr0);
1757       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1758       next0 = SNAT_OUT2IN_NEXT_DROP;
1759       goto out;
1760     }
1761
1762   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
1763                     !icmp_is_error_message (icmp0)))
1764     {
1765       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1766       next0 = SNAT_OUT2IN_NEXT_DROP;
1767       goto out;
1768     }
1769
1770   goto out;
1771
1772 out:
1773   *p_proto = protocol;
1774   if (ses0)
1775     {
1776       p_value->addr = new_addr0;
1777       p_value->fib_index = sm->inside_fib_index;
1778       p_value->port = ses0->in_port;
1779     }
1780   *p_dont_translate = dont_translate;
1781   if (d)
1782     *(snat_det_session_t**)d = ses0;
1783   if (e)
1784     *(snat_det_map_t**)e = dm0;
1785   return next0;
1786 }
1787
1788 /**********************/
1789 /*** worker handoff ***/
1790 /**********************/
1791 static uword
1792 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1793                                vlib_node_runtime_t * node,
1794                                vlib_frame_t * frame)
1795 {
1796   snat_main_t *sm = &snat_main;
1797   vlib_thread_main_t *tm = vlib_get_thread_main ();
1798   u32 n_left_from, *from, *to_next = 0;
1799   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1800   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1801     = 0;
1802   vlib_frame_queue_elt_t *hf = 0;
1803   vlib_frame_t *f = 0;
1804   int i;
1805   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1806   u32 next_worker_index = 0;
1807   u32 current_worker_index = ~0;
1808   u32 thread_index = vlib_get_thread_index ();
1809
1810   ASSERT (vec_len (sm->workers));
1811
1812   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1813     {
1814       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1815
1816       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1817                                sm->first_worker_index + sm->num_workers - 1,
1818                                (vlib_frame_queue_t *) (~0));
1819     }
1820
1821   from = vlib_frame_vector_args (frame);
1822   n_left_from = frame->n_vectors;
1823
1824   while (n_left_from > 0)
1825     {
1826       u32 bi0;
1827       vlib_buffer_t *b0;
1828       u32 sw_if_index0;
1829       u32 rx_fib_index0;
1830       ip4_header_t * ip0;
1831       u8 do_handoff;
1832
1833       bi0 = from[0];
1834       from += 1;
1835       n_left_from -= 1;
1836
1837       b0 = vlib_get_buffer (vm, bi0);
1838
1839       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1840       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1841
1842       ip0 = vlib_buffer_get_current (b0);
1843
1844       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1845
1846       if (PREDICT_FALSE (next_worker_index != thread_index))
1847         {
1848           do_handoff = 1;
1849
1850           if (next_worker_index != current_worker_index)
1851             {
1852               if (hf)
1853                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1854
1855               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1856                                                       next_worker_index,
1857                                                       handoff_queue_elt_by_worker_index);
1858
1859               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1860               to_next_worker = &hf->buffer_index[hf->n_vectors];
1861               current_worker_index = next_worker_index;
1862             }
1863
1864           /* enqueue to correct worker thread */
1865           to_next_worker[0] = bi0;
1866           to_next_worker++;
1867           n_left_to_next_worker--;
1868
1869           if (n_left_to_next_worker == 0)
1870             {
1871               hf->n_vectors = VLIB_FRAME_SIZE;
1872               vlib_put_frame_queue_elt (hf);
1873               current_worker_index = ~0;
1874               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1875               hf = 0;
1876             }
1877         }
1878       else
1879         {
1880           do_handoff = 0;
1881           /* if this is 1st frame */
1882           if (!f)
1883             {
1884               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1885               to_next = vlib_frame_vector_args (f);
1886             }
1887
1888           to_next[0] = bi0;
1889           to_next += 1;
1890           f->n_vectors++;
1891         }
1892
1893       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1894                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1895         {
1896           snat_out2in_worker_handoff_trace_t *t =
1897             vlib_add_trace (vm, node, b0, sizeof (*t));
1898           t->next_worker_index = next_worker_index;
1899           t->do_handoff = do_handoff;
1900         }
1901     }
1902
1903   if (f)
1904     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1905
1906   if (hf)
1907     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1908
1909   /* Ship frames to the worker nodes */
1910   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1911     {
1912       if (handoff_queue_elt_by_worker_index[i])
1913         {
1914           hf = handoff_queue_elt_by_worker_index[i];
1915           /*
1916            * It works better to let the handoff node
1917            * rate-adapt, always ship the handoff queue element.
1918            */
1919           if (1 || hf->n_vectors == hf->last_n_vectors)
1920             {
1921               vlib_put_frame_queue_elt (hf);
1922               handoff_queue_elt_by_worker_index[i] = 0;
1923             }
1924           else
1925             hf->last_n_vectors = hf->n_vectors;
1926         }
1927       congested_handoff_queue_by_worker_index[i] =
1928         (vlib_frame_queue_t *) (~0);
1929     }
1930   hf = 0;
1931   current_worker_index = ~0;
1932   return frame->n_vectors;
1933 }
1934
1935 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1936   .function = snat_out2in_worker_handoff_fn,
1937   .name = "snat-out2in-worker-handoff",
1938   .vector_size = sizeof (u32),
1939   .format_trace = format_snat_out2in_worker_handoff_trace,
1940   .type = VLIB_NODE_TYPE_INTERNAL,
1941
1942   .n_next_nodes = 1,
1943
1944   .next_nodes = {
1945     [0] = "error-drop",
1946   },
1947 };
1948
1949 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1950
1951 static uword
1952 snat_out2in_fast_node_fn (vlib_main_t * vm,
1953                           vlib_node_runtime_t * node,
1954                           vlib_frame_t * frame)
1955 {
1956   u32 n_left_from, * from, * to_next;
1957   snat_out2in_next_t next_index;
1958   u32 pkts_processed = 0;
1959   snat_main_t * sm = &snat_main;
1960
1961   from = vlib_frame_vector_args (frame);
1962   n_left_from = frame->n_vectors;
1963   next_index = node->cached_next_index;
1964
1965   while (n_left_from > 0)
1966     {
1967       u32 n_left_to_next;
1968
1969       vlib_get_next_frame (vm, node, next_index,
1970                            to_next, n_left_to_next);
1971
1972       while (n_left_from > 0 && n_left_to_next > 0)
1973         {
1974           u32 bi0;
1975           vlib_buffer_t * b0;
1976           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1977           u32 sw_if_index0;
1978           ip4_header_t * ip0;
1979           ip_csum_t sum0;
1980           u32 new_addr0, old_addr0;
1981           u16 new_port0, old_port0;
1982           udp_header_t * udp0;
1983           tcp_header_t * tcp0;
1984           icmp46_header_t * icmp0;
1985           snat_session_key_t key0, sm0;
1986           u32 proto0;
1987           u32 rx_fib_index0;
1988
1989           /* speculatively enqueue b0 to the current next frame */
1990           bi0 = from[0];
1991           to_next[0] = bi0;
1992           from += 1;
1993           to_next += 1;
1994           n_left_from -= 1;
1995           n_left_to_next -= 1;
1996
1997           b0 = vlib_get_buffer (vm, bi0);
1998
1999           ip0 = vlib_buffer_get_current (b0);
2000           udp0 = ip4_next_header (ip0);
2001           tcp0 = (tcp_header_t *) udp0;
2002           icmp0 = (icmp46_header_t *) udp0;
2003
2004           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2005           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2006
2007           vnet_feature_next (sw_if_index0, &next0, b0);
2008
2009           if (PREDICT_FALSE(ip0->ttl == 1))
2010             {
2011               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2012               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2013                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2014                                            0);
2015               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2016               goto trace00;
2017             }
2018
2019           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2020
2021           if (PREDICT_FALSE (proto0 == ~0))
2022               goto trace00;
2023
2024           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2025             {
2026               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2027                                   rx_fib_index0, node, next0, ~0, 0, 0);
2028               goto trace00;
2029             }
2030
2031           key0.addr = ip0->dst_address;
2032           key0.port = udp0->dst_port;
2033           key0.fib_index = rx_fib_index0;
2034
2035           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2036             {
2037               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2038               goto trace00;
2039             }
2040
2041           new_addr0 = sm0.addr.as_u32;
2042           new_port0 = sm0.port;
2043           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2044           old_addr0 = ip0->dst_address.as_u32;
2045           ip0->dst_address.as_u32 = new_addr0;
2046
2047           sum0 = ip0->checksum;
2048           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2049                                  ip4_header_t,
2050                                  dst_address /* changed member */);
2051           ip0->checksum = ip_csum_fold (sum0);
2052
2053           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2054             {
2055                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2056                 {
2057                   old_port0 = tcp0->dst_port;
2058                   tcp0->dst_port = new_port0;
2059
2060                   sum0 = tcp0->checksum;
2061                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2062                                          ip4_header_t,
2063                                          dst_address /* changed member */);
2064
2065                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2066                                          ip4_header_t /* cheat */,
2067                                          length /* changed member */);
2068                   tcp0->checksum = ip_csum_fold(sum0);
2069                 }
2070               else
2071                 {
2072                   old_port0 = udp0->dst_port;
2073                   udp0->dst_port = new_port0;
2074                   udp0->checksum = 0;
2075                 }
2076             }
2077           else
2078             {
2079               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2080                 {
2081                   sum0 = tcp0->checksum;
2082                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2083                                          ip4_header_t,
2084                                          dst_address /* changed member */);
2085
2086                   tcp0->checksum = ip_csum_fold(sum0);
2087                 }
2088             }
2089
2090         trace00:
2091
2092           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2093                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2094             {
2095               snat_out2in_trace_t *t =
2096                  vlib_add_trace (vm, node, b0, sizeof (*t));
2097               t->sw_if_index = sw_if_index0;
2098               t->next_index = next0;
2099             }
2100
2101           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2102
2103           /* verify speculative enqueue, maybe switch current next frame */
2104           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2105                                            to_next, n_left_to_next,
2106                                            bi0, next0);
2107         }
2108
2109       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2110     }
2111
2112   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2113                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2114                                pkts_processed);
2115   return frame->n_vectors;
2116 }
2117
2118 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2119   .function = snat_out2in_fast_node_fn,
2120   .name = "snat-out2in-fast",
2121   .vector_size = sizeof (u32),
2122   .format_trace = format_snat_out2in_fast_trace,
2123   .type = VLIB_NODE_TYPE_INTERNAL,
2124
2125   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2126   .error_strings = snat_out2in_error_strings,
2127
2128   .runtime_data_bytes = sizeof (snat_runtime_t),
2129
2130   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2131
2132   /* edit / add dispositions here */
2133   .next_nodes = {
2134     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2135     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2136     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2137   },
2138 };
2139 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);