src/plugins/snat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <snat/snat.h>
  26 #include <snat/snat_ipfix_logging.h>
  27 #include <snat/snat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     SNAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out SNAT session key.
 121  * @param out2in Out2in SNAT session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140
 141   user_key.addr = in2out.addr;
 142   user_key.fib_index = in2out.fib_index;
 143   kv0.key = user_key.as_u64;
 144
 145   /* Ever heard of the "user" = inside ip4 address before? */
 146   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 147     {
 148       /* no, make a new one */
 149       pool_get (sm->per_thread_data[thread_index].users, u);
 150       memset (u, 0, sizeof (*u));
 151       u->addr = in2out.addr;
 152       u->fib_index = in2out.fib_index;
 153
 154       pool_get (sm->per_thread_data[thread_index].list_pool,
 155                 per_user_list_head_elt);
 156
 157       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 158         sm->per_thread_data[thread_index].list_pool;
 159
 160       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 161                        u->sessions_per_user_list_head_index);
 162
 163       kv0.value = u - sm->per_thread_data[thread_index].users;
 164
 165       /* add user */
 166       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 167
 168       /* add non-traslated packets worker lookup */
 169       kv0.value = thread_index;
 170       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 171     }
 172   else
 173     {
 174       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 175                              value0.value);
 176     }
 177
 178   pool_get (sm->per_thread_data[thread_index].sessions, s);
 179   memset (s, 0, sizeof (*s));
 180
 181   s->outside_address_index = ~0;
 182   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 183   u->nstaticsessions++;
 184
 185   /* Create list elts */
 186   pool_get (sm->per_thread_data[thread_index].list_pool,
 187             per_user_translation_list_elt);
 188   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 189                    per_user_translation_list_elt -
 190                    sm->per_thread_data[thread_index].list_pool);
 191
 192   per_user_translation_list_elt->value =
 193     s - sm->per_thread_data[thread_index].sessions;
 194   s->per_user_index =
 195     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 196   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 197
 198   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 199                       s->per_user_list_head_index,
 200                       per_user_translation_list_elt -
 201                       sm->per_thread_data[thread_index].list_pool);
 202
 203   s->in2out = in2out;
 204   s->out2in = out2in;
 205   s->in2out.protocol = out2in.protocol;
 206
 207   /* Add to translation hashes */
 208   kv0.key = s->in2out.as_u64;
 209   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 210   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 211       clib_warning ("in2out key add failed");
 212
 213   kv0.key = s->out2in.as_u64;
 214   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 215
 216   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 217       clib_warning ("out2in key add failed");
 218
 219   /* log NAT event */
 220   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 221                                       s->out2in.addr.as_u32,
 222                                       s->in2out.protocol,
 223                                       s->in2out.port,
 224                                       s->out2in.port,
 225                                       s->in2out.fib_index);
 226    return s;
 227 }
 228
 229 static_always_inline
 230 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 231                                  snat_session_key_t *p_key0)
 232 {
 233   icmp46_header_t *icmp0;
 234   snat_session_key_t key0;
 235   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 236   ip4_header_t *inner_ip0;
 237   void *l4_header = 0;
 238   icmp46_header_t *inner_icmp0;
 239
 240   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 241   echo0 = (icmp_echo_header_t *)(icmp0+1);
 242
 243   if (!icmp_is_error_message (icmp0))
 244     {
 245       key0.protocol = SNAT_PROTOCOL_ICMP;
 246       key0.addr = ip0->dst_address;
 247       key0.port = echo0->identifier;
 248     }
 249   else
 250     {
 251       inner_ip0 = (ip4_header_t *)(echo0+1);
 252       l4_header = ip4_next_header (inner_ip0);
 253       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 254       key0.addr = inner_ip0->src_address;
 255       switch (key0.protocol)
 256         {
 257         case SNAT_PROTOCOL_ICMP:
 258           inner_icmp0 = (icmp46_header_t*)l4_header;
 259           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 260           key0.port = inner_echo0->identifier;
 261           break;
 262         case SNAT_PROTOCOL_UDP:
 263         case SNAT_PROTOCOL_TCP:
 264           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 265           break;
 266         default:
 267           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 268         }
 269     }
 270   *p_key0 = key0;
 271   return -1; /* success */
 272 }
 273
 274 /**
 275  * Get address and port values to be used for packet SNAT translation
 276  * and create session if needed
 277  *
 278  * @param[in,out] sm             SNAT main
 279  * @param[in,out] node           SNAT node runtime
 280  * @param[in] thread_index       thread index
 281  * @param[in,out] b0             buffer containing packet to be translated
 282  * @param[out] p_proto           protocol used for matching
 283  * @param[out] p_value           address and port after NAT translation
 284  * @param[out] p_dont_translate  if packet should not be translated
 285  * @param d                      optional parameter
 286  * @param e                      optional parameter
 287  */
 288 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 289                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 290                            snat_session_key_t *p_value,
 291                            u8 *p_dont_translate, void *d, void *e)
 292 {
 293   ip4_header_t *ip0;
 294   icmp46_header_t *icmp0;
 295   u32 sw_if_index0;
 296   u32 rx_fib_index0;
 297   snat_session_key_t key0;
 298   snat_session_key_t sm0;
 299   snat_session_t *s0 = 0;
 300   u8 dont_translate = 0;
 301   clib_bihash_kv_8_8_t kv0, value0;
 302   u8 is_addr_only;
 303   u32 next0 = ~0;
 304   int err;
 305
 306   ip0 = vlib_buffer_get_current (b0);
 307   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 308   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 309   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 310
 311   key0.protocol = 0;
 312
 313   err = icmp_get_key (ip0, &key0);
 314   if (err != -1)
 315     {
 316       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 317       next0 = SNAT_OUT2IN_NEXT_DROP;
 318       goto out;
 319     }
 320   key0.fib_index = rx_fib_index0;
 321
 322   kv0.key = key0.as_u64;
 323
 324   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 325     {
 326       /* Try to match static mapping by external address and port,
 327          destination address and port in packet */
 328       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 329         {
 330           /* Don't NAT packet aimed at the intfc address */
 331           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 332                                               ip0->dst_address.as_u32)))
 333             {
 334               dont_translate = 1;
 335               goto out;
 336             }
 337           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 338           next0 = SNAT_OUT2IN_NEXT_DROP;
 339           goto out;
 340         }
 341
 342       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 343                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 344         {
 345           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 346           next0 = SNAT_OUT2IN_NEXT_DROP;
 347           goto out;
 348         }
 349
 350       /* Create session initiated by host from external network */
 351       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 352                                              node, thread_index);
 353
 354       if (!s0)
 355         {
 356           next0 = SNAT_OUT2IN_NEXT_DROP;
 357           goto out;
 358         }
 359     }
 360   else
 361     {
 362       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 363                         icmp0->type != ICMP4_echo_request &&
 364                         !icmp_is_error_message (icmp0)))
 365         {
 366           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 367           next0 = SNAT_OUT2IN_NEXT_DROP;
 368           goto out;
 369         }
 370
 371       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 372                               value0.value);
 373     }
 374
 375 out:
 376   *p_proto = key0.protocol;
 377   if (s0)
 378     *p_value = s0->in2out;
 379   *p_dont_translate = dont_translate;
 380   if (d)
 381     *(snat_session_t**)d = s0;
 382   return next0;
 383 }
 384
 385 /**
 386  * Get address and port values to be used for packet SNAT translation
 387  *
 388  * @param[in] sm                 SNAT main
 389  * @param[in,out] node           SNAT node runtime
 390  * @param[in] thread_index       thread index
 391  * @param[in,out] b0             buffer containing packet to be translated
 392  * @param[out] p_proto           protocol used for matching
 393  * @param[out] p_value           address and port after NAT translation
 394  * @param[out] p_dont_translate  if packet should not be translated
 395  * @param d                      optional parameter
 396  * @param e                      optional parameter
 397  */
 398 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 399                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 400                            snat_session_key_t *p_value,
 401                            u8 *p_dont_translate, void *d, void *e)
 402 {
 403   ip4_header_t *ip0;
 404   icmp46_header_t *icmp0;
 405   u32 sw_if_index0;
 406   u32 rx_fib_index0;
 407   snat_session_key_t key0;
 408   snat_session_key_t sm0;
 409   u8 dont_translate = 0;
 410   u8 is_addr_only;
 411   u32 next0 = ~0;
 412   int err;
 413
 414   ip0 = vlib_buffer_get_current (b0);
 415   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 416   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 417   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 418
 419   err = icmp_get_key (ip0, &key0);
 420   if (err != -1)
 421     {
 422       b0->error = node->errors[err];
 423       next0 = SNAT_OUT2IN_NEXT_DROP;
 424       goto out2;
 425     }
 426   key0.fib_index = rx_fib_index0;
 427
 428   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 429     {
 430       /* Don't NAT packet aimed at the intfc address */
 431       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 432         {
 433           dont_translate = 1;
 434           goto out;
 435         }
 436       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 437       next0 = SNAT_OUT2IN_NEXT_DROP;
 438       goto out;
 439     }
 440
 441   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 442                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 443                     !icmp_is_error_message (icmp0)))
 444     {
 445       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 446       next0 = SNAT_OUT2IN_NEXT_DROP;
 447       goto out;
 448     }
 449
 450 out:
 451   *p_value = sm0;
 452 out2:
 453   *p_proto = key0.protocol;
 454   *p_dont_translate = dont_translate;
 455   return next0;
 456 }
 457
 458 static inline u32 icmp_out2in (snat_main_t *sm,
 459                                vlib_buffer_t * b0,
 460                                ip4_header_t * ip0,
 461                                icmp46_header_t * icmp0,
 462                                u32 sw_if_index0,
 463                                u32 rx_fib_index0,
 464                                vlib_node_runtime_t * node,
 465                                u32 next0,
 466                                u32 thread_index,
 467                                void *d,
 468                                void *e)
 469 {
 470   snat_session_key_t sm0;
 471   u8 protocol;
 472   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 473   ip4_header_t *inner_ip0 = 0;
 474   void *l4_header = 0;
 475   icmp46_header_t *inner_icmp0;
 476   u8 dont_translate;
 477   u32 new_addr0, old_addr0;
 478   u16 old_id0, new_id0;
 479   ip_csum_t sum0;
 480   u16 checksum0;
 481   u32 next0_tmp;
 482
 483   echo0 = (icmp_echo_header_t *)(icmp0+1);
 484
 485   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 486                                        &protocol, &sm0, &dont_translate, d, e);
 487   if (next0_tmp != ~0)
 488     next0 = next0_tmp;
 489   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 490     goto out;
 491
 492   sum0 = ip_incremental_checksum (0, icmp0,
 493                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 494   checksum0 = ~ip_csum_fold (sum0);
 495   if (checksum0 != 0 && checksum0 != 0xffff)
 496     {
 497       next0 = SNAT_OUT2IN_NEXT_DROP;
 498       goto out;
 499     }
 500
 501   old_addr0 = ip0->dst_address.as_u32;
 502   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 503   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 504
 505   sum0 = ip0->checksum;
 506   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 507                          dst_address /* changed member */);
 508   ip0->checksum = ip_csum_fold (sum0);
 509
 510   if (!icmp_is_error_message (icmp0))
 511     {
 512       new_id0 = sm0.port;
 513       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 514         {
 515           old_id0 = echo0->identifier;
 516           new_id0 = sm0.port;
 517           echo0->identifier = new_id0;
 518
 519           sum0 = icmp0->checksum;
 520           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 521                                  identifier /* changed member */);
 522           icmp0->checksum = ip_csum_fold (sum0);
 523         }
 524     }
 525   else
 526     {
 527       inner_ip0 = (ip4_header_t *)(echo0+1);
 528       l4_header = ip4_next_header (inner_ip0);
 529
 530       if (!ip4_header_checksum_is_valid (inner_ip0))
 531         {
 532           next0 = SNAT_OUT2IN_NEXT_DROP;
 533           goto out;
 534         }
 535
 536       old_addr0 = inner_ip0->src_address.as_u32;
 537       inner_ip0->src_address = sm0.addr;
 538       new_addr0 = inner_ip0->src_address.as_u32;
 539
 540       sum0 = icmp0->checksum;
 541       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 542                              src_address /* changed member */);
 543       icmp0->checksum = ip_csum_fold (sum0);
 544
 545       switch (protocol)
 546         {
 547         case SNAT_PROTOCOL_ICMP:
 548           inner_icmp0 = (icmp46_header_t*)l4_header;
 549           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 550
 551           old_id0 = inner_echo0->identifier;
 552           new_id0 = sm0.port;
 553           inner_echo0->identifier = new_id0;
 554
 555           sum0 = icmp0->checksum;
 556           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 557                                  identifier);
 558           icmp0->checksum = ip_csum_fold (sum0);
 559           break;
 560         case SNAT_PROTOCOL_UDP:
 561         case SNAT_PROTOCOL_TCP:
 562           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 563           new_id0 = sm0.port;
 564           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 565
 566           sum0 = icmp0->checksum;
 567           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 568                                  src_port);
 569           icmp0->checksum = ip_csum_fold (sum0);
 570           break;
 571         default:
 572           ASSERT(0);
 573         }
 574     }
 575
 576 out:
 577   return next0;
 578 }
 579
 580
 581 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 582                                          vlib_buffer_t * b0,
 583                                          ip4_header_t * ip0,
 584                                          icmp46_header_t * icmp0,
 585                                          u32 sw_if_index0,
 586                                          u32 rx_fib_index0,
 587                                          vlib_node_runtime_t * node,
 588                                          u32 next0, f64 now,
 589                                          u32 thread_index,
 590                                          snat_session_t ** p_s0)
 591 {
 592   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 593                       next0, thread_index, p_s0, 0);
 594   snat_session_t * s0 = *p_s0;
 595   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 596     {
 597       /* Accounting */
 598       s0->last_heard = now;
 599       s0->total_pkts++;
 600       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 601       /* Per-user LRU list maintenance for dynamic translation */
 602       if (!snat_is_session_static (s0))
 603         {
 604           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 605                              s0->per_user_index);
 606           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 607                               s0->per_user_list_head_index,
 608                               s0->per_user_index);
 609         }
 610     }
 611   return next0;
 612 }
 613
 614 static void
 615 snat_out2in_unknown_proto (snat_main_t *sm,
 616                            vlib_buffer_t * b,
 617                            ip4_header_t * ip,
 618                            u32 rx_fib_index)
 619 {
 620   clib_bihash_kv_8_8_t kv, value;
 621   snat_static_mapping_t *m;
 622   snat_session_key_t m_key;
 623   u32 old_addr, new_addr;
 624   ip_csum_t sum;
 625
 626   m_key.addr = ip->dst_address;
 627   m_key.port = 0;
 628   m_key.protocol = 0;
 629   m_key.fib_index = rx_fib_index;
 630   kv.key = m_key.as_u64;
 631   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 632     return;
 633
 634   m = pool_elt_at_index (sm->static_mappings, value.value);
 635
 636   old_addr = ip->dst_address.as_u32;
 637   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 638   sum = ip->checksum;
 639   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 640   ip->checksum = ip_csum_fold (sum);
 641
 642   vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 643 }
 644
 645 static uword
 646 snat_out2in_node_fn (vlib_main_t * vm,
 647                   vlib_node_runtime_t * node,
 648                   vlib_frame_t * frame)
 649 {
 650   u32 n_left_from, * from, * to_next;
 651   snat_out2in_next_t next_index;
 652   u32 pkts_processed = 0;
 653   snat_main_t * sm = &snat_main;
 654   f64 now = vlib_time_now (vm);
 655   u32 thread_index = vlib_get_thread_index ();
 656
 657   from = vlib_frame_vector_args (frame);
 658   n_left_from = frame->n_vectors;
 659   next_index = node->cached_next_index;
 660
 661   while (n_left_from > 0)
 662     {
 663       u32 n_left_to_next;
 664
 665       vlib_get_next_frame (vm, node, next_index,
 666                            to_next, n_left_to_next);
 667
 668       while (n_left_from >= 4 && n_left_to_next >= 2)
 669         {
 670           u32 bi0, bi1;
 671           vlib_buffer_t * b0, * b1;
 672           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 673           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 674           u32 sw_if_index0, sw_if_index1;
 675           ip4_header_t * ip0, *ip1;
 676           ip_csum_t sum0, sum1;
 677           u32 new_addr0, old_addr0;
 678           u16 new_port0, old_port0;
 679           u32 new_addr1, old_addr1;
 680           u16 new_port1, old_port1;
 681           udp_header_t * udp0, * udp1;
 682           tcp_header_t * tcp0, * tcp1;
 683           icmp46_header_t * icmp0, * icmp1;
 684           snat_session_key_t key0, key1, sm0, sm1;
 685           u32 rx_fib_index0, rx_fib_index1;
 686           u32 proto0, proto1;
 687           snat_session_t * s0 = 0, * s1 = 0;
 688           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 689
 690           /* Prefetch next iteration. */
 691           {
 692             vlib_buffer_t * p2, * p3;
 693
 694             p2 = vlib_get_buffer (vm, from[2]);
 695             p3 = vlib_get_buffer (vm, from[3]);
 696
 697             vlib_prefetch_buffer_header (p2, LOAD);
 698             vlib_prefetch_buffer_header (p3, LOAD);
 699
 700             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 701             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 702           }
 703
 704           /* speculatively enqueue b0 and b1 to the current next frame */
 705           to_next[0] = bi0 = from[0];
 706           to_next[1] = bi1 = from[1];
 707           from += 2;
 708           to_next += 2;
 709           n_left_from -= 2;
 710           n_left_to_next -= 2;
 711
 712           b0 = vlib_get_buffer (vm, bi0);
 713           b1 = vlib_get_buffer (vm, bi1);
 714
 715           ip0 = vlib_buffer_get_current (b0);
 716           udp0 = ip4_next_header (ip0);
 717           tcp0 = (tcp_header_t *) udp0;
 718           icmp0 = (icmp46_header_t *) udp0;
 719
 720           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 721           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 722                                    sw_if_index0);
 723
 724           if (PREDICT_FALSE(ip0->ttl == 1))
 725             {
 726               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 727               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 728                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 729                                            0);
 730               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 731               goto trace0;
 732             }
 733
 734           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 735
 736           if (PREDICT_FALSE (proto0 == ~0))
 737             {
 738               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0);
 739               goto trace0;
 740             }
 741
 742           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 743             {
 744               next0 = icmp_out2in_slow_path
 745                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 746                  next0, now, thread_index, &s0);
 747               goto trace0;
 748             }
 749
 750           key0.addr = ip0->dst_address;
 751           key0.port = udp0->dst_port;
 752           key0.protocol = proto0;
 753           key0.fib_index = rx_fib_index0;
 754
 755           kv0.key = key0.as_u64;
 756
 757           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 758             {
 759               /* Try to match static mapping by external address and port,
 760                  destination address and port in packet */
 761               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
 762                 {
 763                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 764                   /*
 765                    * Send DHCP packets to the ipv4 stack, or we won't
 766                    * be able to use dhcp client on the outside interface
 767                    */
 768                   if (proto0 != SNAT_PROTOCOL_UDP
 769                       || (udp0->dst_port
 770                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 771                     next0 = SNAT_OUT2IN_NEXT_DROP;
 772                   goto trace0;
 773                 }
 774
 775               /* Create session initiated by host from external network */
 776               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
 777                                                      thread_index);
 778               if (!s0)
 779                 {
 780                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 781                   next0 = SNAT_OUT2IN_NEXT_DROP;
 782                   goto trace0;
 783                 }
 784             }
 785           else
 786             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 787                                     value0.value);
 788
 789           old_addr0 = ip0->dst_address.as_u32;
 790           ip0->dst_address = s0->in2out.addr;
 791           new_addr0 = ip0->dst_address.as_u32;
 792           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 793
 794           sum0 = ip0->checksum;
 795           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 796                                  ip4_header_t,
 797                                  dst_address /* changed member */);
 798           ip0->checksum = ip_csum_fold (sum0);
 799
 800           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 801             {
 802               old_port0 = tcp0->dst_port;
 803               tcp0->dst_port = s0->in2out.port;
 804               new_port0 = tcp0->dst_port;
 805
 806               sum0 = tcp0->checksum;
 807               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 808                                      ip4_header_t,
 809                                      dst_address /* changed member */);
 810
 811               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 812                                      ip4_header_t /* cheat */,
 813                                      length /* changed member */);
 814               tcp0->checksum = ip_csum_fold(sum0);
 815             }
 816           else
 817             {
 818               old_port0 = udp0->dst_port;
 819               udp0->dst_port = s0->in2out.port;
 820               udp0->checksum = 0;
 821             }
 822
 823           /* Accounting */
 824           s0->last_heard = now;
 825           s0->total_pkts++;
 826           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
 827           /* Per-user LRU list maintenance for dynamic translation */
 828           if (!snat_is_session_static (s0))
 829             {
 830               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 831                                  s0->per_user_index);
 832               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 833                                   s0->per_user_list_head_index,
 834                                   s0->per_user_index);
 835             }
 836         trace0:
 837
 838           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 839                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 840             {
 841               snat_out2in_trace_t *t =
 842                  vlib_add_trace (vm, node, b0, sizeof (*t));
 843               t->sw_if_index = sw_if_index0;
 844               t->next_index = next0;
 845               t->session_index = ~0;
 846               if (s0)
 847                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
 848             }
 849
 850           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 851
 852
 853           ip1 = vlib_buffer_get_current (b1);
 854           udp1 = ip4_next_header (ip1);
 855           tcp1 = (tcp_header_t *) udp1;
 856           icmp1 = (icmp46_header_t *) udp1;
 857
 858           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 859           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 860                                    sw_if_index1);
 861
 862           if (PREDICT_FALSE(ip1->ttl == 1))
 863             {
 864               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 865               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 866                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 867                                            0);
 868               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 869               goto trace1;
 870             }
 871
 872           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 873
 874           if (PREDICT_FALSE (proto1 == ~0))
 875             {
 876               snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1);
 877               goto trace1;
 878             }
 879
 880           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 881             {
 882               next1 = icmp_out2in_slow_path
 883                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 884                  next1, now, thread_index, &s1);
 885               goto trace1;
 886             }
 887
 888           key1.addr = ip1->dst_address;
 889           key1.port = udp1->dst_port;
 890           key1.protocol = proto1;
 891           key1.fib_index = rx_fib_index1;
 892
 893           kv1.key = key1.as_u64;
 894
 895           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
 896             {
 897               /* Try to match static mapping by external address and port,
 898                  destination address and port in packet */
 899               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
 900                 {
 901                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 902                   /*
 903                    * Send DHCP packets to the ipv4 stack, or we won't
 904                    * be able to use dhcp client on the outside interface
 905                    */
 906                   if (proto1 != SNAT_PROTOCOL_UDP
 907                       || (udp1->dst_port
 908                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
 909                     next1 = SNAT_OUT2IN_NEXT_DROP;
 910                   goto trace1;
 911                 }
 912
 913               /* Create session initiated by host from external network */
 914               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
 915                                                      thread_index);
 916               if (!s1)
 917                 {
 918                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 919                   next1 = SNAT_OUT2IN_NEXT_DROP;
 920                   goto trace1;
 921                 }
 922             }
 923           else
 924             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 925                                     value1.value);
 926
 927           old_addr1 = ip1->dst_address.as_u32;
 928           ip1->dst_address = s1->in2out.addr;
 929           new_addr1 = ip1->dst_address.as_u32;
 930           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 931
 932           sum1 = ip1->checksum;
 933           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 934                                  ip4_header_t,
 935                                  dst_address /* changed member */);
 936           ip1->checksum = ip_csum_fold (sum1);
 937
 938           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
 939             {
 940               old_port1 = tcp1->dst_port;
 941               tcp1->dst_port = s1->in2out.port;
 942               new_port1 = tcp1->dst_port;
 943
 944               sum1 = tcp1->checksum;
 945               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 946                                      ip4_header_t,
 947                                      dst_address /* changed member */);
 948
 949               sum1 = ip_csum_update (sum1, old_port1, new_port1,
 950                                      ip4_header_t /* cheat */,
 951                                      length /* changed member */);
 952               tcp1->checksum = ip_csum_fold(sum1);
 953             }
 954           else
 955             {
 956               old_port1 = udp1->dst_port;
 957               udp1->dst_port = s1->in2out.port;
 958               udp1->checksum = 0;
 959             }
 960
 961           /* Accounting */
 962           s1->last_heard = now;
 963           s1->total_pkts++;
 964           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
 965           /* Per-user LRU list maintenance for dynamic translation */
 966           if (!snat_is_session_static (s1))
 967             {
 968               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 969                                  s1->per_user_index);
 970               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 971                                   s1->per_user_list_head_index,
 972                                   s1->per_user_index);
 973             }
 974         trace1:
 975
 976           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 977                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
 978             {
 979               snat_out2in_trace_t *t =
 980                  vlib_add_trace (vm, node, b1, sizeof (*t));
 981               t->sw_if_index = sw_if_index1;
 982               t->next_index = next1;
 983               t->session_index = ~0;
 984               if (s1)
 985                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
 986             }
 987
 988           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
 989
 990           /* verify speculative enqueues, maybe switch current next frame */
 991           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 992                                            to_next, n_left_to_next,
 993                                            bi0, bi1, next0, next1);
 994         }
 995
 996       while (n_left_from > 0 && n_left_to_next > 0)
 997         {
 998           u32 bi0;
 999           vlib_buffer_t * b0;
1000           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1001           u32 sw_if_index0;
1002           ip4_header_t * ip0;
1003           ip_csum_t sum0;
1004           u32 new_addr0, old_addr0;
1005           u16 new_port0, old_port0;
1006           udp_header_t * udp0;
1007           tcp_header_t * tcp0;
1008           icmp46_header_t * icmp0;
1009           snat_session_key_t key0, sm0;
1010           u32 rx_fib_index0;
1011           u32 proto0;
1012           snat_session_t * s0 = 0;
1013           clib_bihash_kv_8_8_t kv0, value0;
1014
1015           /* speculatively enqueue b0 to the current next frame */
1016           bi0 = from[0];
1017           to_next[0] = bi0;
1018           from += 1;
1019           to_next += 1;
1020           n_left_from -= 1;
1021           n_left_to_next -= 1;
1022
1023           b0 = vlib_get_buffer (vm, bi0);
1024
1025           ip0 = vlib_buffer_get_current (b0);
1026           udp0 = ip4_next_header (ip0);
1027           tcp0 = (tcp_header_t *) udp0;
1028           icmp0 = (icmp46_header_t *) udp0;
1029
1030           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1031           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1032                                    sw_if_index0);
1033
1034           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1035
1036           if (PREDICT_FALSE (proto0 == ~0))
1037             {
1038               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0);
1039               goto trace00;
1040             }
1041
1042           if (PREDICT_FALSE(ip0->ttl == 1))
1043             {
1044               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1045               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1046                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1047                                            0);
1048               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1049               goto trace00;
1050             }
1051
1052           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1053             {
1054               next0 = icmp_out2in_slow_path
1055                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1056                  next0, now, thread_index, &s0);
1057               goto trace00;
1058             }
1059
1060           key0.addr = ip0->dst_address;
1061           key0.port = udp0->dst_port;
1062           key0.protocol = proto0;
1063           key0.fib_index = rx_fib_index0;
1064
1065           kv0.key = key0.as_u64;
1066
1067           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1068             {
1069               /* Try to match static mapping by external address and port,
1070                  destination address and port in packet */
1071               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1072                 {
1073                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1074                   /*
1075                    * Send DHCP packets to the ipv4 stack, or we won't
1076                    * be able to use dhcp client on the outside interface
1077                    */
1078                   if (proto0 != SNAT_PROTOCOL_UDP
1079                       || (udp0->dst_port
1080                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1081
1082                     next0 = SNAT_OUT2IN_NEXT_DROP;
1083                   goto trace00;
1084                 }
1085
1086               /* Create session initiated by host from external network */
1087               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1088                                                      thread_index);
1089               if (!s0)
1090                 {
1091                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1092                     next0 = SNAT_OUT2IN_NEXT_DROP;
1093                   goto trace00;
1094                 }
1095             }
1096           else
1097             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1098                                     value0.value);
1099
1100           old_addr0 = ip0->dst_address.as_u32;
1101           ip0->dst_address = s0->in2out.addr;
1102           new_addr0 = ip0->dst_address.as_u32;
1103           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1104
1105           sum0 = ip0->checksum;
1106           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1107                                  ip4_header_t,
1108                                  dst_address /* changed member */);
1109           ip0->checksum = ip_csum_fold (sum0);
1110
1111           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1112             {
1113               old_port0 = tcp0->dst_port;
1114               tcp0->dst_port = s0->in2out.port;
1115               new_port0 = tcp0->dst_port;
1116
1117               sum0 = tcp0->checksum;
1118               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1119                                      ip4_header_t,
1120                                      dst_address /* changed member */);
1121
1122               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1123                                      ip4_header_t /* cheat */,
1124                                      length /* changed member */);
1125               tcp0->checksum = ip_csum_fold(sum0);
1126             }
1127           else
1128             {
1129               old_port0 = udp0->dst_port;
1130               udp0->dst_port = s0->in2out.port;
1131               udp0->checksum = 0;
1132             }
1133
1134           /* Accounting */
1135           s0->last_heard = now;
1136           s0->total_pkts++;
1137           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1138           /* Per-user LRU list maintenance for dynamic translation */
1139           if (!snat_is_session_static (s0))
1140             {
1141               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1142                                  s0->per_user_index);
1143               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1144                                   s0->per_user_list_head_index,
1145                                   s0->per_user_index);
1146             }
1147         trace00:
1148
1149           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1150                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1151             {
1152               snat_out2in_trace_t *t =
1153                  vlib_add_trace (vm, node, b0, sizeof (*t));
1154               t->sw_if_index = sw_if_index0;
1155               t->next_index = next0;
1156               t->session_index = ~0;
1157               if (s0)
1158                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1159             }
1160
1161           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1162
1163           /* verify speculative enqueue, maybe switch current next frame */
1164           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1165                                            to_next, n_left_to_next,
1166                                            bi0, next0);
1167         }
1168
1169       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1170     }
1171
1172   vlib_node_increment_counter (vm, snat_out2in_node.index,
1173                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1174                                pkts_processed);
1175   return frame->n_vectors;
1176 }
1177
1178 VLIB_REGISTER_NODE (snat_out2in_node) = {
1179   .function = snat_out2in_node_fn,
1180   .name = "snat-out2in",
1181   .vector_size = sizeof (u32),
1182   .format_trace = format_snat_out2in_trace,
1183   .type = VLIB_NODE_TYPE_INTERNAL,
1184
1185   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1186   .error_strings = snat_out2in_error_strings,
1187
1188   .runtime_data_bytes = sizeof (snat_runtime_t),
1189
1190   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1191
1192   /* edit / add dispositions here */
1193   .next_nodes = {
1194     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1195     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1196     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1197   },
1198 };
1199 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1200
1201 /**************************/
1202 /*** deterministic mode ***/
1203 /**************************/
1204 static uword
1205 snat_det_out2in_node_fn (vlib_main_t * vm,
1206                          vlib_node_runtime_t * node,
1207                          vlib_frame_t * frame)
1208 {
1209   u32 n_left_from, * from, * to_next;
1210   snat_out2in_next_t next_index;
1211   u32 pkts_processed = 0;
1212   snat_main_t * sm = &snat_main;
1213   u32 thread_index = vlib_get_thread_index ();
1214
1215   from = vlib_frame_vector_args (frame);
1216   n_left_from = frame->n_vectors;
1217   next_index = node->cached_next_index;
1218
1219   while (n_left_from > 0)
1220     {
1221       u32 n_left_to_next;
1222
1223       vlib_get_next_frame (vm, node, next_index,
1224                            to_next, n_left_to_next);
1225
1226       while (n_left_from >= 4 && n_left_to_next >= 2)
1227         {
1228           u32 bi0, bi1;
1229           vlib_buffer_t * b0, * b1;
1230           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1231           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1232           u32 sw_if_index0, sw_if_index1;
1233           ip4_header_t * ip0, * ip1;
1234           ip_csum_t sum0, sum1;
1235           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1236           u16 new_port0, old_port0, old_port1, new_port1;
1237           udp_header_t * udp0, * udp1;
1238           tcp_header_t * tcp0, * tcp1;
1239           u32 proto0, proto1;
1240           snat_det_out_key_t key0, key1;
1241           snat_det_map_t * dm0, * dm1;
1242           snat_det_session_t * ses0 = 0, * ses1 = 0;
1243           u32 rx_fib_index0, rx_fib_index1;
1244           icmp46_header_t * icmp0, * icmp1;
1245
1246           /* Prefetch next iteration. */
1247           {
1248             vlib_buffer_t * p2, * p3;
1249
1250             p2 = vlib_get_buffer (vm, from[2]);
1251             p3 = vlib_get_buffer (vm, from[3]);
1252
1253             vlib_prefetch_buffer_header (p2, LOAD);
1254             vlib_prefetch_buffer_header (p3, LOAD);
1255
1256             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1257             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1258           }
1259
1260           /* speculatively enqueue b0 and b1 to the current next frame */
1261           to_next[0] = bi0 = from[0];
1262           to_next[1] = bi1 = from[1];
1263           from += 2;
1264           to_next += 2;
1265           n_left_from -= 2;
1266           n_left_to_next -= 2;
1267
1268           b0 = vlib_get_buffer (vm, bi0);
1269           b1 = vlib_get_buffer (vm, bi1);
1270
1271           ip0 = vlib_buffer_get_current (b0);
1272           udp0 = ip4_next_header (ip0);
1273           tcp0 = (tcp_header_t *) udp0;
1274
1275           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1276
1277           if (PREDICT_FALSE(ip0->ttl == 1))
1278             {
1279               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1280               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1281                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1282                                            0);
1283               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1284               goto trace0;
1285             }
1286
1287           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1288
1289           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1290             {
1291               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1292               icmp0 = (icmp46_header_t *) udp0;
1293
1294               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1295                                   rx_fib_index0, node, next0, thread_index,
1296                                   &ses0, &dm0);
1297               goto trace0;
1298             }
1299
1300           key0.ext_host_addr = ip0->src_address;
1301           key0.ext_host_port = tcp0->src;
1302           key0.out_port = tcp0->dst;
1303
1304           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1305           if (PREDICT_FALSE(!dm0))
1306             {
1307               clib_warning("unknown dst address:  %U",
1308                            format_ip4_address, &ip0->dst_address);
1309               next0 = SNAT_OUT2IN_NEXT_DROP;
1310               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1311               goto trace0;
1312             }
1313
1314           snat_det_reverse(dm0, &ip0->dst_address,
1315                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1316
1317           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1318           if (PREDICT_FALSE(!ses0))
1319             {
1320               clib_warning("no match src %U:%d dst %U:%d for user %U",
1321                            format_ip4_address, &ip0->src_address,
1322                            clib_net_to_host_u16 (tcp0->src),
1323                            format_ip4_address, &ip0->dst_address,
1324                            clib_net_to_host_u16 (tcp0->dst),
1325                            format_ip4_address, &new_addr0);
1326               next0 = SNAT_OUT2IN_NEXT_DROP;
1327               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1328               goto trace0;
1329             }
1330           new_port0 = ses0->in_port;
1331
1332           old_addr0 = ip0->dst_address;
1333           ip0->dst_address = new_addr0;
1334           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1335
1336           sum0 = ip0->checksum;
1337           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1338                                  ip4_header_t,
1339                                  dst_address /* changed member */);
1340           ip0->checksum = ip_csum_fold (sum0);
1341
1342           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1343             {
1344               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1345                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1346               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1347                 snat_det_ses_close(dm0, ses0);
1348
1349               old_port0 = tcp0->dst;
1350               tcp0->dst = new_port0;
1351
1352               sum0 = tcp0->checksum;
1353               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1354                                      ip4_header_t,
1355                                      dst_address /* changed member */);
1356
1357               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1358                                      ip4_header_t /* cheat */,
1359                                      length /* changed member */);
1360               tcp0->checksum = ip_csum_fold(sum0);
1361             }
1362           else
1363             {
1364               old_port0 = udp0->dst_port;
1365               udp0->dst_port = new_port0;
1366               udp0->checksum = 0;
1367             }
1368
1369         trace0:
1370
1371           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1372                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1373             {
1374               snat_out2in_trace_t *t =
1375                  vlib_add_trace (vm, node, b0, sizeof (*t));
1376               t->sw_if_index = sw_if_index0;
1377               t->next_index = next0;
1378               t->session_index = ~0;
1379               if (ses0)
1380                 t->session_index = ses0 - dm0->sessions;
1381             }
1382
1383           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1384
1385           b1 = vlib_get_buffer (vm, bi1);
1386
1387           ip1 = vlib_buffer_get_current (b1);
1388           udp1 = ip4_next_header (ip1);
1389           tcp1 = (tcp_header_t *) udp1;
1390
1391           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1392
1393           if (PREDICT_FALSE(ip1->ttl == 1))
1394             {
1395               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1396               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1397                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1398                                            0);
1399               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1400               goto trace1;
1401             }
1402
1403           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1404
1405           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1406             {
1407               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1408               icmp1 = (icmp46_header_t *) udp1;
1409
1410               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1411                                   rx_fib_index1, node, next1, thread_index,
1412                                   &ses1, &dm1);
1413               goto trace1;
1414             }
1415
1416           key1.ext_host_addr = ip1->src_address;
1417           key1.ext_host_port = tcp1->src;
1418           key1.out_port = tcp1->dst;
1419
1420           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1421           if (PREDICT_FALSE(!dm1))
1422             {
1423               clib_warning("unknown dst address:  %U",
1424                            format_ip4_address, &ip1->dst_address);
1425               next1 = SNAT_OUT2IN_NEXT_DROP;
1426               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1427               goto trace1;
1428             }
1429
1430           snat_det_reverse(dm1, &ip1->dst_address,
1431                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1432
1433           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1434           if (PREDICT_FALSE(!ses1))
1435             {
1436               clib_warning("no match src %U:%d dst %U:%d for user %U",
1437                            format_ip4_address, &ip1->src_address,
1438                            clib_net_to_host_u16 (tcp1->src),
1439                            format_ip4_address, &ip1->dst_address,
1440                            clib_net_to_host_u16 (tcp1->dst),
1441                            format_ip4_address, &new_addr1);
1442               next1 = SNAT_OUT2IN_NEXT_DROP;
1443               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1444               goto trace1;
1445             }
1446           new_port1 = ses1->in_port;
1447
1448           old_addr1 = ip1->dst_address;
1449           ip1->dst_address = new_addr1;
1450           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1451
1452           sum1 = ip1->checksum;
1453           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1454                                  ip4_header_t,
1455                                  dst_address /* changed member */);
1456           ip1->checksum = ip_csum_fold (sum1);
1457
1458           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1459             {
1460               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1461                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1462               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1463                 snat_det_ses_close(dm1, ses1);
1464
1465               old_port1 = tcp1->dst;
1466               tcp1->dst = new_port1;
1467
1468               sum1 = tcp1->checksum;
1469               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1470                                      ip4_header_t,
1471                                      dst_address /* changed member */);
1472
1473               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1474                                      ip4_header_t /* cheat */,
1475                                      length /* changed member */);
1476               tcp1->checksum = ip_csum_fold(sum1);
1477             }
1478           else
1479             {
1480               old_port1 = udp1->dst_port;
1481               udp1->dst_port = new_port1;
1482               udp1->checksum = 0;
1483             }
1484
1485         trace1:
1486
1487           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1488                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1489             {
1490               snat_out2in_trace_t *t =
1491                  vlib_add_trace (vm, node, b1, sizeof (*t));
1492               t->sw_if_index = sw_if_index1;
1493               t->next_index = next1;
1494               t->session_index = ~0;
1495               if (ses1)
1496                 t->session_index = ses1 - dm1->sessions;
1497             }
1498
1499           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1500
1501           /* verify speculative enqueues, maybe switch current next frame */
1502           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1503                                            to_next, n_left_to_next,
1504                                            bi0, bi1, next0, next1);
1505          }
1506
1507       while (n_left_from > 0 && n_left_to_next > 0)
1508         {
1509           u32 bi0;
1510           vlib_buffer_t * b0;
1511           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1512           u32 sw_if_index0;
1513           ip4_header_t * ip0;
1514           ip_csum_t sum0;
1515           ip4_address_t new_addr0, old_addr0;
1516           u16 new_port0, old_port0;
1517           udp_header_t * udp0;
1518           tcp_header_t * tcp0;
1519           u32 proto0;
1520           snat_det_out_key_t key0;
1521           snat_det_map_t * dm0;
1522           snat_det_session_t * ses0 = 0;
1523           u32 rx_fib_index0;
1524           icmp46_header_t * icmp0;
1525
1526           /* speculatively enqueue b0 to the current next frame */
1527           bi0 = from[0];
1528           to_next[0] = bi0;
1529           from += 1;
1530           to_next += 1;
1531           n_left_from -= 1;
1532           n_left_to_next -= 1;
1533
1534           b0 = vlib_get_buffer (vm, bi0);
1535
1536           ip0 = vlib_buffer_get_current (b0);
1537           udp0 = ip4_next_header (ip0);
1538           tcp0 = (tcp_header_t *) udp0;
1539
1540           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1541
1542           if (PREDICT_FALSE(ip0->ttl == 1))
1543             {
1544               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1545               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1546                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1547                                            0);
1548               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1549               goto trace00;
1550             }
1551
1552           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1553
1554           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1555             {
1556               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1557               icmp0 = (icmp46_header_t *) udp0;
1558
1559               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1560                                   rx_fib_index0, node, next0, thread_index,
1561                                   &ses0, &dm0);
1562               goto trace00;
1563             }
1564
1565           key0.ext_host_addr = ip0->src_address;
1566           key0.ext_host_port = tcp0->src;
1567           key0.out_port = tcp0->dst;
1568
1569           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1570           if (PREDICT_FALSE(!dm0))
1571             {
1572               clib_warning("unknown dst address:  %U",
1573                            format_ip4_address, &ip0->dst_address);
1574               next0 = SNAT_OUT2IN_NEXT_DROP;
1575               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1576               goto trace00;
1577             }
1578
1579           snat_det_reverse(dm0, &ip0->dst_address,
1580                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1581
1582           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1583           if (PREDICT_FALSE(!ses0))
1584             {
1585               clib_warning("no match src %U:%d dst %U:%d for user %U",
1586                            format_ip4_address, &ip0->src_address,
1587                            clib_net_to_host_u16 (tcp0->src),
1588                            format_ip4_address, &ip0->dst_address,
1589                            clib_net_to_host_u16 (tcp0->dst),
1590                            format_ip4_address, &new_addr0);
1591               next0 = SNAT_OUT2IN_NEXT_DROP;
1592               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1593               goto trace00;
1594             }
1595           new_port0 = ses0->in_port;
1596
1597           old_addr0 = ip0->dst_address;
1598           ip0->dst_address = new_addr0;
1599           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1600
1601           sum0 = ip0->checksum;
1602           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1603                                  ip4_header_t,
1604                                  dst_address /* changed member */);
1605           ip0->checksum = ip_csum_fold (sum0);
1606
1607           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1608             {
1609               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1610                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1611               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1612                 snat_det_ses_close(dm0, ses0);
1613
1614               old_port0 = tcp0->dst;
1615               tcp0->dst = new_port0;
1616
1617               sum0 = tcp0->checksum;
1618               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1619                                      ip4_header_t,
1620                                      dst_address /* changed member */);
1621
1622               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1623                                      ip4_header_t /* cheat */,
1624                                      length /* changed member */);
1625               tcp0->checksum = ip_csum_fold(sum0);
1626             }
1627           else
1628             {
1629               old_port0 = udp0->dst_port;
1630               udp0->dst_port = new_port0;
1631               udp0->checksum = 0;
1632             }
1633
1634         trace00:
1635
1636           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1637                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1638             {
1639               snat_out2in_trace_t *t =
1640                  vlib_add_trace (vm, node, b0, sizeof (*t));
1641               t->sw_if_index = sw_if_index0;
1642               t->next_index = next0;
1643               t->session_index = ~0;
1644               if (ses0)
1645                 t->session_index = ses0 - dm0->sessions;
1646             }
1647
1648           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1649
1650           /* verify speculative enqueue, maybe switch current next frame */
1651           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1652                                            to_next, n_left_to_next,
1653                                            bi0, next0);
1654         }
1655
1656       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1657     }
1658
1659   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1660                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1661                                pkts_processed);
1662   return frame->n_vectors;
1663 }
1664
1665 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1666   .function = snat_det_out2in_node_fn,
1667   .name = "snat-det-out2in",
1668   .vector_size = sizeof (u32),
1669   .format_trace = format_snat_out2in_trace,
1670   .type = VLIB_NODE_TYPE_INTERNAL,
1671
1672   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1673   .error_strings = snat_out2in_error_strings,
1674
1675   .runtime_data_bytes = sizeof (snat_runtime_t),
1676
1677   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1678
1679   /* edit / add dispositions here */
1680   .next_nodes = {
1681     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1682     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1683     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1684   },
1685 };
1686 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1687
1688 /**
1689  * Get address and port values to be used for packet SNAT translation
1690  * and create session if needed
1691  *
1692  * @param[in,out] sm             SNAT main
1693  * @param[in,out] node           SNAT node runtime
1694  * @param[in] thread_index       thread index
1695  * @param[in,out] b0             buffer containing packet to be translated
1696  * @param[out] p_proto           protocol used for matching
1697  * @param[out] p_value           address and port after NAT translation
1698  * @param[out] p_dont_translate  if packet should not be translated
1699  * @param d                      optional parameter
1700  * @param e                      optional parameter
1701  */
1702 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
1703                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
1704                           snat_session_key_t *p_value,
1705                           u8 *p_dont_translate, void *d, void *e)
1706 {
1707   ip4_header_t *ip0;
1708   icmp46_header_t *icmp0;
1709   u32 sw_if_index0;
1710   u8 protocol;
1711   snat_det_out_key_t key0;
1712   u8 dont_translate = 0;
1713   u32 next0 = ~0;
1714   icmp_echo_header_t *echo0, *inner_echo0 = 0;
1715   ip4_header_t *inner_ip0;
1716   void *l4_header = 0;
1717   icmp46_header_t *inner_icmp0;
1718   snat_det_map_t * dm0 = 0;
1719   ip4_address_t new_addr0 = {{0}};
1720   snat_det_session_t * ses0 = 0;
1721   ip4_address_t out_addr;
1722
1723   ip0 = vlib_buffer_get_current (b0);
1724   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1725   echo0 = (icmp_echo_header_t *)(icmp0+1);
1726   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1727
1728   if (!icmp_is_error_message (icmp0))
1729     {
1730       protocol = SNAT_PROTOCOL_ICMP;
1731       key0.ext_host_addr = ip0->src_address;
1732       key0.ext_host_port = 0;
1733       key0.out_port = echo0->identifier;
1734       out_addr = ip0->dst_address;
1735     }
1736   else
1737     {
1738       inner_ip0 = (ip4_header_t *)(echo0+1);
1739       l4_header = ip4_next_header (inner_ip0);
1740       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
1741       key0.ext_host_addr = inner_ip0->dst_address;
1742       out_addr = inner_ip0->src_address;
1743       switch (protocol)
1744         {
1745         case SNAT_PROTOCOL_ICMP:
1746           inner_icmp0 = (icmp46_header_t*)l4_header;
1747           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1748           key0.ext_host_port = 0;
1749           key0.out_port = inner_echo0->identifier;
1750           break;
1751         case SNAT_PROTOCOL_UDP:
1752         case SNAT_PROTOCOL_TCP:
1753           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1754           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
1755           break;
1756         default:
1757           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1758           next0 = SNAT_OUT2IN_NEXT_DROP;
1759           goto out;
1760         }
1761     }
1762
1763   dm0 = snat_det_map_by_out(sm, &out_addr);
1764   if (PREDICT_FALSE(!dm0))
1765     {
1766       /* Don't NAT packet aimed at the intfc address */
1767       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1768                                           ip0->dst_address.as_u32)))
1769         {
1770           dont_translate = 1;
1771           goto out;
1772         }
1773       clib_warning("unknown dst address:  %U",
1774                    format_ip4_address, &ip0->dst_address);
1775       goto out;
1776     }
1777
1778   snat_det_reverse(dm0, &ip0->dst_address,
1779                    clib_net_to_host_u16(key0.out_port), &new_addr0);
1780
1781   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1782   if (PREDICT_FALSE(!ses0))
1783     {
1784       /* Don't NAT packet aimed at the intfc address */
1785       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1786                                           ip0->dst_address.as_u32)))
1787         {
1788           dont_translate = 1;
1789           goto out;
1790         }
1791       clib_warning("no match src %U:%d dst %U:%d for user %U",
1792                    format_ip4_address, &key0.ext_host_addr,
1793                    clib_net_to_host_u16 (key0.ext_host_port),
1794                    format_ip4_address, &out_addr,
1795                    clib_net_to_host_u16 (key0.out_port),
1796                    format_ip4_address, &new_addr0);
1797       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1798       next0 = SNAT_OUT2IN_NEXT_DROP;
1799       goto out;
1800     }
1801
1802   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
1803                     !icmp_is_error_message (icmp0)))
1804     {
1805       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1806       next0 = SNAT_OUT2IN_NEXT_DROP;
1807       goto out;
1808     }
1809
1810   goto out;
1811
1812 out:
1813   *p_proto = protocol;
1814   if (ses0)
1815     {
1816       p_value->addr = new_addr0;
1817       p_value->fib_index = sm->inside_fib_index;
1818       p_value->port = ses0->in_port;
1819     }
1820   *p_dont_translate = dont_translate;
1821   if (d)
1822     *(snat_det_session_t**)d = ses0;
1823   if (e)
1824     *(snat_det_map_t**)e = dm0;
1825   return next0;
1826 }
1827
1828 /**********************/
1829 /*** worker handoff ***/
1830 /**********************/
1831 static uword
1832 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1833                                vlib_node_runtime_t * node,
1834                                vlib_frame_t * frame)
1835 {
1836   snat_main_t *sm = &snat_main;
1837   vlib_thread_main_t *tm = vlib_get_thread_main ();
1838   u32 n_left_from, *from, *to_next = 0;
1839   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1840   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1841     = 0;
1842   vlib_frame_queue_elt_t *hf = 0;
1843   vlib_frame_t *f = 0;
1844   int i;
1845   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1846   u32 next_worker_index = 0;
1847   u32 current_worker_index = ~0;
1848   u32 thread_index = vlib_get_thread_index ();
1849
1850   ASSERT (vec_len (sm->workers));
1851
1852   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1853     {
1854       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1855
1856       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1857                                sm->first_worker_index + sm->num_workers - 1,
1858                                (vlib_frame_queue_t *) (~0));
1859     }
1860
1861   from = vlib_frame_vector_args (frame);
1862   n_left_from = frame->n_vectors;
1863
1864   while (n_left_from > 0)
1865     {
1866       u32 bi0;
1867       vlib_buffer_t *b0;
1868       u32 sw_if_index0;
1869       u32 rx_fib_index0;
1870       ip4_header_t * ip0;
1871       u8 do_handoff;
1872
1873       bi0 = from[0];
1874       from += 1;
1875       n_left_from -= 1;
1876
1877       b0 = vlib_get_buffer (vm, bi0);
1878
1879       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1880       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1881
1882       ip0 = vlib_buffer_get_current (b0);
1883
1884       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1885
1886       if (PREDICT_FALSE (next_worker_index != thread_index))
1887         {
1888           do_handoff = 1;
1889
1890           if (next_worker_index != current_worker_index)
1891             {
1892               if (hf)
1893                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1894
1895               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1896                                                       next_worker_index,
1897                                                       handoff_queue_elt_by_worker_index);
1898
1899               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1900               to_next_worker = &hf->buffer_index[hf->n_vectors];
1901               current_worker_index = next_worker_index;
1902             }
1903
1904           /* enqueue to correct worker thread */
1905           to_next_worker[0] = bi0;
1906           to_next_worker++;
1907           n_left_to_next_worker--;
1908
1909           if (n_left_to_next_worker == 0)
1910             {
1911               hf->n_vectors = VLIB_FRAME_SIZE;
1912               vlib_put_frame_queue_elt (hf);
1913               current_worker_index = ~0;
1914               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1915               hf = 0;
1916             }
1917         }
1918       else
1919         {
1920           do_handoff = 0;
1921           /* if this is 1st frame */
1922           if (!f)
1923             {
1924               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1925               to_next = vlib_frame_vector_args (f);
1926             }
1927
1928           to_next[0] = bi0;
1929           to_next += 1;
1930           f->n_vectors++;
1931         }
1932
1933       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1934                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1935         {
1936           snat_out2in_worker_handoff_trace_t *t =
1937             vlib_add_trace (vm, node, b0, sizeof (*t));
1938           t->next_worker_index = next_worker_index;
1939           t->do_handoff = do_handoff;
1940         }
1941     }
1942
1943   if (f)
1944     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1945
1946   if (hf)
1947     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1948
1949   /* Ship frames to the worker nodes */
1950   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1951     {
1952       if (handoff_queue_elt_by_worker_index[i])
1953         {
1954           hf = handoff_queue_elt_by_worker_index[i];
1955           /*
1956            * It works better to let the handoff node
1957            * rate-adapt, always ship the handoff queue element.
1958            */
1959           if (1 || hf->n_vectors == hf->last_n_vectors)
1960             {
1961               vlib_put_frame_queue_elt (hf);
1962               handoff_queue_elt_by_worker_index[i] = 0;
1963             }
1964           else
1965             hf->last_n_vectors = hf->n_vectors;
1966         }
1967       congested_handoff_queue_by_worker_index[i] =
1968         (vlib_frame_queue_t *) (~0);
1969     }
1970   hf = 0;
1971   current_worker_index = ~0;
1972   return frame->n_vectors;
1973 }
1974
1975 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1976   .function = snat_out2in_worker_handoff_fn,
1977   .name = "snat-out2in-worker-handoff",
1978   .vector_size = sizeof (u32),
1979   .format_trace = format_snat_out2in_worker_handoff_trace,
1980   .type = VLIB_NODE_TYPE_INTERNAL,
1981
1982   .n_next_nodes = 1,
1983
1984   .next_nodes = {
1985     [0] = "error-drop",
1986   },
1987 };
1988
1989 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1990
1991 static uword
1992 snat_out2in_fast_node_fn (vlib_main_t * vm,
1993                           vlib_node_runtime_t * node,
1994                           vlib_frame_t * frame)
1995 {
1996   u32 n_left_from, * from, * to_next;
1997   snat_out2in_next_t next_index;
1998   u32 pkts_processed = 0;
1999   snat_main_t * sm = &snat_main;
2000
2001   from = vlib_frame_vector_args (frame);
2002   n_left_from = frame->n_vectors;
2003   next_index = node->cached_next_index;
2004
2005   while (n_left_from > 0)
2006     {
2007       u32 n_left_to_next;
2008
2009       vlib_get_next_frame (vm, node, next_index,
2010                            to_next, n_left_to_next);
2011
2012       while (n_left_from > 0 && n_left_to_next > 0)
2013         {
2014           u32 bi0;
2015           vlib_buffer_t * b0;
2016           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2017           u32 sw_if_index0;
2018           ip4_header_t * ip0;
2019           ip_csum_t sum0;
2020           u32 new_addr0, old_addr0;
2021           u16 new_port0, old_port0;
2022           udp_header_t * udp0;
2023           tcp_header_t * tcp0;
2024           icmp46_header_t * icmp0;
2025           snat_session_key_t key0, sm0;
2026           u32 proto0;
2027           u32 rx_fib_index0;
2028
2029           /* speculatively enqueue b0 to the current next frame */
2030           bi0 = from[0];
2031           to_next[0] = bi0;
2032           from += 1;
2033           to_next += 1;
2034           n_left_from -= 1;
2035           n_left_to_next -= 1;
2036
2037           b0 = vlib_get_buffer (vm, bi0);
2038
2039           ip0 = vlib_buffer_get_current (b0);
2040           udp0 = ip4_next_header (ip0);
2041           tcp0 = (tcp_header_t *) udp0;
2042           icmp0 = (icmp46_header_t *) udp0;
2043
2044           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2045           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2046
2047           vnet_feature_next (sw_if_index0, &next0, b0);
2048
2049           if (PREDICT_FALSE(ip0->ttl == 1))
2050             {
2051               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2052               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2053                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2054                                            0);
2055               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2056               goto trace00;
2057             }
2058
2059           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2060
2061           if (PREDICT_FALSE (proto0 == ~0))
2062               goto trace00;
2063
2064           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2065             {
2066               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2067                                   rx_fib_index0, node, next0, ~0, 0, 0);
2068               goto trace00;
2069             }
2070
2071           key0.addr = ip0->dst_address;
2072           key0.port = udp0->dst_port;
2073           key0.fib_index = rx_fib_index0;
2074
2075           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2076             {
2077               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2078               goto trace00;
2079             }
2080
2081           new_addr0 = sm0.addr.as_u32;
2082           new_port0 = sm0.port;
2083           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2084           old_addr0 = ip0->dst_address.as_u32;
2085           ip0->dst_address.as_u32 = new_addr0;
2086
2087           sum0 = ip0->checksum;
2088           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2089                                  ip4_header_t,
2090                                  dst_address /* changed member */);
2091           ip0->checksum = ip_csum_fold (sum0);
2092
2093           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2094             {
2095                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2096                 {
2097                   old_port0 = tcp0->dst_port;
2098                   tcp0->dst_port = new_port0;
2099
2100                   sum0 = tcp0->checksum;
2101                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2102                                          ip4_header_t,
2103                                          dst_address /* changed member */);
2104
2105                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2106                                          ip4_header_t /* cheat */,
2107                                          length /* changed member */);
2108                   tcp0->checksum = ip_csum_fold(sum0);
2109                 }
2110               else
2111                 {
2112                   old_port0 = udp0->dst_port;
2113                   udp0->dst_port = new_port0;
2114                   udp0->checksum = 0;
2115                 }
2116             }
2117           else
2118             {
2119               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2120                 {
2121                   sum0 = tcp0->checksum;
2122                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2123                                          ip4_header_t,
2124                                          dst_address /* changed member */);
2125
2126                   tcp0->checksum = ip_csum_fold(sum0);
2127                 }
2128             }
2129
2130         trace00:
2131
2132           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2133                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2134             {
2135               snat_out2in_trace_t *t =
2136                  vlib_add_trace (vm, node, b0, sizeof (*t));
2137               t->sw_if_index = sw_if_index0;
2138               t->next_index = next0;
2139             }
2140
2141           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2142
2143           /* verify speculative enqueue, maybe switch current next frame */
2144           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2145                                            to_next, n_left_to_next,
2146                                            bi0, next0);
2147         }
2148
2149       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2150     }
2151
2152   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2153                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2154                                pkts_processed);
2155   return frame->n_vectors;
2156 }
2157
2158 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2159   .function = snat_out2in_fast_node_fn,
2160   .name = "snat-out2in-fast",
2161   .vector_size = sizeof (u32),
2162   .format_trace = format_snat_out2in_fast_trace,
2163   .type = VLIB_NODE_TYPE_INTERNAL,
2164
2165   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2166   .error_strings = snat_out2in_error_strings,
2167
2168   .runtime_data_bytes = sizeof (snat_runtime_t),
2169
2170   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2171
2172   /* edit / add dispositions here */
2173   .next_nodes = {
2174     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2175     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2176     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2177   },
2178 };
2179 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);