src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     NAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out NAT44 session key.
 121  * @param out2in Out2in NAT44 session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140   ip4_header_t *ip0;
 141
 142   ip0 = vlib_buffer_get_current (b0);
 143
 144   user_key.addr = in2out.addr;
 145   user_key.fib_index = in2out.fib_index;
 146   kv0.key = user_key.as_u64;
 147
 148   /* Ever heard of the "user" = inside ip4 address before? */
 149   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 150     {
 151       /* no, make a new one */
 152       pool_get (sm->per_thread_data[thread_index].users, u);
 153       memset (u, 0, sizeof (*u));
 154       u->addr = in2out.addr;
 155       u->fib_index = in2out.fib_index;
 156
 157       pool_get (sm->per_thread_data[thread_index].list_pool,
 158                 per_user_list_head_elt);
 159
 160       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 161         sm->per_thread_data[thread_index].list_pool;
 162
 163       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 164                        u->sessions_per_user_list_head_index);
 165
 166       kv0.value = u - sm->per_thread_data[thread_index].users;
 167
 168       /* add user */
 169       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 170
 171       /* add non-traslated packets worker lookup */
 172       kv0.value = thread_index;
 173       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 174     }
 175   else
 176     {
 177       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 178                              value0.value);
 179     }
 180
 181   pool_get (sm->per_thread_data[thread_index].sessions, s);
 182   memset (s, 0, sizeof (*s));
 183
 184   s->outside_address_index = ~0;
 185   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 186   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 187   u->nstaticsessions++;
 188
 189   /* Create list elts */
 190   pool_get (sm->per_thread_data[thread_index].list_pool,
 191             per_user_translation_list_elt);
 192   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 193                    per_user_translation_list_elt -
 194                    sm->per_thread_data[thread_index].list_pool);
 195
 196   per_user_translation_list_elt->value =
 197     s - sm->per_thread_data[thread_index].sessions;
 198   s->per_user_index =
 199     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 200   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 201
 202   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 203                       s->per_user_list_head_index,
 204                       per_user_translation_list_elt -
 205                       sm->per_thread_data[thread_index].list_pool);
 206
 207   s->in2out = in2out;
 208   s->out2in = out2in;
 209   s->in2out.protocol = out2in.protocol;
 210
 211   /* Add to translation hashes */
 212   kv0.key = s->in2out.as_u64;
 213   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 214   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 215       clib_warning ("in2out key add failed");
 216
 217   kv0.key = s->out2in.as_u64;
 218   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 219
 220   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 221       clib_warning ("out2in key add failed");
 222
 223   /* log NAT event */
 224   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 225                                       s->out2in.addr.as_u32,
 226                                       s->in2out.protocol,
 227                                       s->in2out.port,
 228                                       s->out2in.port,
 229                                       s->in2out.fib_index);
 230    return s;
 231 }
 232
 233 static_always_inline
 234 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 235                                  snat_session_key_t *p_key0)
 236 {
 237   icmp46_header_t *icmp0;
 238   snat_session_key_t key0;
 239   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 240   ip4_header_t *inner_ip0;
 241   void *l4_header = 0;
 242   icmp46_header_t *inner_icmp0;
 243
 244   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 245   echo0 = (icmp_echo_header_t *)(icmp0+1);
 246
 247   if (!icmp_is_error_message (icmp0))
 248     {
 249       key0.protocol = SNAT_PROTOCOL_ICMP;
 250       key0.addr = ip0->dst_address;
 251       key0.port = echo0->identifier;
 252     }
 253   else
 254     {
 255       inner_ip0 = (ip4_header_t *)(echo0+1);
 256       l4_header = ip4_next_header (inner_ip0);
 257       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 258       key0.addr = inner_ip0->src_address;
 259       switch (key0.protocol)
 260         {
 261         case SNAT_PROTOCOL_ICMP:
 262           inner_icmp0 = (icmp46_header_t*)l4_header;
 263           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 264           key0.port = inner_echo0->identifier;
 265           break;
 266         case SNAT_PROTOCOL_UDP:
 267         case SNAT_PROTOCOL_TCP:
 268           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 269           break;
 270         default:
 271           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 272         }
 273     }
 274   *p_key0 = key0;
 275   return -1; /* success */
 276 }
 277
 278 /**
 279  * Get address and port values to be used for ICMP packet translation
 280  * and create session if needed
 281  *
 282  * @param[in,out] sm             NAT main
 283  * @param[in,out] node           NAT node runtime
 284  * @param[in] thread_index       thread index
 285  * @param[in,out] b0             buffer containing packet to be translated
 286  * @param[out] p_proto           protocol used for matching
 287  * @param[out] p_value           address and port after NAT translation
 288  * @param[out] p_dont_translate  if packet should not be translated
 289  * @param d                      optional parameter
 290  * @param e                      optional parameter
 291  */
 292 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 293                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 294                            snat_session_key_t *p_value,
 295                            u8 *p_dont_translate, void *d, void *e)
 296 {
 297   ip4_header_t *ip0;
 298   icmp46_header_t *icmp0;
 299   u32 sw_if_index0;
 300   u32 rx_fib_index0;
 301   snat_session_key_t key0;
 302   snat_session_key_t sm0;
 303   snat_session_t *s0 = 0;
 304   u8 dont_translate = 0;
 305   clib_bihash_kv_8_8_t kv0, value0;
 306   u8 is_addr_only;
 307   u32 next0 = ~0;
 308   int err;
 309
 310   ip0 = vlib_buffer_get_current (b0);
 311   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 312   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 313   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 314
 315   key0.protocol = 0;
 316
 317   err = icmp_get_key (ip0, &key0);
 318   if (err != -1)
 319     {
 320       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 321       next0 = SNAT_OUT2IN_NEXT_DROP;
 322       goto out;
 323     }
 324   key0.fib_index = rx_fib_index0;
 325
 326   kv0.key = key0.as_u64;
 327
 328   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 329     {
 330       /* Try to match static mapping by external address and port,
 331          destination address and port in packet */
 332       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 333         {
 334           /* Don't NAT packet aimed at the intfc address */
 335           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 336                                               ip0->dst_address.as_u32)))
 337             {
 338               dont_translate = 1;
 339               goto out;
 340             }
 341           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 342           next0 = SNAT_OUT2IN_NEXT_DROP;
 343           goto out;
 344         }
 345
 346       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 347                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 348         {
 349           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 350           next0 = SNAT_OUT2IN_NEXT_DROP;
 351           goto out;
 352         }
 353
 354       /* Create session initiated by host from external network */
 355       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 356                                              node, thread_index);
 357
 358       if (!s0)
 359         {
 360           next0 = SNAT_OUT2IN_NEXT_DROP;
 361           goto out;
 362         }
 363     }
 364   else
 365     {
 366       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 367                         icmp0->type != ICMP4_echo_request &&
 368                         !icmp_is_error_message (icmp0)))
 369         {
 370           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 371           next0 = SNAT_OUT2IN_NEXT_DROP;
 372           goto out;
 373         }
 374
 375       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 376                               value0.value);
 377     }
 378
 379 out:
 380   *p_proto = key0.protocol;
 381   if (s0)
 382     *p_value = s0->in2out;
 383   *p_dont_translate = dont_translate;
 384   if (d)
 385     *(snat_session_t**)d = s0;
 386   return next0;
 387 }
 388
 389 /**
 390  * Get address and port values to be used for ICMP packet translation
 391  *
 392  * @param[in] sm                 NAT main
 393  * @param[in,out] node           NAT node runtime
 394  * @param[in] thread_index       thread index
 395  * @param[in,out] b0             buffer containing packet to be translated
 396  * @param[out] p_proto           protocol used for matching
 397  * @param[out] p_value           address and port after NAT translation
 398  * @param[out] p_dont_translate  if packet should not be translated
 399  * @param d                      optional parameter
 400  * @param e                      optional parameter
 401  */
 402 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 403                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 404                            snat_session_key_t *p_value,
 405                            u8 *p_dont_translate, void *d, void *e)
 406 {
 407   ip4_header_t *ip0;
 408   icmp46_header_t *icmp0;
 409   u32 sw_if_index0;
 410   u32 rx_fib_index0;
 411   snat_session_key_t key0;
 412   snat_session_key_t sm0;
 413   u8 dont_translate = 0;
 414   u8 is_addr_only;
 415   u32 next0 = ~0;
 416   int err;
 417
 418   ip0 = vlib_buffer_get_current (b0);
 419   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 420   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 421   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 422
 423   err = icmp_get_key (ip0, &key0);
 424   if (err != -1)
 425     {
 426       b0->error = node->errors[err];
 427       next0 = SNAT_OUT2IN_NEXT_DROP;
 428       goto out2;
 429     }
 430   key0.fib_index = rx_fib_index0;
 431
 432   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 433     {
 434       /* Don't NAT packet aimed at the intfc address */
 435       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 436         {
 437           dont_translate = 1;
 438           goto out;
 439         }
 440       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 441       next0 = SNAT_OUT2IN_NEXT_DROP;
 442       goto out;
 443     }
 444
 445   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 446                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 447                     !icmp_is_error_message (icmp0)))
 448     {
 449       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 450       next0 = SNAT_OUT2IN_NEXT_DROP;
 451       goto out;
 452     }
 453
 454 out:
 455   *p_value = sm0;
 456 out2:
 457   *p_proto = key0.protocol;
 458   *p_dont_translate = dont_translate;
 459   return next0;
 460 }
 461
 462 static inline u32 icmp_out2in (snat_main_t *sm,
 463                                vlib_buffer_t * b0,
 464                                ip4_header_t * ip0,
 465                                icmp46_header_t * icmp0,
 466                                u32 sw_if_index0,
 467                                u32 rx_fib_index0,
 468                                vlib_node_runtime_t * node,
 469                                u32 next0,
 470                                u32 thread_index,
 471                                void *d,
 472                                void *e)
 473 {
 474   snat_session_key_t sm0;
 475   u8 protocol;
 476   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 477   ip4_header_t *inner_ip0 = 0;
 478   void *l4_header = 0;
 479   icmp46_header_t *inner_icmp0;
 480   u8 dont_translate;
 481   u32 new_addr0, old_addr0;
 482   u16 old_id0, new_id0;
 483   ip_csum_t sum0;
 484   u16 checksum0;
 485   u32 next0_tmp;
 486
 487   echo0 = (icmp_echo_header_t *)(icmp0+1);
 488
 489   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 490                                        &protocol, &sm0, &dont_translate, d, e);
 491   if (next0_tmp != ~0)
 492     next0 = next0_tmp;
 493   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 494     goto out;
 495
 496   sum0 = ip_incremental_checksum (0, icmp0,
 497                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 498   checksum0 = ~ip_csum_fold (sum0);
 499   if (checksum0 != 0 && checksum0 != 0xffff)
 500     {
 501       next0 = SNAT_OUT2IN_NEXT_DROP;
 502       goto out;
 503     }
 504
 505   old_addr0 = ip0->dst_address.as_u32;
 506   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 507   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 508
 509   sum0 = ip0->checksum;
 510   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 511                          dst_address /* changed member */);
 512   ip0->checksum = ip_csum_fold (sum0);
 513
 514   if (!icmp_is_error_message (icmp0))
 515     {
 516       new_id0 = sm0.port;
 517       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 518         {
 519           old_id0 = echo0->identifier;
 520           new_id0 = sm0.port;
 521           echo0->identifier = new_id0;
 522
 523           sum0 = icmp0->checksum;
 524           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 525                                  identifier /* changed member */);
 526           icmp0->checksum = ip_csum_fold (sum0);
 527         }
 528     }
 529   else
 530     {
 531       inner_ip0 = (ip4_header_t *)(echo0+1);
 532       l4_header = ip4_next_header (inner_ip0);
 533
 534       if (!ip4_header_checksum_is_valid (inner_ip0))
 535         {
 536           next0 = SNAT_OUT2IN_NEXT_DROP;
 537           goto out;
 538         }
 539
 540       old_addr0 = inner_ip0->src_address.as_u32;
 541       inner_ip0->src_address = sm0.addr;
 542       new_addr0 = inner_ip0->src_address.as_u32;
 543
 544       sum0 = icmp0->checksum;
 545       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 546                              src_address /* changed member */);
 547       icmp0->checksum = ip_csum_fold (sum0);
 548
 549       switch (protocol)
 550         {
 551         case SNAT_PROTOCOL_ICMP:
 552           inner_icmp0 = (icmp46_header_t*)l4_header;
 553           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 554
 555           old_id0 = inner_echo0->identifier;
 556           new_id0 = sm0.port;
 557           inner_echo0->identifier = new_id0;
 558
 559           sum0 = icmp0->checksum;
 560           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 561                                  identifier);
 562           icmp0->checksum = ip_csum_fold (sum0);
 563           break;
 564         case SNAT_PROTOCOL_UDP:
 565         case SNAT_PROTOCOL_TCP:
 566           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 567           new_id0 = sm0.port;
 568           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 569
 570           sum0 = icmp0->checksum;
 571           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 572                                  src_port);
 573           icmp0->checksum = ip_csum_fold (sum0);
 574           break;
 575         default:
 576           ASSERT(0);
 577         }
 578     }
 579
 580 out:
 581   return next0;
 582 }
 583
 584
 585 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 586                                          vlib_buffer_t * b0,
 587                                          ip4_header_t * ip0,
 588                                          icmp46_header_t * icmp0,
 589                                          u32 sw_if_index0,
 590                                          u32 rx_fib_index0,
 591                                          vlib_node_runtime_t * node,
 592                                          u32 next0, f64 now,
 593                                          u32 thread_index,
 594                                          snat_session_t ** p_s0)
 595 {
 596   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 597                       next0, thread_index, p_s0, 0);
 598   snat_session_t * s0 = *p_s0;
 599   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 600     {
 601       /* Accounting */
 602       s0->last_heard = now;
 603       s0->total_pkts++;
 604       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 605       /* Per-user LRU list maintenance for dynamic translation */
 606       if (!snat_is_session_static (s0))
 607         {
 608           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 609                              s0->per_user_index);
 610           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 611                               s0->per_user_list_head_index,
 612                               s0->per_user_index);
 613         }
 614     }
 615   return next0;
 616 }
 617
 618 static void
 619 snat_out2in_unknown_proto (snat_main_t *sm,
 620                            vlib_buffer_t * b,
 621                            ip4_header_t * ip,
 622                            u32 rx_fib_index,
 623                            u32 thread_index,
 624                            f64 now,
 625                            vlib_main_t * vm)
 626 {
 627   clib_bihash_kv_8_8_t kv, value;
 628   clib_bihash_kv_16_8_t s_kv, s_value;
 629   snat_static_mapping_t *m;
 630   snat_session_key_t m_key;
 631   u32 old_addr, new_addr;
 632   ip_csum_t sum;
 633   nat_ed_ses_key_t key;
 634   snat_session_t * s;
 635   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 636   snat_user_key_t u_key;
 637   snat_user_t *u;
 638   dlist_elt_t *head, *elt;
 639
 640   old_addr = ip->dst_address.as_u32;
 641
 642   key.l_addr = ip->dst_address;
 643   key.r_addr = ip->src_address;
 644   key.fib_index = rx_fib_index;
 645   key.proto = ip->protocol;
 646   key.rsvd = 0;
 647   key.l_port = 0;
 648   s_kv.key[0] = key.as_u64[0];
 649   s_kv.key[1] = key.as_u64[1];
 650
 651   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 652     {
 653       s = pool_elt_at_index (tsm->sessions, s_value.value);
 654       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 655     }
 656   else
 657     {
 658       m_key.addr = ip->dst_address;
 659       m_key.port = 0;
 660       m_key.protocol = 0;
 661       m_key.fib_index = rx_fib_index;
 662       kv.key = m_key.as_u64;
 663       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 664         return;
 665
 666       m = pool_elt_at_index (sm->static_mappings, value.value);
 667
 668       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 669
 670       u_key.addr = ip->src_address;
 671       u_key.fib_index = m->fib_index;
 672       kv.key = u_key.as_u64;
 673
 674       /* Ever heard of the "user" = src ip4 address before? */
 675       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
 676         {
 677           /* no, make a new one */
 678           pool_get (tsm->users, u);
 679           memset (u, 0, sizeof (*u));
 680           u->addr = ip->src_address;
 681           u->fib_index = rx_fib_index;
 682
 683           pool_get (tsm->list_pool, head);
 684           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 685
 686           clib_dlist_init (tsm->list_pool,
 687                            u->sessions_per_user_list_head_index);
 688
 689           kv.value = u - tsm->users;
 690
 691           /* add user */
 692           clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
 693         }
 694       else
 695         {
 696           u = pool_elt_at_index (tsm->users, value.value);
 697         }
 698
 699       /* Create a new session */
 700       pool_get (tsm->sessions, s);
 701       memset (s, 0, sizeof (*s));
 702
 703       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 704       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 705       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 706       s->outside_address_index = ~0;
 707       s->out2in.addr.as_u32 = old_addr;
 708       s->out2in.fib_index = rx_fib_index;
 709       s->in2out.addr.as_u32 = new_addr;
 710       s->in2out.fib_index = m->fib_index;
 711       s->in2out.port = s->out2in.port = ip->protocol;
 712       u->nstaticsessions++;
 713
 714       /* Create list elts */
 715       pool_get (tsm->list_pool, elt);
 716       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 717       elt->value = s - tsm->sessions;
 718       s->per_user_index = elt - tsm->list_pool;
 719       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 720       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 721                           s->per_user_index);
 722
 723       /* Add to lookup tables */
 724       s_kv.value = s - tsm->sessions;
 725       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 726         clib_warning ("out2in key add failed");
 727
 728       key.l_addr = ip->dst_address;
 729       key.fib_index = m->fib_index;
 730       s_kv.key[0] = key.as_u64[0];
 731       s_kv.key[1] = key.as_u64[1];
 732       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 733         clib_warning ("in2out key add failed");
 734    }
 735
 736   /* Update IP checksum */
 737   sum = ip->checksum;
 738   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 739   ip->checksum = ip_csum_fold (sum);
 740
 741   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 742
 743   /* Accounting */
 744   s->last_heard = now;
 745   s->total_pkts++;
 746   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 747   /* Per-user LRU list maintenance */
 748   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 749   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 750                       s->per_user_index);
 751 }
 752
 753 static snat_session_t *
 754 snat_out2in_lb (snat_main_t *sm,
 755                 vlib_buffer_t * b,
 756                 ip4_header_t * ip,
 757                 u32 rx_fib_index,
 758                 u32 thread_index,
 759                 f64 now,
 760                 vlib_main_t * vm)
 761 {
 762   nat_ed_ses_key_t key;
 763   clib_bihash_kv_16_8_t s_kv, s_value;
 764   udp_header_t *udp = ip4_next_header (ip);
 765   tcp_header_t *tcp = (tcp_header_t *) udp;
 766   snat_session_t *s = 0;
 767   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 768   snat_session_key_t e_key, l_key;
 769   clib_bihash_kv_8_8_t kv, value;
 770   u32 old_addr, new_addr;
 771   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 772   u16 new_port, old_port;
 773   ip_csum_t sum;
 774   snat_user_key_t u_key;
 775   snat_user_t *u;
 776   dlist_elt_t *head, *elt;
 777
 778   old_addr = ip->dst_address.as_u32;
 779
 780   key.l_addr = ip->dst_address;
 781   key.r_addr = ip->src_address;
 782   key.fib_index = rx_fib_index;
 783   key.proto = ip->protocol;
 784   key.rsvd = 0;
 785   key.l_port = udp->dst_port;
 786   s_kv.key[0] = key.as_u64[0];
 787   s_kv.key[1] = key.as_u64[1];
 788
 789   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 790     {
 791       s = pool_elt_at_index (tsm->sessions, s_value.value);
 792     }
 793   else
 794     {
 795       e_key.addr = ip->dst_address;
 796       e_key.port = udp->dst_port;
 797       e_key.protocol = proto;
 798       e_key.fib_index = rx_fib_index;
 799       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
 800         return 0;
 801
 802       u_key.addr = l_key.addr;
 803       u_key.fib_index = l_key.fib_index;
 804       kv.key = u_key.as_u64;
 805
 806       /* Ever heard of the "user" = src ip4 address before? */
 807       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
 808         {
 809           /* no, make a new one */
 810           pool_get (tsm->users, u);
 811           memset (u, 0, sizeof (*u));
 812           u->addr = l_key.addr;
 813           u->fib_index = l_key.fib_index;
 814
 815           pool_get (tsm->list_pool, head);
 816           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 817
 818           clib_dlist_init (tsm->list_pool,
 819                            u->sessions_per_user_list_head_index);
 820
 821           kv.value = u - tsm->users;
 822
 823           /* add user */
 824           if (clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1))
 825             clib_warning ("user key add failed");
 826         }
 827       else
 828         {
 829           u = pool_elt_at_index (tsm->users, value.value);
 830         }
 831
 832       /* Create a new session */
 833       pool_get (tsm->sessions, s);
 834       memset (s, 0, sizeof (*s));
 835
 836       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 837       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 838       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 839       s->outside_address_index = ~0;
 840       s->out2in = e_key;
 841       s->in2out = l_key;
 842       u->nstaticsessions++;
 843
 844       /* Create list elts */
 845       pool_get (tsm->list_pool, elt);
 846       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 847       elt->value = s - tsm->sessions;
 848       s->per_user_index = elt - tsm->list_pool;
 849       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 850       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 851                           s->per_user_index);
 852
 853       /* Add to lookup tables */
 854       s_kv.value = s - tsm->sessions;
 855       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 856         clib_warning ("out2in-ed key add failed");
 857
 858       key.l_addr = l_key.addr;
 859       key.fib_index = l_key.fib_index;
 860       key.l_port = l_key.port;
 861       s_kv.key[0] = key.as_u64[0];
 862       s_kv.key[1] = key.as_u64[1];
 863       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 864         clib_warning ("in2out-ed key add failed");
 865     }
 866
 867   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 868
 869   /* Update IP checksum */
 870   sum = ip->checksum;
 871   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 872   ip->checksum = ip_csum_fold (sum);
 873
 874   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 875     {
 876       old_port = tcp->dst_port;
 877       tcp->dst_port = s->in2out.port;
 878       new_port = tcp->dst_port;
 879
 880       sum = tcp->checksum;
 881       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 882       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 883       tcp->checksum = ip_csum_fold(sum);
 884     }
 885   else
 886     {
 887       udp->dst_port = s->in2out.port;
 888       udp->checksum = 0;
 889     }
 890
 891   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 892
 893   /* Accounting */
 894   s->last_heard = now;
 895   s->total_pkts++;
 896   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 897   return s;
 898 }
 899
 900 static uword
 901 snat_out2in_node_fn (vlib_main_t * vm,
 902                   vlib_node_runtime_t * node,
 903                   vlib_frame_t * frame)
 904 {
 905   u32 n_left_from, * from, * to_next;
 906   snat_out2in_next_t next_index;
 907   u32 pkts_processed = 0;
 908   snat_main_t * sm = &snat_main;
 909   f64 now = vlib_time_now (vm);
 910   u32 thread_index = vlib_get_thread_index ();
 911
 912   from = vlib_frame_vector_args (frame);
 913   n_left_from = frame->n_vectors;
 914   next_index = node->cached_next_index;
 915
 916   while (n_left_from > 0)
 917     {
 918       u32 n_left_to_next;
 919
 920       vlib_get_next_frame (vm, node, next_index,
 921                            to_next, n_left_to_next);
 922
 923       while (n_left_from >= 4 && n_left_to_next >= 2)
 924         {
 925           u32 bi0, bi1;
 926           vlib_buffer_t * b0, * b1;
 927           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 928           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 929           u32 sw_if_index0, sw_if_index1;
 930           ip4_header_t * ip0, *ip1;
 931           ip_csum_t sum0, sum1;
 932           u32 new_addr0, old_addr0;
 933           u16 new_port0, old_port0;
 934           u32 new_addr1, old_addr1;
 935           u16 new_port1, old_port1;
 936           udp_header_t * udp0, * udp1;
 937           tcp_header_t * tcp0, * tcp1;
 938           icmp46_header_t * icmp0, * icmp1;
 939           snat_session_key_t key0, key1, sm0, sm1;
 940           u32 rx_fib_index0, rx_fib_index1;
 941           u32 proto0, proto1;
 942           snat_session_t * s0 = 0, * s1 = 0;
 943           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 944
 945           /* Prefetch next iteration. */
 946           {
 947             vlib_buffer_t * p2, * p3;
 948
 949             p2 = vlib_get_buffer (vm, from[2]);
 950             p3 = vlib_get_buffer (vm, from[3]);
 951
 952             vlib_prefetch_buffer_header (p2, LOAD);
 953             vlib_prefetch_buffer_header (p3, LOAD);
 954
 955             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 956             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 957           }
 958
 959           /* speculatively enqueue b0 and b1 to the current next frame */
 960           to_next[0] = bi0 = from[0];
 961           to_next[1] = bi1 = from[1];
 962           from += 2;
 963           to_next += 2;
 964           n_left_from -= 2;
 965           n_left_to_next -= 2;
 966
 967           b0 = vlib_get_buffer (vm, bi0);
 968           b1 = vlib_get_buffer (vm, bi1);
 969
 970           vnet_buffer (b0)->snat.flags = 0;
 971           vnet_buffer (b1)->snat.flags = 0;
 972
 973           ip0 = vlib_buffer_get_current (b0);
 974           udp0 = ip4_next_header (ip0);
 975           tcp0 = (tcp_header_t *) udp0;
 976           icmp0 = (icmp46_header_t *) udp0;
 977
 978           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 979           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 980                                    sw_if_index0);
 981
 982           if (PREDICT_FALSE(ip0->ttl == 1))
 983             {
 984               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 985               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 986                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 987                                            0);
 988               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 989               goto trace0;
 990             }
 991
 992           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 993
 994           if (PREDICT_FALSE (proto0 == ~0))
 995             {
 996               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
 997                                         thread_index, now, vm);
 998               goto trace0;
 999             }
1000
1001           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1002             {
1003               next0 = icmp_out2in_slow_path
1004                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1005                  next0, now, thread_index, &s0);
1006               goto trace0;
1007             }
1008
1009           key0.addr = ip0->dst_address;
1010           key0.port = udp0->dst_port;
1011           key0.protocol = proto0;
1012           key0.fib_index = rx_fib_index0;
1013
1014           kv0.key = key0.as_u64;
1015
1016           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1017             {
1018               /* Try to match static mapping by external address and port,
1019                  destination address and port in packet */
1020               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1021                 {
1022                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1023                   /*
1024                    * Send DHCP packets to the ipv4 stack, or we won't
1025                    * be able to use dhcp client on the outside interface
1026                    */
1027                   if (proto0 != SNAT_PROTOCOL_UDP
1028                       || (udp0->dst_port
1029                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1030                     next0 = SNAT_OUT2IN_NEXT_DROP;
1031                   goto trace0;
1032                 }
1033
1034               /* Create session initiated by host from external network */
1035               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1036                                                      thread_index);
1037               if (!s0)
1038                 {
1039                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1040                   next0 = SNAT_OUT2IN_NEXT_DROP;
1041                   goto trace0;
1042                 }
1043             }
1044           else
1045             {
1046               if (PREDICT_FALSE (value0.value == ~0ULL))
1047                 {
1048                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index, now,
1049                                  vm);
1050                   goto trace0;
1051                 }
1052               else
1053                 {
1054                   s0 = pool_elt_at_index (
1055                     sm->per_thread_data[thread_index].sessions,
1056                     value0.value);
1057                 }
1058             }
1059
1060           old_addr0 = ip0->dst_address.as_u32;
1061           ip0->dst_address = s0->in2out.addr;
1062           new_addr0 = ip0->dst_address.as_u32;
1063           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1064
1065           sum0 = ip0->checksum;
1066           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1067                                  ip4_header_t,
1068                                  dst_address /* changed member */);
1069           ip0->checksum = ip_csum_fold (sum0);
1070
1071           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1072             {
1073               old_port0 = tcp0->dst_port;
1074               tcp0->dst_port = s0->in2out.port;
1075               new_port0 = tcp0->dst_port;
1076
1077               sum0 = tcp0->checksum;
1078               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1079                                      ip4_header_t,
1080                                      dst_address /* changed member */);
1081
1082               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1083                                      ip4_header_t /* cheat */,
1084                                      length /* changed member */);
1085               tcp0->checksum = ip_csum_fold(sum0);
1086             }
1087           else
1088             {
1089               old_port0 = udp0->dst_port;
1090               udp0->dst_port = s0->in2out.port;
1091               udp0->checksum = 0;
1092             }
1093
1094           /* Accounting */
1095           s0->last_heard = now;
1096           s0->total_pkts++;
1097           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1098           /* Per-user LRU list maintenance for dynamic translation */
1099           if (!snat_is_session_static (s0))
1100             {
1101               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1102                                  s0->per_user_index);
1103               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1104                                   s0->per_user_list_head_index,
1105                                   s0->per_user_index);
1106             }
1107         trace0:
1108
1109           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1110                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1111             {
1112               snat_out2in_trace_t *t =
1113                  vlib_add_trace (vm, node, b0, sizeof (*t));
1114               t->sw_if_index = sw_if_index0;
1115               t->next_index = next0;
1116               t->session_index = ~0;
1117               if (s0)
1118                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1119             }
1120
1121           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1122
1123
1124           ip1 = vlib_buffer_get_current (b1);
1125           udp1 = ip4_next_header (ip1);
1126           tcp1 = (tcp_header_t *) udp1;
1127           icmp1 = (icmp46_header_t *) udp1;
1128
1129           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1130           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1131                                    sw_if_index1);
1132
1133           if (PREDICT_FALSE(ip1->ttl == 1))
1134             {
1135               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1136               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1137                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1138                                            0);
1139               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1140               goto trace1;
1141             }
1142
1143           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1144
1145           if (PREDICT_FALSE (proto1 == ~0))
1146             {
1147               snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1148                                         thread_index, now, vm);
1149               goto trace1;
1150             }
1151
1152           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1153             {
1154               next1 = icmp_out2in_slow_path
1155                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1156                  next1, now, thread_index, &s1);
1157               goto trace1;
1158             }
1159
1160           key1.addr = ip1->dst_address;
1161           key1.port = udp1->dst_port;
1162           key1.protocol = proto1;
1163           key1.fib_index = rx_fib_index1;
1164
1165           kv1.key = key1.as_u64;
1166
1167           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
1168             {
1169               /* Try to match static mapping by external address and port,
1170                  destination address and port in packet */
1171               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1172                 {
1173                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1174                   /*
1175                    * Send DHCP packets to the ipv4 stack, or we won't
1176                    * be able to use dhcp client on the outside interface
1177                    */
1178                   if (proto1 != SNAT_PROTOCOL_UDP
1179                       || (udp1->dst_port
1180                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1181                     next1 = SNAT_OUT2IN_NEXT_DROP;
1182                   goto trace1;
1183                 }
1184
1185               /* Create session initiated by host from external network */
1186               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1187                                                      thread_index);
1188               if (!s1)
1189                 {
1190                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1191                   next1 = SNAT_OUT2IN_NEXT_DROP;
1192                   goto trace1;
1193                 }
1194             }
1195           else
1196             {
1197               if (PREDICT_FALSE (value1.value == ~0ULL))
1198                 {
1199                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index, now,
1200                                  vm);
1201                   goto trace1;
1202                 }
1203               else
1204                 {
1205                   s1 = pool_elt_at_index (
1206                     sm->per_thread_data[thread_index].sessions,
1207                     value1.value);
1208                 }
1209             }
1210
1211           old_addr1 = ip1->dst_address.as_u32;
1212           ip1->dst_address = s1->in2out.addr;
1213           new_addr1 = ip1->dst_address.as_u32;
1214           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1215
1216           sum1 = ip1->checksum;
1217           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1218                                  ip4_header_t,
1219                                  dst_address /* changed member */);
1220           ip1->checksum = ip_csum_fold (sum1);
1221
1222           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1223             {
1224               old_port1 = tcp1->dst_port;
1225               tcp1->dst_port = s1->in2out.port;
1226               new_port1 = tcp1->dst_port;
1227
1228               sum1 = tcp1->checksum;
1229               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1230                                      ip4_header_t,
1231                                      dst_address /* changed member */);
1232
1233               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1234                                      ip4_header_t /* cheat */,
1235                                      length /* changed member */);
1236               tcp1->checksum = ip_csum_fold(sum1);
1237             }
1238           else
1239             {
1240               old_port1 = udp1->dst_port;
1241               udp1->dst_port = s1->in2out.port;
1242               udp1->checksum = 0;
1243             }
1244
1245           /* Accounting */
1246           s1->last_heard = now;
1247           s1->total_pkts++;
1248           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1249           /* Per-user LRU list maintenance for dynamic translation */
1250           if (!snat_is_session_static (s1))
1251             {
1252               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1253                                  s1->per_user_index);
1254               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1255                                   s1->per_user_list_head_index,
1256                                   s1->per_user_index);
1257             }
1258         trace1:
1259
1260           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1261                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1262             {
1263               snat_out2in_trace_t *t =
1264                  vlib_add_trace (vm, node, b1, sizeof (*t));
1265               t->sw_if_index = sw_if_index1;
1266               t->next_index = next1;
1267               t->session_index = ~0;
1268               if (s1)
1269                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1270             }
1271
1272           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1273
1274           /* verify speculative enqueues, maybe switch current next frame */
1275           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1276                                            to_next, n_left_to_next,
1277                                            bi0, bi1, next0, next1);
1278         }
1279
1280       while (n_left_from > 0 && n_left_to_next > 0)
1281         {
1282           u32 bi0;
1283           vlib_buffer_t * b0;
1284           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1285           u32 sw_if_index0;
1286           ip4_header_t * ip0;
1287           ip_csum_t sum0;
1288           u32 new_addr0, old_addr0;
1289           u16 new_port0, old_port0;
1290           udp_header_t * udp0;
1291           tcp_header_t * tcp0;
1292           icmp46_header_t * icmp0;
1293           snat_session_key_t key0, sm0;
1294           u32 rx_fib_index0;
1295           u32 proto0;
1296           snat_session_t * s0 = 0;
1297           clib_bihash_kv_8_8_t kv0, value0;
1298
1299           /* speculatively enqueue b0 to the current next frame */
1300           bi0 = from[0];
1301           to_next[0] = bi0;
1302           from += 1;
1303           to_next += 1;
1304           n_left_from -= 1;
1305           n_left_to_next -= 1;
1306
1307           b0 = vlib_get_buffer (vm, bi0);
1308
1309           vnet_buffer (b0)->snat.flags = 0;
1310
1311           ip0 = vlib_buffer_get_current (b0);
1312           udp0 = ip4_next_header (ip0);
1313           tcp0 = (tcp_header_t *) udp0;
1314           icmp0 = (icmp46_header_t *) udp0;
1315
1316           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1317           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1318                                    sw_if_index0);
1319
1320           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1321
1322           if (PREDICT_FALSE (proto0 == ~0))
1323             {
1324               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1325                                         thread_index, now, vm);
1326               goto trace00;
1327             }
1328
1329           if (PREDICT_FALSE(ip0->ttl == 1))
1330             {
1331               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1332               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1333                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1334                                            0);
1335               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1336               goto trace00;
1337             }
1338
1339           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1340             {
1341               next0 = icmp_out2in_slow_path
1342                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1343                  next0, now, thread_index, &s0);
1344               goto trace00;
1345             }
1346
1347           key0.addr = ip0->dst_address;
1348           key0.port = udp0->dst_port;
1349           key0.protocol = proto0;
1350           key0.fib_index = rx_fib_index0;
1351
1352           kv0.key = key0.as_u64;
1353
1354           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1355             {
1356               /* Try to match static mapping by external address and port,
1357                  destination address and port in packet */
1358               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1359                 {
1360                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1361                   /*
1362                    * Send DHCP packets to the ipv4 stack, or we won't
1363                    * be able to use dhcp client on the outside interface
1364                    */
1365                   if (proto0 != SNAT_PROTOCOL_UDP
1366                       || (udp0->dst_port
1367                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1368
1369                     next0 = SNAT_OUT2IN_NEXT_DROP;
1370                   goto trace00;
1371                 }
1372
1373               /* Create session initiated by host from external network */
1374               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1375                                                      thread_index);
1376               if (!s0)
1377                 {
1378                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1379                     next0 = SNAT_OUT2IN_NEXT_DROP;
1380                   goto trace00;
1381                 }
1382             }
1383           else
1384             {
1385               if (PREDICT_FALSE (value0.value == ~0ULL))
1386                 {
1387                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index, now,
1388                                  vm);
1389                   goto trace00;
1390                 }
1391               else
1392                 {
1393                   s0 = pool_elt_at_index (
1394                     sm->per_thread_data[thread_index].sessions,
1395                     value0.value);
1396                 }
1397             }
1398
1399           old_addr0 = ip0->dst_address.as_u32;
1400           ip0->dst_address = s0->in2out.addr;
1401           new_addr0 = ip0->dst_address.as_u32;
1402           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1403
1404           sum0 = ip0->checksum;
1405           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1406                                  ip4_header_t,
1407                                  dst_address /* changed member */);
1408           ip0->checksum = ip_csum_fold (sum0);
1409
1410           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1411             {
1412               old_port0 = tcp0->dst_port;
1413               tcp0->dst_port = s0->in2out.port;
1414               new_port0 = tcp0->dst_port;
1415
1416               sum0 = tcp0->checksum;
1417               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1418                                      ip4_header_t,
1419                                      dst_address /* changed member */);
1420
1421               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1422                                      ip4_header_t /* cheat */,
1423                                      length /* changed member */);
1424               tcp0->checksum = ip_csum_fold(sum0);
1425             }
1426           else
1427             {
1428               old_port0 = udp0->dst_port;
1429               udp0->dst_port = s0->in2out.port;
1430               udp0->checksum = 0;
1431             }
1432
1433           /* Accounting */
1434           s0->last_heard = now;
1435           s0->total_pkts++;
1436           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1437           /* Per-user LRU list maintenance for dynamic translation */
1438           if (!snat_is_session_static (s0))
1439             {
1440               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1441                                  s0->per_user_index);
1442               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1443                                   s0->per_user_list_head_index,
1444                                   s0->per_user_index);
1445             }
1446         trace00:
1447
1448           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1449                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1450             {
1451               snat_out2in_trace_t *t =
1452                  vlib_add_trace (vm, node, b0, sizeof (*t));
1453               t->sw_if_index = sw_if_index0;
1454               t->next_index = next0;
1455               t->session_index = ~0;
1456               if (s0)
1457                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1458             }
1459
1460           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1461
1462           /* verify speculative enqueue, maybe switch current next frame */
1463           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1464                                            to_next, n_left_to_next,
1465                                            bi0, next0);
1466         }
1467
1468       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1469     }
1470
1471   vlib_node_increment_counter (vm, snat_out2in_node.index,
1472                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1473                                pkts_processed);
1474   return frame->n_vectors;
1475 }
1476
1477 VLIB_REGISTER_NODE (snat_out2in_node) = {
1478   .function = snat_out2in_node_fn,
1479   .name = "nat44-out2in",
1480   .vector_size = sizeof (u32),
1481   .format_trace = format_snat_out2in_trace,
1482   .type = VLIB_NODE_TYPE_INTERNAL,
1483
1484   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1485   .error_strings = snat_out2in_error_strings,
1486
1487   .runtime_data_bytes = sizeof (snat_runtime_t),
1488
1489   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1490
1491   /* edit / add dispositions here */
1492   .next_nodes = {
1493     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1494     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1495     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1496   },
1497 };
1498 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1499
1500 /**************************/
1501 /*** deterministic mode ***/
1502 /**************************/
1503 static uword
1504 snat_det_out2in_node_fn (vlib_main_t * vm,
1505                          vlib_node_runtime_t * node,
1506                          vlib_frame_t * frame)
1507 {
1508   u32 n_left_from, * from, * to_next;
1509   snat_out2in_next_t next_index;
1510   u32 pkts_processed = 0;
1511   snat_main_t * sm = &snat_main;
1512   u32 thread_index = vlib_get_thread_index ();
1513
1514   from = vlib_frame_vector_args (frame);
1515   n_left_from = frame->n_vectors;
1516   next_index = node->cached_next_index;
1517
1518   while (n_left_from > 0)
1519     {
1520       u32 n_left_to_next;
1521
1522       vlib_get_next_frame (vm, node, next_index,
1523                            to_next, n_left_to_next);
1524
1525       while (n_left_from >= 4 && n_left_to_next >= 2)
1526         {
1527           u32 bi0, bi1;
1528           vlib_buffer_t * b0, * b1;
1529           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1530           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1531           u32 sw_if_index0, sw_if_index1;
1532           ip4_header_t * ip0, * ip1;
1533           ip_csum_t sum0, sum1;
1534           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1535           u16 new_port0, old_port0, old_port1, new_port1;
1536           udp_header_t * udp0, * udp1;
1537           tcp_header_t * tcp0, * tcp1;
1538           u32 proto0, proto1;
1539           snat_det_out_key_t key0, key1;
1540           snat_det_map_t * dm0, * dm1;
1541           snat_det_session_t * ses0 = 0, * ses1 = 0;
1542           u32 rx_fib_index0, rx_fib_index1;
1543           icmp46_header_t * icmp0, * icmp1;
1544
1545           /* Prefetch next iteration. */
1546           {
1547             vlib_buffer_t * p2, * p3;
1548
1549             p2 = vlib_get_buffer (vm, from[2]);
1550             p3 = vlib_get_buffer (vm, from[3]);
1551
1552             vlib_prefetch_buffer_header (p2, LOAD);
1553             vlib_prefetch_buffer_header (p3, LOAD);
1554
1555             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1556             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1557           }
1558
1559           /* speculatively enqueue b0 and b1 to the current next frame */
1560           to_next[0] = bi0 = from[0];
1561           to_next[1] = bi1 = from[1];
1562           from += 2;
1563           to_next += 2;
1564           n_left_from -= 2;
1565           n_left_to_next -= 2;
1566
1567           b0 = vlib_get_buffer (vm, bi0);
1568           b1 = vlib_get_buffer (vm, bi1);
1569
1570           ip0 = vlib_buffer_get_current (b0);
1571           udp0 = ip4_next_header (ip0);
1572           tcp0 = (tcp_header_t *) udp0;
1573
1574           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1575
1576           if (PREDICT_FALSE(ip0->ttl == 1))
1577             {
1578               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1579               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1580                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1581                                            0);
1582               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1583               goto trace0;
1584             }
1585
1586           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1587
1588           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1589             {
1590               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1591               icmp0 = (icmp46_header_t *) udp0;
1592
1593               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1594                                   rx_fib_index0, node, next0, thread_index,
1595                                   &ses0, &dm0);
1596               goto trace0;
1597             }
1598
1599           key0.ext_host_addr = ip0->src_address;
1600           key0.ext_host_port = tcp0->src;
1601           key0.out_port = tcp0->dst;
1602
1603           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1604           if (PREDICT_FALSE(!dm0))
1605             {
1606               clib_warning("unknown dst address:  %U",
1607                            format_ip4_address, &ip0->dst_address);
1608               next0 = SNAT_OUT2IN_NEXT_DROP;
1609               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1610               goto trace0;
1611             }
1612
1613           snat_det_reverse(dm0, &ip0->dst_address,
1614                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1615
1616           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1617           if (PREDICT_FALSE(!ses0))
1618             {
1619               clib_warning("no match src %U:%d dst %U:%d for user %U",
1620                            format_ip4_address, &ip0->src_address,
1621                            clib_net_to_host_u16 (tcp0->src),
1622                            format_ip4_address, &ip0->dst_address,
1623                            clib_net_to_host_u16 (tcp0->dst),
1624                            format_ip4_address, &new_addr0);
1625               next0 = SNAT_OUT2IN_NEXT_DROP;
1626               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1627               goto trace0;
1628             }
1629           new_port0 = ses0->in_port;
1630
1631           old_addr0 = ip0->dst_address;
1632           ip0->dst_address = new_addr0;
1633           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1634
1635           sum0 = ip0->checksum;
1636           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1637                                  ip4_header_t,
1638                                  dst_address /* changed member */);
1639           ip0->checksum = ip_csum_fold (sum0);
1640
1641           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1642             {
1643               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1644                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1645               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1646                 snat_det_ses_close(dm0, ses0);
1647
1648               old_port0 = tcp0->dst;
1649               tcp0->dst = new_port0;
1650
1651               sum0 = tcp0->checksum;
1652               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1653                                      ip4_header_t,
1654                                      dst_address /* changed member */);
1655
1656               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1657                                      ip4_header_t /* cheat */,
1658                                      length /* changed member */);
1659               tcp0->checksum = ip_csum_fold(sum0);
1660             }
1661           else
1662             {
1663               old_port0 = udp0->dst_port;
1664               udp0->dst_port = new_port0;
1665               udp0->checksum = 0;
1666             }
1667
1668         trace0:
1669
1670           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1671                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1672             {
1673               snat_out2in_trace_t *t =
1674                  vlib_add_trace (vm, node, b0, sizeof (*t));
1675               t->sw_if_index = sw_if_index0;
1676               t->next_index = next0;
1677               t->session_index = ~0;
1678               if (ses0)
1679                 t->session_index = ses0 - dm0->sessions;
1680             }
1681
1682           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1683
1684           b1 = vlib_get_buffer (vm, bi1);
1685
1686           ip1 = vlib_buffer_get_current (b1);
1687           udp1 = ip4_next_header (ip1);
1688           tcp1 = (tcp_header_t *) udp1;
1689
1690           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1691
1692           if (PREDICT_FALSE(ip1->ttl == 1))
1693             {
1694               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1695               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1696                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1697                                            0);
1698               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1699               goto trace1;
1700             }
1701
1702           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1703
1704           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1705             {
1706               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1707               icmp1 = (icmp46_header_t *) udp1;
1708
1709               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1710                                   rx_fib_index1, node, next1, thread_index,
1711                                   &ses1, &dm1);
1712               goto trace1;
1713             }
1714
1715           key1.ext_host_addr = ip1->src_address;
1716           key1.ext_host_port = tcp1->src;
1717           key1.out_port = tcp1->dst;
1718
1719           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1720           if (PREDICT_FALSE(!dm1))
1721             {
1722               clib_warning("unknown dst address:  %U",
1723                            format_ip4_address, &ip1->dst_address);
1724               next1 = SNAT_OUT2IN_NEXT_DROP;
1725               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1726               goto trace1;
1727             }
1728
1729           snat_det_reverse(dm1, &ip1->dst_address,
1730                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1731
1732           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1733           if (PREDICT_FALSE(!ses1))
1734             {
1735               clib_warning("no match src %U:%d dst %U:%d for user %U",
1736                            format_ip4_address, &ip1->src_address,
1737                            clib_net_to_host_u16 (tcp1->src),
1738                            format_ip4_address, &ip1->dst_address,
1739                            clib_net_to_host_u16 (tcp1->dst),
1740                            format_ip4_address, &new_addr1);
1741               next1 = SNAT_OUT2IN_NEXT_DROP;
1742               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1743               goto trace1;
1744             }
1745           new_port1 = ses1->in_port;
1746
1747           old_addr1 = ip1->dst_address;
1748           ip1->dst_address = new_addr1;
1749           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1750
1751           sum1 = ip1->checksum;
1752           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1753                                  ip4_header_t,
1754                                  dst_address /* changed member */);
1755           ip1->checksum = ip_csum_fold (sum1);
1756
1757           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1758             {
1759               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1760                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1761               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1762                 snat_det_ses_close(dm1, ses1);
1763
1764               old_port1 = tcp1->dst;
1765               tcp1->dst = new_port1;
1766
1767               sum1 = tcp1->checksum;
1768               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1769                                      ip4_header_t,
1770                                      dst_address /* changed member */);
1771
1772               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1773                                      ip4_header_t /* cheat */,
1774                                      length /* changed member */);
1775               tcp1->checksum = ip_csum_fold(sum1);
1776             }
1777           else
1778             {
1779               old_port1 = udp1->dst_port;
1780               udp1->dst_port = new_port1;
1781               udp1->checksum = 0;
1782             }
1783
1784         trace1:
1785
1786           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1787                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1788             {
1789               snat_out2in_trace_t *t =
1790                  vlib_add_trace (vm, node, b1, sizeof (*t));
1791               t->sw_if_index = sw_if_index1;
1792               t->next_index = next1;
1793               t->session_index = ~0;
1794               if (ses1)
1795                 t->session_index = ses1 - dm1->sessions;
1796             }
1797
1798           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1799
1800           /* verify speculative enqueues, maybe switch current next frame */
1801           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1802                                            to_next, n_left_to_next,
1803                                            bi0, bi1, next0, next1);
1804          }
1805
1806       while (n_left_from > 0 && n_left_to_next > 0)
1807         {
1808           u32 bi0;
1809           vlib_buffer_t * b0;
1810           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1811           u32 sw_if_index0;
1812           ip4_header_t * ip0;
1813           ip_csum_t sum0;
1814           ip4_address_t new_addr0, old_addr0;
1815           u16 new_port0, old_port0;
1816           udp_header_t * udp0;
1817           tcp_header_t * tcp0;
1818           u32 proto0;
1819           snat_det_out_key_t key0;
1820           snat_det_map_t * dm0;
1821           snat_det_session_t * ses0 = 0;
1822           u32 rx_fib_index0;
1823           icmp46_header_t * icmp0;
1824
1825           /* speculatively enqueue b0 to the current next frame */
1826           bi0 = from[0];
1827           to_next[0] = bi0;
1828           from += 1;
1829           to_next += 1;
1830           n_left_from -= 1;
1831           n_left_to_next -= 1;
1832
1833           b0 = vlib_get_buffer (vm, bi0);
1834
1835           ip0 = vlib_buffer_get_current (b0);
1836           udp0 = ip4_next_header (ip0);
1837           tcp0 = (tcp_header_t *) udp0;
1838
1839           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1840
1841           if (PREDICT_FALSE(ip0->ttl == 1))
1842             {
1843               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1844               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1845                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1846                                            0);
1847               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1848               goto trace00;
1849             }
1850
1851           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1852
1853           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1854             {
1855               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1856               icmp0 = (icmp46_header_t *) udp0;
1857
1858               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1859                                   rx_fib_index0, node, next0, thread_index,
1860                                   &ses0, &dm0);
1861               goto trace00;
1862             }
1863
1864           key0.ext_host_addr = ip0->src_address;
1865           key0.ext_host_port = tcp0->src;
1866           key0.out_port = tcp0->dst;
1867
1868           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1869           if (PREDICT_FALSE(!dm0))
1870             {
1871               clib_warning("unknown dst address:  %U",
1872                            format_ip4_address, &ip0->dst_address);
1873               next0 = SNAT_OUT2IN_NEXT_DROP;
1874               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1875               goto trace00;
1876             }
1877
1878           snat_det_reverse(dm0, &ip0->dst_address,
1879                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1880
1881           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1882           if (PREDICT_FALSE(!ses0))
1883             {
1884               clib_warning("no match src %U:%d dst %U:%d for user %U",
1885                            format_ip4_address, &ip0->src_address,
1886                            clib_net_to_host_u16 (tcp0->src),
1887                            format_ip4_address, &ip0->dst_address,
1888                            clib_net_to_host_u16 (tcp0->dst),
1889                            format_ip4_address, &new_addr0);
1890               next0 = SNAT_OUT2IN_NEXT_DROP;
1891               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1892               goto trace00;
1893             }
1894           new_port0 = ses0->in_port;
1895
1896           old_addr0 = ip0->dst_address;
1897           ip0->dst_address = new_addr0;
1898           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1899
1900           sum0 = ip0->checksum;
1901           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1902                                  ip4_header_t,
1903                                  dst_address /* changed member */);
1904           ip0->checksum = ip_csum_fold (sum0);
1905
1906           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1907             {
1908               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1909                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1910               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1911                 snat_det_ses_close(dm0, ses0);
1912
1913               old_port0 = tcp0->dst;
1914               tcp0->dst = new_port0;
1915
1916               sum0 = tcp0->checksum;
1917               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1918                                      ip4_header_t,
1919                                      dst_address /* changed member */);
1920
1921               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1922                                      ip4_header_t /* cheat */,
1923                                      length /* changed member */);
1924               tcp0->checksum = ip_csum_fold(sum0);
1925             }
1926           else
1927             {
1928               old_port0 = udp0->dst_port;
1929               udp0->dst_port = new_port0;
1930               udp0->checksum = 0;
1931             }
1932
1933         trace00:
1934
1935           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1936                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1937             {
1938               snat_out2in_trace_t *t =
1939                  vlib_add_trace (vm, node, b0, sizeof (*t));
1940               t->sw_if_index = sw_if_index0;
1941               t->next_index = next0;
1942               t->session_index = ~0;
1943               if (ses0)
1944                 t->session_index = ses0 - dm0->sessions;
1945             }
1946
1947           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1948
1949           /* verify speculative enqueue, maybe switch current next frame */
1950           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1951                                            to_next, n_left_to_next,
1952                                            bi0, next0);
1953         }
1954
1955       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1956     }
1957
1958   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1959                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1960                                pkts_processed);
1961   return frame->n_vectors;
1962 }
1963
1964 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1965   .function = snat_det_out2in_node_fn,
1966   .name = "nat44-det-out2in",
1967   .vector_size = sizeof (u32),
1968   .format_trace = format_snat_out2in_trace,
1969   .type = VLIB_NODE_TYPE_INTERNAL,
1970
1971   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1972   .error_strings = snat_out2in_error_strings,
1973
1974   .runtime_data_bytes = sizeof (snat_runtime_t),
1975
1976   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1977
1978   /* edit / add dispositions here */
1979   .next_nodes = {
1980     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1981     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1982     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1983   },
1984 };
1985 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1986
1987 /**
1988  * Get address and port values to be used for ICMP packet translation
1989  * and create session if needed
1990  *
1991  * @param[in,out] sm             NAT main
1992  * @param[in,out] node           NAT node runtime
1993  * @param[in] thread_index       thread index
1994  * @param[in,out] b0             buffer containing packet to be translated
1995  * @param[out] p_proto           protocol used for matching
1996  * @param[out] p_value           address and port after NAT translation
1997  * @param[out] p_dont_translate  if packet should not be translated
1998  * @param d                      optional parameter
1999  * @param e                      optional parameter
2000  */
2001 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2002                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2003                           snat_session_key_t *p_value,
2004                           u8 *p_dont_translate, void *d, void *e)
2005 {
2006   ip4_header_t *ip0;
2007   icmp46_header_t *icmp0;
2008   u32 sw_if_index0;
2009   u8 protocol;
2010   snat_det_out_key_t key0;
2011   u8 dont_translate = 0;
2012   u32 next0 = ~0;
2013   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2014   ip4_header_t *inner_ip0;
2015   void *l4_header = 0;
2016   icmp46_header_t *inner_icmp0;
2017   snat_det_map_t * dm0 = 0;
2018   ip4_address_t new_addr0 = {{0}};
2019   snat_det_session_t * ses0 = 0;
2020   ip4_address_t out_addr;
2021
2022   ip0 = vlib_buffer_get_current (b0);
2023   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2024   echo0 = (icmp_echo_header_t *)(icmp0+1);
2025   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2026
2027   if (!icmp_is_error_message (icmp0))
2028     {
2029       protocol = SNAT_PROTOCOL_ICMP;
2030       key0.ext_host_addr = ip0->src_address;
2031       key0.ext_host_port = 0;
2032       key0.out_port = echo0->identifier;
2033       out_addr = ip0->dst_address;
2034     }
2035   else
2036     {
2037       inner_ip0 = (ip4_header_t *)(echo0+1);
2038       l4_header = ip4_next_header (inner_ip0);
2039       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2040       key0.ext_host_addr = inner_ip0->dst_address;
2041       out_addr = inner_ip0->src_address;
2042       switch (protocol)
2043         {
2044         case SNAT_PROTOCOL_ICMP:
2045           inner_icmp0 = (icmp46_header_t*)l4_header;
2046           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2047           key0.ext_host_port = 0;
2048           key0.out_port = inner_echo0->identifier;
2049           break;
2050         case SNAT_PROTOCOL_UDP:
2051         case SNAT_PROTOCOL_TCP:
2052           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2053           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2054           break;
2055         default:
2056           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2057           next0 = SNAT_OUT2IN_NEXT_DROP;
2058           goto out;
2059         }
2060     }
2061
2062   dm0 = snat_det_map_by_out(sm, &out_addr);
2063   if (PREDICT_FALSE(!dm0))
2064     {
2065       /* Don't NAT packet aimed at the intfc address */
2066       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2067                                           ip0->dst_address.as_u32)))
2068         {
2069           dont_translate = 1;
2070           goto out;
2071         }
2072       clib_warning("unknown dst address:  %U",
2073                    format_ip4_address, &ip0->dst_address);
2074       goto out;
2075     }
2076
2077   snat_det_reverse(dm0, &ip0->dst_address,
2078                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2079
2080   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2081   if (PREDICT_FALSE(!ses0))
2082     {
2083       /* Don't NAT packet aimed at the intfc address */
2084       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2085                                           ip0->dst_address.as_u32)))
2086         {
2087           dont_translate = 1;
2088           goto out;
2089         }
2090       clib_warning("no match src %U:%d dst %U:%d for user %U",
2091                    format_ip4_address, &key0.ext_host_addr,
2092                    clib_net_to_host_u16 (key0.ext_host_port),
2093                    format_ip4_address, &out_addr,
2094                    clib_net_to_host_u16 (key0.out_port),
2095                    format_ip4_address, &new_addr0);
2096       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2097       next0 = SNAT_OUT2IN_NEXT_DROP;
2098       goto out;
2099     }
2100
2101   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2102                     !icmp_is_error_message (icmp0)))
2103     {
2104       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2105       next0 = SNAT_OUT2IN_NEXT_DROP;
2106       goto out;
2107     }
2108
2109   goto out;
2110
2111 out:
2112   *p_proto = protocol;
2113   if (ses0)
2114     {
2115       p_value->addr = new_addr0;
2116       p_value->fib_index = sm->inside_fib_index;
2117       p_value->port = ses0->in_port;
2118     }
2119   *p_dont_translate = dont_translate;
2120   if (d)
2121     *(snat_det_session_t**)d = ses0;
2122   if (e)
2123     *(snat_det_map_t**)e = dm0;
2124   return next0;
2125 }
2126
2127 /**********************/
2128 /*** worker handoff ***/
2129 /**********************/
2130 static uword
2131 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2132                                vlib_node_runtime_t * node,
2133                                vlib_frame_t * frame)
2134 {
2135   snat_main_t *sm = &snat_main;
2136   vlib_thread_main_t *tm = vlib_get_thread_main ();
2137   u32 n_left_from, *from, *to_next = 0;
2138   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2139   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2140     = 0;
2141   vlib_frame_queue_elt_t *hf = 0;
2142   vlib_frame_t *f = 0;
2143   int i;
2144   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2145   u32 next_worker_index = 0;
2146   u32 current_worker_index = ~0;
2147   u32 thread_index = vlib_get_thread_index ();
2148
2149   ASSERT (vec_len (sm->workers));
2150
2151   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2152     {
2153       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2154
2155       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2156                                sm->first_worker_index + sm->num_workers - 1,
2157                                (vlib_frame_queue_t *) (~0));
2158     }
2159
2160   from = vlib_frame_vector_args (frame);
2161   n_left_from = frame->n_vectors;
2162
2163   while (n_left_from > 0)
2164     {
2165       u32 bi0;
2166       vlib_buffer_t *b0;
2167       u32 sw_if_index0;
2168       u32 rx_fib_index0;
2169       ip4_header_t * ip0;
2170       u8 do_handoff;
2171
2172       bi0 = from[0];
2173       from += 1;
2174       n_left_from -= 1;
2175
2176       b0 = vlib_get_buffer (vm, bi0);
2177
2178       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2179       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2180
2181       ip0 = vlib_buffer_get_current (b0);
2182
2183       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2184
2185       if (PREDICT_FALSE (next_worker_index != thread_index))
2186         {
2187           do_handoff = 1;
2188
2189           if (next_worker_index != current_worker_index)
2190             {
2191               if (hf)
2192                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2193
2194               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2195                                                       next_worker_index,
2196                                                       handoff_queue_elt_by_worker_index);
2197
2198               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2199               to_next_worker = &hf->buffer_index[hf->n_vectors];
2200               current_worker_index = next_worker_index;
2201             }
2202
2203           /* enqueue to correct worker thread */
2204           to_next_worker[0] = bi0;
2205           to_next_worker++;
2206           n_left_to_next_worker--;
2207
2208           if (n_left_to_next_worker == 0)
2209             {
2210               hf->n_vectors = VLIB_FRAME_SIZE;
2211               vlib_put_frame_queue_elt (hf);
2212               current_worker_index = ~0;
2213               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2214               hf = 0;
2215             }
2216         }
2217       else
2218         {
2219           do_handoff = 0;
2220           /* if this is 1st frame */
2221           if (!f)
2222             {
2223               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2224               to_next = vlib_frame_vector_args (f);
2225             }
2226
2227           to_next[0] = bi0;
2228           to_next += 1;
2229           f->n_vectors++;
2230         }
2231
2232       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2233                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2234         {
2235           snat_out2in_worker_handoff_trace_t *t =
2236             vlib_add_trace (vm, node, b0, sizeof (*t));
2237           t->next_worker_index = next_worker_index;
2238           t->do_handoff = do_handoff;
2239         }
2240     }
2241
2242   if (f)
2243     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2244
2245   if (hf)
2246     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2247
2248   /* Ship frames to the worker nodes */
2249   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2250     {
2251       if (handoff_queue_elt_by_worker_index[i])
2252         {
2253           hf = handoff_queue_elt_by_worker_index[i];
2254           /*
2255            * It works better to let the handoff node
2256            * rate-adapt, always ship the handoff queue element.
2257            */
2258           if (1 || hf->n_vectors == hf->last_n_vectors)
2259             {
2260               vlib_put_frame_queue_elt (hf);
2261               handoff_queue_elt_by_worker_index[i] = 0;
2262             }
2263           else
2264             hf->last_n_vectors = hf->n_vectors;
2265         }
2266       congested_handoff_queue_by_worker_index[i] =
2267         (vlib_frame_queue_t *) (~0);
2268     }
2269   hf = 0;
2270   current_worker_index = ~0;
2271   return frame->n_vectors;
2272 }
2273
2274 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2275   .function = snat_out2in_worker_handoff_fn,
2276   .name = "nat44-out2in-worker-handoff",
2277   .vector_size = sizeof (u32),
2278   .format_trace = format_snat_out2in_worker_handoff_trace,
2279   .type = VLIB_NODE_TYPE_INTERNAL,
2280
2281   .n_next_nodes = 1,
2282
2283   .next_nodes = {
2284     [0] = "error-drop",
2285   },
2286 };
2287
2288 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2289
2290 static uword
2291 snat_out2in_fast_node_fn (vlib_main_t * vm,
2292                           vlib_node_runtime_t * node,
2293                           vlib_frame_t * frame)
2294 {
2295   u32 n_left_from, * from, * to_next;
2296   snat_out2in_next_t next_index;
2297   u32 pkts_processed = 0;
2298   snat_main_t * sm = &snat_main;
2299
2300   from = vlib_frame_vector_args (frame);
2301   n_left_from = frame->n_vectors;
2302   next_index = node->cached_next_index;
2303
2304   while (n_left_from > 0)
2305     {
2306       u32 n_left_to_next;
2307
2308       vlib_get_next_frame (vm, node, next_index,
2309                            to_next, n_left_to_next);
2310
2311       while (n_left_from > 0 && n_left_to_next > 0)
2312         {
2313           u32 bi0;
2314           vlib_buffer_t * b0;
2315           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2316           u32 sw_if_index0;
2317           ip4_header_t * ip0;
2318           ip_csum_t sum0;
2319           u32 new_addr0, old_addr0;
2320           u16 new_port0, old_port0;
2321           udp_header_t * udp0;
2322           tcp_header_t * tcp0;
2323           icmp46_header_t * icmp0;
2324           snat_session_key_t key0, sm0;
2325           u32 proto0;
2326           u32 rx_fib_index0;
2327
2328           /* speculatively enqueue b0 to the current next frame */
2329           bi0 = from[0];
2330           to_next[0] = bi0;
2331           from += 1;
2332           to_next += 1;
2333           n_left_from -= 1;
2334           n_left_to_next -= 1;
2335
2336           b0 = vlib_get_buffer (vm, bi0);
2337
2338           ip0 = vlib_buffer_get_current (b0);
2339           udp0 = ip4_next_header (ip0);
2340           tcp0 = (tcp_header_t *) udp0;
2341           icmp0 = (icmp46_header_t *) udp0;
2342
2343           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2344           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2345
2346           vnet_feature_next (sw_if_index0, &next0, b0);
2347
2348           if (PREDICT_FALSE(ip0->ttl == 1))
2349             {
2350               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2351               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2352                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2353                                            0);
2354               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2355               goto trace00;
2356             }
2357
2358           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2359
2360           if (PREDICT_FALSE (proto0 == ~0))
2361               goto trace00;
2362
2363           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2364             {
2365               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2366                                   rx_fib_index0, node, next0, ~0, 0, 0);
2367               goto trace00;
2368             }
2369
2370           key0.addr = ip0->dst_address;
2371           key0.port = udp0->dst_port;
2372           key0.fib_index = rx_fib_index0;
2373
2374           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2375             {
2376               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2377               goto trace00;
2378             }
2379
2380           new_addr0 = sm0.addr.as_u32;
2381           new_port0 = sm0.port;
2382           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2383           old_addr0 = ip0->dst_address.as_u32;
2384           ip0->dst_address.as_u32 = new_addr0;
2385
2386           sum0 = ip0->checksum;
2387           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2388                                  ip4_header_t,
2389                                  dst_address /* changed member */);
2390           ip0->checksum = ip_csum_fold (sum0);
2391
2392           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2393             {
2394                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2395                 {
2396                   old_port0 = tcp0->dst_port;
2397                   tcp0->dst_port = new_port0;
2398
2399                   sum0 = tcp0->checksum;
2400                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2401                                          ip4_header_t,
2402                                          dst_address /* changed member */);
2403
2404                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2405                                          ip4_header_t /* cheat */,
2406                                          length /* changed member */);
2407                   tcp0->checksum = ip_csum_fold(sum0);
2408                 }
2409               else
2410                 {
2411                   old_port0 = udp0->dst_port;
2412                   udp0->dst_port = new_port0;
2413                   udp0->checksum = 0;
2414                 }
2415             }
2416           else
2417             {
2418               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2419                 {
2420                   sum0 = tcp0->checksum;
2421                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2422                                          ip4_header_t,
2423                                          dst_address /* changed member */);
2424
2425                   tcp0->checksum = ip_csum_fold(sum0);
2426                 }
2427             }
2428
2429         trace00:
2430
2431           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2432                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2433             {
2434               snat_out2in_trace_t *t =
2435                  vlib_add_trace (vm, node, b0, sizeof (*t));
2436               t->sw_if_index = sw_if_index0;
2437               t->next_index = next0;
2438             }
2439
2440           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2441
2442           /* verify speculative enqueue, maybe switch current next frame */
2443           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2444                                            to_next, n_left_to_next,
2445                                            bi0, next0);
2446         }
2447
2448       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2449     }
2450
2451   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2452                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2453                                pkts_processed);
2454   return frame->n_vectors;
2455 }
2456
2457 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2458   .function = snat_out2in_fast_node_fn,
2459   .name = "nat44-out2in-fast",
2460   .vector_size = sizeof (u32),
2461   .format_trace = format_snat_out2in_fast_trace,
2462   .type = VLIB_NODE_TYPE_INTERNAL,
2463
2464   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2465   .error_strings = snat_out2in_error_strings,
2466
2467   .runtime_data_bytes = sizeof (snat_runtime_t),
2468
2469   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2470
2471   /* edit / add dispositions here */
2472   .next_nodes = {
2473     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2474     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2475     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2476   },
2477 };
2478 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);