src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")
  91
  92 typedef enum {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96   SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char * snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 /**
 113  * @brief Create session for static mapping.
 114  *
 115  * Create NAT session initiated by host from external network with static
 116  * mapping.
 117  *
 118  * @param sm     NAT main.
 119  * @param b0     Vlib buffer.
 120  * @param in2out In2out NAT44 session key.
 121  * @param out2in Out2in NAT44 session key.
 122  * @param node   Vlib node.
 123  *
 124  * @returns SNAT session if successfully created otherwise 0.
 125  */
 126 static inline snat_session_t *
 127 create_session_for_static_mapping (snat_main_t *sm,
 128                                    vlib_buffer_t *b0,
 129                                    snat_session_key_t in2out,
 130                                    snat_session_key_t out2in,
 131                                    vlib_node_runtime_t * node,
 132                                    u32 thread_index)
 133 {
 134   snat_user_t *u;
 135   snat_user_key_t user_key;
 136   snat_session_t *s;
 137   clib_bihash_kv_8_8_t kv0, value0;
 138   dlist_elt_t * per_user_translation_list_elt;
 139   dlist_elt_t * per_user_list_head_elt;
 140   ip4_header_t *ip0;
 141
 142   ip0 = vlib_buffer_get_current (b0);
 143
 144   user_key.addr = in2out.addr;
 145   user_key.fib_index = in2out.fib_index;
 146   kv0.key = user_key.as_u64;
 147
 148   /* Ever heard of the "user" = inside ip4 address before? */
 149   if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
 150     {
 151       /* no, make a new one */
 152       pool_get (sm->per_thread_data[thread_index].users, u);
 153       memset (u, 0, sizeof (*u));
 154       u->addr = in2out.addr;
 155       u->fib_index = in2out.fib_index;
 156
 157       pool_get (sm->per_thread_data[thread_index].list_pool,
 158                 per_user_list_head_elt);
 159
 160       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 161         sm->per_thread_data[thread_index].list_pool;
 162
 163       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 164                        u->sessions_per_user_list_head_index);
 165
 166       kv0.value = u - sm->per_thread_data[thread_index].users;
 167
 168       /* add user */
 169       clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
 170
 171       /* add non-traslated packets worker lookup */
 172       kv0.value = thread_index;
 173       clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
 174     }
 175   else
 176     {
 177       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 178                              value0.value);
 179     }
 180
 181   pool_get (sm->per_thread_data[thread_index].sessions, s);
 182   memset (s, 0, sizeof (*s));
 183
 184   s->outside_address_index = ~0;
 185   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 186   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 187   u->nstaticsessions++;
 188
 189   /* Create list elts */
 190   pool_get (sm->per_thread_data[thread_index].list_pool,
 191             per_user_translation_list_elt);
 192   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 193                    per_user_translation_list_elt -
 194                    sm->per_thread_data[thread_index].list_pool);
 195
 196   per_user_translation_list_elt->value =
 197     s - sm->per_thread_data[thread_index].sessions;
 198   s->per_user_index =
 199     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 200   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 201
 202   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 203                       s->per_user_list_head_index,
 204                       per_user_translation_list_elt -
 205                       sm->per_thread_data[thread_index].list_pool);
 206
 207   s->in2out = in2out;
 208   s->out2in = out2in;
 209   s->in2out.protocol = out2in.protocol;
 210
 211   /* Add to translation hashes */
 212   kv0.key = s->in2out.as_u64;
 213   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 214   if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
 215       clib_warning ("in2out key add failed");
 216
 217   kv0.key = s->out2in.as_u64;
 218   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 219
 220   if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
 221       clib_warning ("out2in key add failed");
 222
 223   /* log NAT event */
 224   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 225                                       s->out2in.addr.as_u32,
 226                                       s->in2out.protocol,
 227                                       s->in2out.port,
 228                                       s->out2in.port,
 229                                       s->in2out.fib_index);
 230    return s;
 231 }
 232
 233 static_always_inline
 234 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 235                                  snat_session_key_t *p_key0)
 236 {
 237   icmp46_header_t *icmp0;
 238   snat_session_key_t key0;
 239   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 240   ip4_header_t *inner_ip0;
 241   void *l4_header = 0;
 242   icmp46_header_t *inner_icmp0;
 243
 244   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 245   echo0 = (icmp_echo_header_t *)(icmp0+1);
 246
 247   if (!icmp_is_error_message (icmp0))
 248     {
 249       key0.protocol = SNAT_PROTOCOL_ICMP;
 250       key0.addr = ip0->dst_address;
 251       key0.port = echo0->identifier;
 252     }
 253   else
 254     {
 255       inner_ip0 = (ip4_header_t *)(echo0+1);
 256       l4_header = ip4_next_header (inner_ip0);
 257       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 258       key0.addr = inner_ip0->src_address;
 259       switch (key0.protocol)
 260         {
 261         case SNAT_PROTOCOL_ICMP:
 262           inner_icmp0 = (icmp46_header_t*)l4_header;
 263           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 264           key0.port = inner_echo0->identifier;
 265           break;
 266         case SNAT_PROTOCOL_UDP:
 267         case SNAT_PROTOCOL_TCP:
 268           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 269           break;
 270         default:
 271           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 272         }
 273     }
 274   *p_key0 = key0;
 275   return -1; /* success */
 276 }
 277
 278 /**
 279  * Get address and port values to be used for ICMP packet translation
 280  * and create session if needed
 281  *
 282  * @param[in,out] sm             NAT main
 283  * @param[in,out] node           NAT node runtime
 284  * @param[in] thread_index       thread index
 285  * @param[in,out] b0             buffer containing packet to be translated
 286  * @param[out] p_proto           protocol used for matching
 287  * @param[out] p_value           address and port after NAT translation
 288  * @param[out] p_dont_translate  if packet should not be translated
 289  * @param d                      optional parameter
 290  * @param e                      optional parameter
 291  */
 292 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 293                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 294                            snat_session_key_t *p_value,
 295                            u8 *p_dont_translate, void *d, void *e)
 296 {
 297   ip4_header_t *ip0;
 298   icmp46_header_t *icmp0;
 299   u32 sw_if_index0;
 300   u32 rx_fib_index0;
 301   snat_session_key_t key0;
 302   snat_session_key_t sm0;
 303   snat_session_t *s0 = 0;
 304   u8 dont_translate = 0;
 305   clib_bihash_kv_8_8_t kv0, value0;
 306   u8 is_addr_only;
 307   u32 next0 = ~0;
 308   int err;
 309
 310   ip0 = vlib_buffer_get_current (b0);
 311   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 312   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 313   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 314
 315   key0.protocol = 0;
 316
 317   err = icmp_get_key (ip0, &key0);
 318   if (err != -1)
 319     {
 320       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 321       next0 = SNAT_OUT2IN_NEXT_DROP;
 322       goto out;
 323     }
 324   key0.fib_index = rx_fib_index0;
 325
 326   kv0.key = key0.as_u64;
 327
 328   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 329     {
 330       /* Try to match static mapping by external address and port,
 331          destination address and port in packet */
 332       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 333         {
 334           /* Don't NAT packet aimed at the intfc address */
 335           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 336                                               ip0->dst_address.as_u32)))
 337             {
 338               dont_translate = 1;
 339               goto out;
 340             }
 341           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 342           next0 = SNAT_OUT2IN_NEXT_DROP;
 343           goto out;
 344         }
 345
 346       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 347                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 348         {
 349           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 350           next0 = SNAT_OUT2IN_NEXT_DROP;
 351           goto out;
 352         }
 353
 354       /* Create session initiated by host from external network */
 355       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 356                                              node, thread_index);
 357
 358       if (!s0)
 359         {
 360           next0 = SNAT_OUT2IN_NEXT_DROP;
 361           goto out;
 362         }
 363     }
 364   else
 365     {
 366       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 367                         icmp0->type != ICMP4_echo_request &&
 368                         !icmp_is_error_message (icmp0)))
 369         {
 370           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 371           next0 = SNAT_OUT2IN_NEXT_DROP;
 372           goto out;
 373         }
 374
 375       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 376                               value0.value);
 377     }
 378
 379 out:
 380   *p_proto = key0.protocol;
 381   if (s0)
 382     *p_value = s0->in2out;
 383   *p_dont_translate = dont_translate;
 384   if (d)
 385     *(snat_session_t**)d = s0;
 386   return next0;
 387 }
 388
 389 /**
 390  * Get address and port values to be used for ICMP packet translation
 391  *
 392  * @param[in] sm                 NAT main
 393  * @param[in,out] node           NAT node runtime
 394  * @param[in] thread_index       thread index
 395  * @param[in,out] b0             buffer containing packet to be translated
 396  * @param[out] p_proto           protocol used for matching
 397  * @param[out] p_value           address and port after NAT translation
 398  * @param[out] p_dont_translate  if packet should not be translated
 399  * @param d                      optional parameter
 400  * @param e                      optional parameter
 401  */
 402 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 403                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 404                            snat_session_key_t *p_value,
 405                            u8 *p_dont_translate, void *d, void *e)
 406 {
 407   ip4_header_t *ip0;
 408   icmp46_header_t *icmp0;
 409   u32 sw_if_index0;
 410   u32 rx_fib_index0;
 411   snat_session_key_t key0;
 412   snat_session_key_t sm0;
 413   u8 dont_translate = 0;
 414   u8 is_addr_only;
 415   u32 next0 = ~0;
 416   int err;
 417
 418   ip0 = vlib_buffer_get_current (b0);
 419   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 420   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 421   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 422
 423   err = icmp_get_key (ip0, &key0);
 424   if (err != -1)
 425     {
 426       b0->error = node->errors[err];
 427       next0 = SNAT_OUT2IN_NEXT_DROP;
 428       goto out2;
 429     }
 430   key0.fib_index = rx_fib_index0;
 431
 432   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 433     {
 434       /* Don't NAT packet aimed at the intfc address */
 435       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 436         {
 437           dont_translate = 1;
 438           goto out;
 439         }
 440       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 441       next0 = SNAT_OUT2IN_NEXT_DROP;
 442       goto out;
 443     }
 444
 445   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 446                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 447                     !icmp_is_error_message (icmp0)))
 448     {
 449       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 450       next0 = SNAT_OUT2IN_NEXT_DROP;
 451       goto out;
 452     }
 453
 454 out:
 455   *p_value = sm0;
 456 out2:
 457   *p_proto = key0.protocol;
 458   *p_dont_translate = dont_translate;
 459   return next0;
 460 }
 461
 462 static inline u32 icmp_out2in (snat_main_t *sm,
 463                                vlib_buffer_t * b0,
 464                                ip4_header_t * ip0,
 465                                icmp46_header_t * icmp0,
 466                                u32 sw_if_index0,
 467                                u32 rx_fib_index0,
 468                                vlib_node_runtime_t * node,
 469                                u32 next0,
 470                                u32 thread_index,
 471                                void *d,
 472                                void *e)
 473 {
 474   snat_session_key_t sm0;
 475   u8 protocol;
 476   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 477   ip4_header_t *inner_ip0 = 0;
 478   void *l4_header = 0;
 479   icmp46_header_t *inner_icmp0;
 480   u8 dont_translate;
 481   u32 new_addr0, old_addr0;
 482   u16 old_id0, new_id0;
 483   ip_csum_t sum0;
 484   u16 checksum0;
 485   u32 next0_tmp;
 486
 487   echo0 = (icmp_echo_header_t *)(icmp0+1);
 488
 489   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 490                                        &protocol, &sm0, &dont_translate, d, e);
 491   if (next0_tmp != ~0)
 492     next0 = next0_tmp;
 493   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 494     goto out;
 495
 496   sum0 = ip_incremental_checksum (0, icmp0,
 497                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 498   checksum0 = ~ip_csum_fold (sum0);
 499   if (checksum0 != 0 && checksum0 != 0xffff)
 500     {
 501       next0 = SNAT_OUT2IN_NEXT_DROP;
 502       goto out;
 503     }
 504
 505   old_addr0 = ip0->dst_address.as_u32;
 506   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 507   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 508
 509   sum0 = ip0->checksum;
 510   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 511                          dst_address /* changed member */);
 512   ip0->checksum = ip_csum_fold (sum0);
 513
 514   if (!icmp_is_error_message (icmp0))
 515     {
 516       new_id0 = sm0.port;
 517       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 518         {
 519           old_id0 = echo0->identifier;
 520           new_id0 = sm0.port;
 521           echo0->identifier = new_id0;
 522
 523           sum0 = icmp0->checksum;
 524           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 525                                  identifier /* changed member */);
 526           icmp0->checksum = ip_csum_fold (sum0);
 527         }
 528     }
 529   else
 530     {
 531       inner_ip0 = (ip4_header_t *)(echo0+1);
 532       l4_header = ip4_next_header (inner_ip0);
 533
 534       if (!ip4_header_checksum_is_valid (inner_ip0))
 535         {
 536           next0 = SNAT_OUT2IN_NEXT_DROP;
 537           goto out;
 538         }
 539
 540       old_addr0 = inner_ip0->src_address.as_u32;
 541       inner_ip0->src_address = sm0.addr;
 542       new_addr0 = inner_ip0->src_address.as_u32;
 543
 544       sum0 = icmp0->checksum;
 545       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 546                              src_address /* changed member */);
 547       icmp0->checksum = ip_csum_fold (sum0);
 548
 549       switch (protocol)
 550         {
 551         case SNAT_PROTOCOL_ICMP:
 552           inner_icmp0 = (icmp46_header_t*)l4_header;
 553           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 554
 555           old_id0 = inner_echo0->identifier;
 556           new_id0 = sm0.port;
 557           inner_echo0->identifier = new_id0;
 558
 559           sum0 = icmp0->checksum;
 560           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 561                                  identifier);
 562           icmp0->checksum = ip_csum_fold (sum0);
 563           break;
 564         case SNAT_PROTOCOL_UDP:
 565         case SNAT_PROTOCOL_TCP:
 566           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 567           new_id0 = sm0.port;
 568           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 569
 570           sum0 = icmp0->checksum;
 571           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 572                                  src_port);
 573           icmp0->checksum = ip_csum_fold (sum0);
 574           break;
 575         default:
 576           ASSERT(0);
 577         }
 578     }
 579
 580 out:
 581   return next0;
 582 }
 583
 584
 585 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 586                                          vlib_buffer_t * b0,
 587                                          ip4_header_t * ip0,
 588                                          icmp46_header_t * icmp0,
 589                                          u32 sw_if_index0,
 590                                          u32 rx_fib_index0,
 591                                          vlib_node_runtime_t * node,
 592                                          u32 next0, f64 now,
 593                                          u32 thread_index,
 594                                          snat_session_t ** p_s0)
 595 {
 596   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 597                       next0, thread_index, p_s0, 0);
 598   snat_session_t * s0 = *p_s0;
 599   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 600     {
 601       /* Accounting */
 602       s0->last_heard = now;
 603       s0->total_pkts++;
 604       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 605       /* Per-user LRU list maintenance for dynamic translation */
 606       if (!snat_is_session_static (s0))
 607         {
 608           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 609                              s0->per_user_index);
 610           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 611                               s0->per_user_list_head_index,
 612                               s0->per_user_index);
 613         }
 614     }
 615   return next0;
 616 }
 617
 618 static void
 619 snat_out2in_unknown_proto (snat_main_t *sm,
 620                            vlib_buffer_t * b,
 621                            ip4_header_t * ip,
 622                            u32 rx_fib_index,
 623                            u32 thread_index,
 624                            f64 now,
 625                            vlib_main_t * vm)
 626 {
 627   clib_bihash_kv_8_8_t kv, value;
 628   clib_bihash_kv_16_8_t s_kv, s_value;
 629   snat_static_mapping_t *m;
 630   snat_session_key_t m_key;
 631   u32 old_addr, new_addr;
 632   ip_csum_t sum;
 633   nat_ed_ses_key_t key;
 634   snat_session_t * s;
 635   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 636   snat_user_key_t u_key;
 637   snat_user_t *u;
 638   dlist_elt_t *head, *elt;
 639
 640   old_addr = ip->dst_address.as_u32;
 641
 642   key.l_addr = ip->dst_address;
 643   key.r_addr = ip->src_address;
 644   key.fib_index = rx_fib_index;
 645   key.proto = ip->protocol;
 646   key.rsvd = 0;
 647   key.l_port = 0;
 648   s_kv.key[0] = key.as_u64[0];
 649   s_kv.key[1] = key.as_u64[1];
 650
 651   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 652     {
 653       s = pool_elt_at_index (tsm->sessions, s_value.value);
 654       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 655     }
 656   else
 657     {
 658       m_key.addr = ip->dst_address;
 659       m_key.port = 0;
 660       m_key.protocol = 0;
 661       m_key.fib_index = rx_fib_index;
 662       kv.key = m_key.as_u64;
 663       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 664         return;
 665
 666       m = pool_elt_at_index (sm->static_mappings, value.value);
 667
 668       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 669
 670       u_key.addr = ip->src_address;
 671       u_key.fib_index = m->fib_index;
 672       kv.key = u_key.as_u64;
 673
 674       /* Ever heard of the "user" = src ip4 address before? */
 675       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
 676         {
 677           /* no, make a new one */
 678           pool_get (tsm->users, u);
 679           memset (u, 0, sizeof (*u));
 680           u->addr = ip->src_address;
 681           u->fib_index = rx_fib_index;
 682
 683           pool_get (tsm->list_pool, head);
 684           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 685
 686           clib_dlist_init (tsm->list_pool,
 687                            u->sessions_per_user_list_head_index);
 688
 689           kv.value = u - tsm->users;
 690
 691           /* add user */
 692           clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
 693         }
 694       else
 695         {
 696           u = pool_elt_at_index (tsm->users, value.value);
 697         }
 698
 699       /* Create a new session */
 700       pool_get (tsm->sessions, s);
 701       memset (s, 0, sizeof (*s));
 702
 703       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 704       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 705       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 706       s->outside_address_index = ~0;
 707       s->out2in.addr.as_u32 = old_addr;
 708       s->out2in.fib_index = rx_fib_index;
 709       s->in2out.addr.as_u32 = new_addr;
 710       s->in2out.fib_index = m->fib_index;
 711       s->in2out.port = s->out2in.port = ip->protocol;
 712       u->nstaticsessions++;
 713
 714       /* Create list elts */
 715       pool_get (tsm->list_pool, elt);
 716       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 717       elt->value = s - tsm->sessions;
 718       s->per_user_index = elt - tsm->list_pool;
 719       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 720       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 721                           s->per_user_index);
 722
 723       /* Add to lookup tables */
 724       s_kv.value = s - tsm->sessions;
 725       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 726         clib_warning ("out2in key add failed");
 727
 728       key.l_addr = ip->dst_address;
 729       key.fib_index = m->fib_index;
 730       s_kv.key[0] = key.as_u64[0];
 731       s_kv.key[1] = key.as_u64[1];
 732       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 733         clib_warning ("in2out key add failed");
 734    }
 735
 736   /* Update IP checksum */
 737   sum = ip->checksum;
 738   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 739   ip->checksum = ip_csum_fold (sum);
 740
 741   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 742
 743   /* Accounting */
 744   s->last_heard = now;
 745   s->total_pkts++;
 746   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 747   /* Per-user LRU list maintenance */
 748   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 749   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 750                       s->per_user_index);
 751 }
 752
 753 static void
 754 snat_out2in_lb (snat_main_t *sm,
 755                 vlib_buffer_t * b,
 756                 ip4_header_t * ip,
 757                 u32 rx_fib_index,
 758                 u32 thread_index,
 759                 f64 now,
 760                 vlib_main_t * vm)
 761 {
 762   nat_ed_ses_key_t key;
 763   clib_bihash_kv_16_8_t s_kv, s_value;
 764   udp_header_t *udp = ip4_next_header (ip);
 765   tcp_header_t *tcp = (tcp_header_t *) udp;
 766   snat_session_t *s = 0;
 767   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 768   snat_session_key_t e_key, l_key;
 769   clib_bihash_kv_8_8_t kv, value;
 770   u32 old_addr, new_addr;
 771   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 772   u16 new_port, old_port;
 773   ip_csum_t sum;
 774   snat_user_key_t u_key;
 775   snat_user_t *u;
 776   dlist_elt_t *head, *elt;
 777
 778   old_addr = ip->dst_address.as_u32;
 779
 780   key.l_addr = ip->dst_address;
 781   key.r_addr = ip->src_address;
 782   key.fib_index = rx_fib_index;
 783   key.proto = ip->protocol;
 784   key.rsvd = 0;
 785   key.l_port = udp->dst_port;
 786   s_kv.key[0] = key.as_u64[0];
 787   s_kv.key[1] = key.as_u64[1];
 788
 789   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 790     {
 791       s = pool_elt_at_index (tsm->sessions, s_value.value);
 792     }
 793   else
 794     {
 795       e_key.addr = ip->dst_address;
 796       e_key.port = udp->dst_port;
 797       e_key.protocol = proto;
 798       e_key.fib_index = rx_fib_index;
 799       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
 800         return;
 801
 802       u_key.addr = l_key.addr;
 803       u_key.fib_index = l_key.fib_index;
 804       kv.key = u_key.as_u64;
 805
 806       /* Ever heard of the "user" = src ip4 address before? */
 807       if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
 808         {
 809           /* no, make a new one */
 810           pool_get (tsm->users, u);
 811           memset (u, 0, sizeof (*u));
 812           u->addr = l_key.addr;
 813           u->fib_index = l_key.fib_index;
 814
 815           pool_get (tsm->list_pool, head);
 816           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 817
 818           clib_dlist_init (tsm->list_pool,
 819                            u->sessions_per_user_list_head_index);
 820
 821           kv.value = u - tsm->users;
 822
 823           /* add user */
 824           if (clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1))
 825             clib_warning ("user key add failed");
 826         }
 827       else
 828         {
 829           u = pool_elt_at_index (tsm->users, value.value);
 830         }
 831
 832       /* Create a new session */
 833       pool_get (tsm->sessions, s);
 834       memset (s, 0, sizeof (*s));
 835
 836       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 837       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 838       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 839       s->outside_address_index = ~0;
 840       s->out2in = e_key;
 841       s->in2out = l_key;
 842       u->nstaticsessions++;
 843
 844       /* Create list elts */
 845       pool_get (tsm->list_pool, elt);
 846       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 847       elt->value = s - tsm->sessions;
 848       s->per_user_index = elt - tsm->list_pool;
 849       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 850       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 851                           s->per_user_index);
 852
 853       /* Add to lookup tables */
 854       s_kv.value = s - tsm->sessions;
 855       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 856         clib_warning ("out2in-ed key add failed");
 857
 858       key.l_addr = l_key.addr;
 859       key.fib_index = l_key.fib_index;
 860       key.l_port = l_key.port;
 861       s_kv.key[0] = key.as_u64[0];
 862       s_kv.key[1] = key.as_u64[1];
 863       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 864         clib_warning ("in2out-ed key add failed");
 865     }
 866
 867   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 868
 869   /* Update IP checksum */
 870   sum = ip->checksum;
 871   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 872   ip->checksum = ip_csum_fold (sum);
 873
 874   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 875     {
 876       old_port = tcp->dst_port;
 877       tcp->dst_port = s->in2out.port;
 878       new_port = tcp->dst_port;
 879
 880       sum = tcp->checksum;
 881       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 882       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 883       tcp->checksum = ip_csum_fold(sum);
 884     }
 885   else
 886     {
 887       udp->dst_port = s->in2out.port;
 888       udp->checksum = 0;
 889     }
 890
 891   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 892
 893   /* Accounting */
 894   s->last_heard = now;
 895   s->total_pkts++;
 896   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 897 }
 898
 899 static uword
 900 snat_out2in_node_fn (vlib_main_t * vm,
 901                   vlib_node_runtime_t * node,
 902                   vlib_frame_t * frame)
 903 {
 904   u32 n_left_from, * from, * to_next;
 905   snat_out2in_next_t next_index;
 906   u32 pkts_processed = 0;
 907   snat_main_t * sm = &snat_main;
 908   f64 now = vlib_time_now (vm);
 909   u32 thread_index = vlib_get_thread_index ();
 910
 911   from = vlib_frame_vector_args (frame);
 912   n_left_from = frame->n_vectors;
 913   next_index = node->cached_next_index;
 914
 915   while (n_left_from > 0)
 916     {
 917       u32 n_left_to_next;
 918
 919       vlib_get_next_frame (vm, node, next_index,
 920                            to_next, n_left_to_next);
 921
 922       while (n_left_from >= 4 && n_left_to_next >= 2)
 923         {
 924           u32 bi0, bi1;
 925           vlib_buffer_t * b0, * b1;
 926           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 927           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 928           u32 sw_if_index0, sw_if_index1;
 929           ip4_header_t * ip0, *ip1;
 930           ip_csum_t sum0, sum1;
 931           u32 new_addr0, old_addr0;
 932           u16 new_port0, old_port0;
 933           u32 new_addr1, old_addr1;
 934           u16 new_port1, old_port1;
 935           udp_header_t * udp0, * udp1;
 936           tcp_header_t * tcp0, * tcp1;
 937           icmp46_header_t * icmp0, * icmp1;
 938           snat_session_key_t key0, key1, sm0, sm1;
 939           u32 rx_fib_index0, rx_fib_index1;
 940           u32 proto0, proto1;
 941           snat_session_t * s0 = 0, * s1 = 0;
 942           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 943
 944           /* Prefetch next iteration. */
 945           {
 946             vlib_buffer_t * p2, * p3;
 947
 948             p2 = vlib_get_buffer (vm, from[2]);
 949             p3 = vlib_get_buffer (vm, from[3]);
 950
 951             vlib_prefetch_buffer_header (p2, LOAD);
 952             vlib_prefetch_buffer_header (p3, LOAD);
 953
 954             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 955             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 956           }
 957
 958           /* speculatively enqueue b0 and b1 to the current next frame */
 959           to_next[0] = bi0 = from[0];
 960           to_next[1] = bi1 = from[1];
 961           from += 2;
 962           to_next += 2;
 963           n_left_from -= 2;
 964           n_left_to_next -= 2;
 965
 966           b0 = vlib_get_buffer (vm, bi0);
 967           b1 = vlib_get_buffer (vm, bi1);
 968
 969           vnet_buffer (b0)->snat.flags = 0;
 970           vnet_buffer (b1)->snat.flags = 0;
 971
 972           ip0 = vlib_buffer_get_current (b0);
 973           udp0 = ip4_next_header (ip0);
 974           tcp0 = (tcp_header_t *) udp0;
 975           icmp0 = (icmp46_header_t *) udp0;
 976
 977           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 978           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 979                                    sw_if_index0);
 980
 981           if (PREDICT_FALSE(ip0->ttl == 1))
 982             {
 983               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 984               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 985                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 986                                            0);
 987               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 988               goto trace0;
 989             }
 990
 991           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 992
 993           if (PREDICT_FALSE (proto0 == ~0))
 994             {
 995               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
 996                                         thread_index, now, vm);
 997               goto trace0;
 998             }
 999
1000           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1001             {
1002               next0 = icmp_out2in_slow_path
1003                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1004                  next0, now, thread_index, &s0);
1005               goto trace0;
1006             }
1007
1008           key0.addr = ip0->dst_address;
1009           key0.port = udp0->dst_port;
1010           key0.protocol = proto0;
1011           key0.fib_index = rx_fib_index0;
1012
1013           kv0.key = key0.as_u64;
1014
1015           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1016             {
1017               /* Try to match static mapping by external address and port,
1018                  destination address and port in packet */
1019               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1020                 {
1021                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1022                   /*
1023                    * Send DHCP packets to the ipv4 stack, or we won't
1024                    * be able to use dhcp client on the outside interface
1025                    */
1026                   if (proto0 != SNAT_PROTOCOL_UDP
1027                       || (udp0->dst_port
1028                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1029                     next0 = SNAT_OUT2IN_NEXT_DROP;
1030                   goto trace0;
1031                 }
1032
1033               /* Create session initiated by host from external network */
1034               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1035                                                      thread_index);
1036               if (!s0)
1037                 {
1038                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1039                   next0 = SNAT_OUT2IN_NEXT_DROP;
1040                   goto trace0;
1041                 }
1042             }
1043           else
1044             {
1045               if (PREDICT_FALSE (value0.value == ~0ULL))
1046                 {
1047                   snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index, now,
1048                                  vm);
1049                   goto trace0;
1050                 }
1051               else
1052                 {
1053                   s0 = pool_elt_at_index (
1054                     sm->per_thread_data[thread_index].sessions,
1055                     value0.value);
1056                 }
1057             }
1058
1059           old_addr0 = ip0->dst_address.as_u32;
1060           ip0->dst_address = s0->in2out.addr;
1061           new_addr0 = ip0->dst_address.as_u32;
1062           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1063
1064           sum0 = ip0->checksum;
1065           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1066                                  ip4_header_t,
1067                                  dst_address /* changed member */);
1068           ip0->checksum = ip_csum_fold (sum0);
1069
1070           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1071             {
1072               old_port0 = tcp0->dst_port;
1073               tcp0->dst_port = s0->in2out.port;
1074               new_port0 = tcp0->dst_port;
1075
1076               sum0 = tcp0->checksum;
1077               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1078                                      ip4_header_t,
1079                                      dst_address /* changed member */);
1080
1081               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1082                                      ip4_header_t /* cheat */,
1083                                      length /* changed member */);
1084               tcp0->checksum = ip_csum_fold(sum0);
1085             }
1086           else
1087             {
1088               old_port0 = udp0->dst_port;
1089               udp0->dst_port = s0->in2out.port;
1090               udp0->checksum = 0;
1091             }
1092
1093           /* Accounting */
1094           s0->last_heard = now;
1095           s0->total_pkts++;
1096           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1097           /* Per-user LRU list maintenance for dynamic translation */
1098           if (!snat_is_session_static (s0))
1099             {
1100               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1101                                  s0->per_user_index);
1102               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1103                                   s0->per_user_list_head_index,
1104                                   s0->per_user_index);
1105             }
1106         trace0:
1107
1108           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1109                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1110             {
1111               snat_out2in_trace_t *t =
1112                  vlib_add_trace (vm, node, b0, sizeof (*t));
1113               t->sw_if_index = sw_if_index0;
1114               t->next_index = next0;
1115               t->session_index = ~0;
1116               if (s0)
1117                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1118             }
1119
1120           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1121
1122
1123           ip1 = vlib_buffer_get_current (b1);
1124           udp1 = ip4_next_header (ip1);
1125           tcp1 = (tcp_header_t *) udp1;
1126           icmp1 = (icmp46_header_t *) udp1;
1127
1128           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1129           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1130                                    sw_if_index1);
1131
1132           if (PREDICT_FALSE(ip1->ttl == 1))
1133             {
1134               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1135               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1136                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1137                                            0);
1138               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1139               goto trace1;
1140             }
1141
1142           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1143
1144           if (PREDICT_FALSE (proto1 == ~0))
1145             {
1146               snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1147                                         thread_index, now, vm);
1148               goto trace1;
1149             }
1150
1151           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1152             {
1153               next1 = icmp_out2in_slow_path
1154                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1155                  next1, now, thread_index, &s1);
1156               goto trace1;
1157             }
1158
1159           key1.addr = ip1->dst_address;
1160           key1.port = udp1->dst_port;
1161           key1.protocol = proto1;
1162           key1.fib_index = rx_fib_index1;
1163
1164           kv1.key = key1.as_u64;
1165
1166           if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
1167             {
1168               /* Try to match static mapping by external address and port,
1169                  destination address and port in packet */
1170               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1171                 {
1172                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1173                   /*
1174                    * Send DHCP packets to the ipv4 stack, or we won't
1175                    * be able to use dhcp client on the outside interface
1176                    */
1177                   if (proto1 != SNAT_PROTOCOL_UDP
1178                       || (udp1->dst_port
1179                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1180                     next1 = SNAT_OUT2IN_NEXT_DROP;
1181                   goto trace1;
1182                 }
1183
1184               /* Create session initiated by host from external network */
1185               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1186                                                      thread_index);
1187               if (!s1)
1188                 {
1189                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1190                   next1 = SNAT_OUT2IN_NEXT_DROP;
1191                   goto trace1;
1192                 }
1193             }
1194           else
1195             {
1196               if (PREDICT_FALSE (value1.value == ~0ULL))
1197                 {
1198                   snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index, now,
1199                                  vm);
1200                   goto trace1;
1201                 }
1202               else
1203                 {
1204                   s1 = pool_elt_at_index (
1205                     sm->per_thread_data[thread_index].sessions,
1206                     value1.value);
1207                 }
1208             }
1209
1210           old_addr1 = ip1->dst_address.as_u32;
1211           ip1->dst_address = s1->in2out.addr;
1212           new_addr1 = ip1->dst_address.as_u32;
1213           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1214
1215           sum1 = ip1->checksum;
1216           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1217                                  ip4_header_t,
1218                                  dst_address /* changed member */);
1219           ip1->checksum = ip_csum_fold (sum1);
1220
1221           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1222             {
1223               old_port1 = tcp1->dst_port;
1224               tcp1->dst_port = s1->in2out.port;
1225               new_port1 = tcp1->dst_port;
1226
1227               sum1 = tcp1->checksum;
1228               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1229                                      ip4_header_t,
1230                                      dst_address /* changed member */);
1231
1232               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1233                                      ip4_header_t /* cheat */,
1234                                      length /* changed member */);
1235               tcp1->checksum = ip_csum_fold(sum1);
1236             }
1237           else
1238             {
1239               old_port1 = udp1->dst_port;
1240               udp1->dst_port = s1->in2out.port;
1241               udp1->checksum = 0;
1242             }
1243
1244           /* Accounting */
1245           s1->last_heard = now;
1246           s1->total_pkts++;
1247           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1248           /* Per-user LRU list maintenance for dynamic translation */
1249           if (!snat_is_session_static (s1))
1250             {
1251               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1252                                  s1->per_user_index);
1253               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1254                                   s1->per_user_list_head_index,
1255                                   s1->per_user_index);
1256             }
1257         trace1:
1258
1259           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1260                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1261             {
1262               snat_out2in_trace_t *t =
1263                  vlib_add_trace (vm, node, b1, sizeof (*t));
1264               t->sw_if_index = sw_if_index1;
1265               t->next_index = next1;
1266               t->session_index = ~0;
1267               if (s1)
1268                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1269             }
1270
1271           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1272
1273           /* verify speculative enqueues, maybe switch current next frame */
1274           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1275                                            to_next, n_left_to_next,
1276                                            bi0, bi1, next0, next1);
1277         }
1278
1279       while (n_left_from > 0 && n_left_to_next > 0)
1280         {
1281           u32 bi0;
1282           vlib_buffer_t * b0;
1283           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1284           u32 sw_if_index0;
1285           ip4_header_t * ip0;
1286           ip_csum_t sum0;
1287           u32 new_addr0, old_addr0;
1288           u16 new_port0, old_port0;
1289           udp_header_t * udp0;
1290           tcp_header_t * tcp0;
1291           icmp46_header_t * icmp0;
1292           snat_session_key_t key0, sm0;
1293           u32 rx_fib_index0;
1294           u32 proto0;
1295           snat_session_t * s0 = 0;
1296           clib_bihash_kv_8_8_t kv0, value0;
1297
1298           /* speculatively enqueue b0 to the current next frame */
1299           bi0 = from[0];
1300           to_next[0] = bi0;
1301           from += 1;
1302           to_next += 1;
1303           n_left_from -= 1;
1304           n_left_to_next -= 1;
1305
1306           b0 = vlib_get_buffer (vm, bi0);
1307
1308           vnet_buffer (b0)->snat.flags = 0;
1309
1310           ip0 = vlib_buffer_get_current (b0);
1311           udp0 = ip4_next_header (ip0);
1312           tcp0 = (tcp_header_t *) udp0;
1313           icmp0 = (icmp46_header_t *) udp0;
1314
1315           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1316           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1317                                    sw_if_index0);
1318
1319           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1320
1321           if (PREDICT_FALSE (proto0 == ~0))
1322             {
1323               snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1324                                         thread_index, now, vm);
1325               goto trace00;
1326             }
1327
1328           if (PREDICT_FALSE(ip0->ttl == 1))
1329             {
1330               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1331               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1332                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1333                                            0);
1334               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1335               goto trace00;
1336             }
1337
1338           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1339             {
1340               next0 = icmp_out2in_slow_path
1341                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1342                  next0, now, thread_index, &s0);
1343               goto trace00;
1344             }
1345
1346           key0.addr = ip0->dst_address;
1347           key0.port = udp0->dst_port;
1348           key0.protocol = proto0;
1349           key0.fib_index = rx_fib_index0;
1350
1351           kv0.key = key0.as_u64;
1352
1353           if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1354             {
1355               /* Try to match static mapping by external address and port,
1356                  destination address and port in packet */
1357               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1358                 {
1359                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1360                   /*
1361                    * Send DHCP packets to the ipv4 stack, or we won't
1362                    * be able to use dhcp client on the outside interface
1363                    */
1364                   if (proto0 != SNAT_PROTOCOL_UDP
1365                       || (udp0->dst_port
1366                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1367
1368                     next0 = SNAT_OUT2IN_NEXT_DROP;
1369                   goto trace00;
1370                 }
1371
1372               /* Create session initiated by host from external network */
1373               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1374                                                      thread_index);
1375               if (!s0)
1376                 {
1377                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1378                     next0 = SNAT_OUT2IN_NEXT_DROP;
1379                   goto trace00;
1380                 }
1381             }
1382           else
1383             {
1384               if (PREDICT_FALSE (value0.value == ~0ULL))
1385                 {
1386                   snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index, now,
1387                                  vm);
1388                   goto trace00;
1389                 }
1390               else
1391                 {
1392                   s0 = pool_elt_at_index (
1393                     sm->per_thread_data[thread_index].sessions,
1394                     value0.value);
1395                 }
1396             }
1397
1398           old_addr0 = ip0->dst_address.as_u32;
1399           ip0->dst_address = s0->in2out.addr;
1400           new_addr0 = ip0->dst_address.as_u32;
1401           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1402
1403           sum0 = ip0->checksum;
1404           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1405                                  ip4_header_t,
1406                                  dst_address /* changed member */);
1407           ip0->checksum = ip_csum_fold (sum0);
1408
1409           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1410             {
1411               old_port0 = tcp0->dst_port;
1412               tcp0->dst_port = s0->in2out.port;
1413               new_port0 = tcp0->dst_port;
1414
1415               sum0 = tcp0->checksum;
1416               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1417                                      ip4_header_t,
1418                                      dst_address /* changed member */);
1419
1420               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1421                                      ip4_header_t /* cheat */,
1422                                      length /* changed member */);
1423               tcp0->checksum = ip_csum_fold(sum0);
1424             }
1425           else
1426             {
1427               old_port0 = udp0->dst_port;
1428               udp0->dst_port = s0->in2out.port;
1429               udp0->checksum = 0;
1430             }
1431
1432           /* Accounting */
1433           s0->last_heard = now;
1434           s0->total_pkts++;
1435           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1436           /* Per-user LRU list maintenance for dynamic translation */
1437           if (!snat_is_session_static (s0))
1438             {
1439               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1440                                  s0->per_user_index);
1441               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1442                                   s0->per_user_list_head_index,
1443                                   s0->per_user_index);
1444             }
1445         trace00:
1446
1447           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1448                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1449             {
1450               snat_out2in_trace_t *t =
1451                  vlib_add_trace (vm, node, b0, sizeof (*t));
1452               t->sw_if_index = sw_if_index0;
1453               t->next_index = next0;
1454               t->session_index = ~0;
1455               if (s0)
1456                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1457             }
1458
1459           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1460
1461           /* verify speculative enqueue, maybe switch current next frame */
1462           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1463                                            to_next, n_left_to_next,
1464                                            bi0, next0);
1465         }
1466
1467       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1468     }
1469
1470   vlib_node_increment_counter (vm, snat_out2in_node.index,
1471                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1472                                pkts_processed);
1473   return frame->n_vectors;
1474 }
1475
1476 VLIB_REGISTER_NODE (snat_out2in_node) = {
1477   .function = snat_out2in_node_fn,
1478   .name = "nat44-out2in",
1479   .vector_size = sizeof (u32),
1480   .format_trace = format_snat_out2in_trace,
1481   .type = VLIB_NODE_TYPE_INTERNAL,
1482
1483   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1484   .error_strings = snat_out2in_error_strings,
1485
1486   .runtime_data_bytes = sizeof (snat_runtime_t),
1487
1488   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1489
1490   /* edit / add dispositions here */
1491   .next_nodes = {
1492     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1493     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1494     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1495   },
1496 };
1497 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1498
1499 /**************************/
1500 /*** deterministic mode ***/
1501 /**************************/
1502 static uword
1503 snat_det_out2in_node_fn (vlib_main_t * vm,
1504                          vlib_node_runtime_t * node,
1505                          vlib_frame_t * frame)
1506 {
1507   u32 n_left_from, * from, * to_next;
1508   snat_out2in_next_t next_index;
1509   u32 pkts_processed = 0;
1510   snat_main_t * sm = &snat_main;
1511   u32 thread_index = vlib_get_thread_index ();
1512
1513   from = vlib_frame_vector_args (frame);
1514   n_left_from = frame->n_vectors;
1515   next_index = node->cached_next_index;
1516
1517   while (n_left_from > 0)
1518     {
1519       u32 n_left_to_next;
1520
1521       vlib_get_next_frame (vm, node, next_index,
1522                            to_next, n_left_to_next);
1523
1524       while (n_left_from >= 4 && n_left_to_next >= 2)
1525         {
1526           u32 bi0, bi1;
1527           vlib_buffer_t * b0, * b1;
1528           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1529           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1530           u32 sw_if_index0, sw_if_index1;
1531           ip4_header_t * ip0, * ip1;
1532           ip_csum_t sum0, sum1;
1533           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1534           u16 new_port0, old_port0, old_port1, new_port1;
1535           udp_header_t * udp0, * udp1;
1536           tcp_header_t * tcp0, * tcp1;
1537           u32 proto0, proto1;
1538           snat_det_out_key_t key0, key1;
1539           snat_det_map_t * dm0, * dm1;
1540           snat_det_session_t * ses0 = 0, * ses1 = 0;
1541           u32 rx_fib_index0, rx_fib_index1;
1542           icmp46_header_t * icmp0, * icmp1;
1543
1544           /* Prefetch next iteration. */
1545           {
1546             vlib_buffer_t * p2, * p3;
1547
1548             p2 = vlib_get_buffer (vm, from[2]);
1549             p3 = vlib_get_buffer (vm, from[3]);
1550
1551             vlib_prefetch_buffer_header (p2, LOAD);
1552             vlib_prefetch_buffer_header (p3, LOAD);
1553
1554             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1555             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1556           }
1557
1558           /* speculatively enqueue b0 and b1 to the current next frame */
1559           to_next[0] = bi0 = from[0];
1560           to_next[1] = bi1 = from[1];
1561           from += 2;
1562           to_next += 2;
1563           n_left_from -= 2;
1564           n_left_to_next -= 2;
1565
1566           b0 = vlib_get_buffer (vm, bi0);
1567           b1 = vlib_get_buffer (vm, bi1);
1568
1569           ip0 = vlib_buffer_get_current (b0);
1570           udp0 = ip4_next_header (ip0);
1571           tcp0 = (tcp_header_t *) udp0;
1572
1573           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1574
1575           if (PREDICT_FALSE(ip0->ttl == 1))
1576             {
1577               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1578               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1579                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1580                                            0);
1581               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1582               goto trace0;
1583             }
1584
1585           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1586
1587           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1588             {
1589               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1590               icmp0 = (icmp46_header_t *) udp0;
1591
1592               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1593                                   rx_fib_index0, node, next0, thread_index,
1594                                   &ses0, &dm0);
1595               goto trace0;
1596             }
1597
1598           key0.ext_host_addr = ip0->src_address;
1599           key0.ext_host_port = tcp0->src;
1600           key0.out_port = tcp0->dst;
1601
1602           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1603           if (PREDICT_FALSE(!dm0))
1604             {
1605               clib_warning("unknown dst address:  %U",
1606                            format_ip4_address, &ip0->dst_address);
1607               next0 = SNAT_OUT2IN_NEXT_DROP;
1608               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1609               goto trace0;
1610             }
1611
1612           snat_det_reverse(dm0, &ip0->dst_address,
1613                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1614
1615           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1616           if (PREDICT_FALSE(!ses0))
1617             {
1618               clib_warning("no match src %U:%d dst %U:%d for user %U",
1619                            format_ip4_address, &ip0->src_address,
1620                            clib_net_to_host_u16 (tcp0->src),
1621                            format_ip4_address, &ip0->dst_address,
1622                            clib_net_to_host_u16 (tcp0->dst),
1623                            format_ip4_address, &new_addr0);
1624               next0 = SNAT_OUT2IN_NEXT_DROP;
1625               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1626               goto trace0;
1627             }
1628           new_port0 = ses0->in_port;
1629
1630           old_addr0 = ip0->dst_address;
1631           ip0->dst_address = new_addr0;
1632           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1633
1634           sum0 = ip0->checksum;
1635           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1636                                  ip4_header_t,
1637                                  dst_address /* changed member */);
1638           ip0->checksum = ip_csum_fold (sum0);
1639
1640           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1641             {
1642               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1643                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1644               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1645                 snat_det_ses_close(dm0, ses0);
1646
1647               old_port0 = tcp0->dst;
1648               tcp0->dst = new_port0;
1649
1650               sum0 = tcp0->checksum;
1651               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1652                                      ip4_header_t,
1653                                      dst_address /* changed member */);
1654
1655               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1656                                      ip4_header_t /* cheat */,
1657                                      length /* changed member */);
1658               tcp0->checksum = ip_csum_fold(sum0);
1659             }
1660           else
1661             {
1662               old_port0 = udp0->dst_port;
1663               udp0->dst_port = new_port0;
1664               udp0->checksum = 0;
1665             }
1666
1667         trace0:
1668
1669           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1670                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1671             {
1672               snat_out2in_trace_t *t =
1673                  vlib_add_trace (vm, node, b0, sizeof (*t));
1674               t->sw_if_index = sw_if_index0;
1675               t->next_index = next0;
1676               t->session_index = ~0;
1677               if (ses0)
1678                 t->session_index = ses0 - dm0->sessions;
1679             }
1680
1681           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1682
1683           b1 = vlib_get_buffer (vm, bi1);
1684
1685           ip1 = vlib_buffer_get_current (b1);
1686           udp1 = ip4_next_header (ip1);
1687           tcp1 = (tcp_header_t *) udp1;
1688
1689           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1690
1691           if (PREDICT_FALSE(ip1->ttl == 1))
1692             {
1693               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1694               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1695                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1696                                            0);
1697               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1698               goto trace1;
1699             }
1700
1701           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1702
1703           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1704             {
1705               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1706               icmp1 = (icmp46_header_t *) udp1;
1707
1708               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1709                                   rx_fib_index1, node, next1, thread_index,
1710                                   &ses1, &dm1);
1711               goto trace1;
1712             }
1713
1714           key1.ext_host_addr = ip1->src_address;
1715           key1.ext_host_port = tcp1->src;
1716           key1.out_port = tcp1->dst;
1717
1718           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1719           if (PREDICT_FALSE(!dm1))
1720             {
1721               clib_warning("unknown dst address:  %U",
1722                            format_ip4_address, &ip1->dst_address);
1723               next1 = SNAT_OUT2IN_NEXT_DROP;
1724               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1725               goto trace1;
1726             }
1727
1728           snat_det_reverse(dm1, &ip1->dst_address,
1729                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1730
1731           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1732           if (PREDICT_FALSE(!ses1))
1733             {
1734               clib_warning("no match src %U:%d dst %U:%d for user %U",
1735                            format_ip4_address, &ip1->src_address,
1736                            clib_net_to_host_u16 (tcp1->src),
1737                            format_ip4_address, &ip1->dst_address,
1738                            clib_net_to_host_u16 (tcp1->dst),
1739                            format_ip4_address, &new_addr1);
1740               next1 = SNAT_OUT2IN_NEXT_DROP;
1741               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1742               goto trace1;
1743             }
1744           new_port1 = ses1->in_port;
1745
1746           old_addr1 = ip1->dst_address;
1747           ip1->dst_address = new_addr1;
1748           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1749
1750           sum1 = ip1->checksum;
1751           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1752                                  ip4_header_t,
1753                                  dst_address /* changed member */);
1754           ip1->checksum = ip_csum_fold (sum1);
1755
1756           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1757             {
1758               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1759                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1760               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1761                 snat_det_ses_close(dm1, ses1);
1762
1763               old_port1 = tcp1->dst;
1764               tcp1->dst = new_port1;
1765
1766               sum1 = tcp1->checksum;
1767               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1768                                      ip4_header_t,
1769                                      dst_address /* changed member */);
1770
1771               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1772                                      ip4_header_t /* cheat */,
1773                                      length /* changed member */);
1774               tcp1->checksum = ip_csum_fold(sum1);
1775             }
1776           else
1777             {
1778               old_port1 = udp1->dst_port;
1779               udp1->dst_port = new_port1;
1780               udp1->checksum = 0;
1781             }
1782
1783         trace1:
1784
1785           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1786                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1787             {
1788               snat_out2in_trace_t *t =
1789                  vlib_add_trace (vm, node, b1, sizeof (*t));
1790               t->sw_if_index = sw_if_index1;
1791               t->next_index = next1;
1792               t->session_index = ~0;
1793               if (ses1)
1794                 t->session_index = ses1 - dm1->sessions;
1795             }
1796
1797           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1798
1799           /* verify speculative enqueues, maybe switch current next frame */
1800           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1801                                            to_next, n_left_to_next,
1802                                            bi0, bi1, next0, next1);
1803          }
1804
1805       while (n_left_from > 0 && n_left_to_next > 0)
1806         {
1807           u32 bi0;
1808           vlib_buffer_t * b0;
1809           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1810           u32 sw_if_index0;
1811           ip4_header_t * ip0;
1812           ip_csum_t sum0;
1813           ip4_address_t new_addr0, old_addr0;
1814           u16 new_port0, old_port0;
1815           udp_header_t * udp0;
1816           tcp_header_t * tcp0;
1817           u32 proto0;
1818           snat_det_out_key_t key0;
1819           snat_det_map_t * dm0;
1820           snat_det_session_t * ses0 = 0;
1821           u32 rx_fib_index0;
1822           icmp46_header_t * icmp0;
1823
1824           /* speculatively enqueue b0 to the current next frame */
1825           bi0 = from[0];
1826           to_next[0] = bi0;
1827           from += 1;
1828           to_next += 1;
1829           n_left_from -= 1;
1830           n_left_to_next -= 1;
1831
1832           b0 = vlib_get_buffer (vm, bi0);
1833
1834           ip0 = vlib_buffer_get_current (b0);
1835           udp0 = ip4_next_header (ip0);
1836           tcp0 = (tcp_header_t *) udp0;
1837
1838           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1839
1840           if (PREDICT_FALSE(ip0->ttl == 1))
1841             {
1842               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1843               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1844                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1845                                            0);
1846               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1847               goto trace00;
1848             }
1849
1850           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1851
1852           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1853             {
1854               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1855               icmp0 = (icmp46_header_t *) udp0;
1856
1857               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1858                                   rx_fib_index0, node, next0, thread_index,
1859                                   &ses0, &dm0);
1860               goto trace00;
1861             }
1862
1863           key0.ext_host_addr = ip0->src_address;
1864           key0.ext_host_port = tcp0->src;
1865           key0.out_port = tcp0->dst;
1866
1867           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1868           if (PREDICT_FALSE(!dm0))
1869             {
1870               clib_warning("unknown dst address:  %U",
1871                            format_ip4_address, &ip0->dst_address);
1872               next0 = SNAT_OUT2IN_NEXT_DROP;
1873               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1874               goto trace00;
1875             }
1876
1877           snat_det_reverse(dm0, &ip0->dst_address,
1878                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1879
1880           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1881           if (PREDICT_FALSE(!ses0))
1882             {
1883               clib_warning("no match src %U:%d dst %U:%d for user %U",
1884                            format_ip4_address, &ip0->src_address,
1885                            clib_net_to_host_u16 (tcp0->src),
1886                            format_ip4_address, &ip0->dst_address,
1887                            clib_net_to_host_u16 (tcp0->dst),
1888                            format_ip4_address, &new_addr0);
1889               next0 = SNAT_OUT2IN_NEXT_DROP;
1890               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1891               goto trace00;
1892             }
1893           new_port0 = ses0->in_port;
1894
1895           old_addr0 = ip0->dst_address;
1896           ip0->dst_address = new_addr0;
1897           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1898
1899           sum0 = ip0->checksum;
1900           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1901                                  ip4_header_t,
1902                                  dst_address /* changed member */);
1903           ip0->checksum = ip_csum_fold (sum0);
1904
1905           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1906             {
1907               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1908                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1909               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1910                 snat_det_ses_close(dm0, ses0);
1911
1912               old_port0 = tcp0->dst;
1913               tcp0->dst = new_port0;
1914
1915               sum0 = tcp0->checksum;
1916               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1917                                      ip4_header_t,
1918                                      dst_address /* changed member */);
1919
1920               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1921                                      ip4_header_t /* cheat */,
1922                                      length /* changed member */);
1923               tcp0->checksum = ip_csum_fold(sum0);
1924             }
1925           else
1926             {
1927               old_port0 = udp0->dst_port;
1928               udp0->dst_port = new_port0;
1929               udp0->checksum = 0;
1930             }
1931
1932         trace00:
1933
1934           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1935                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1936             {
1937               snat_out2in_trace_t *t =
1938                  vlib_add_trace (vm, node, b0, sizeof (*t));
1939               t->sw_if_index = sw_if_index0;
1940               t->next_index = next0;
1941               t->session_index = ~0;
1942               if (ses0)
1943                 t->session_index = ses0 - dm0->sessions;
1944             }
1945
1946           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1947
1948           /* verify speculative enqueue, maybe switch current next frame */
1949           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1950                                            to_next, n_left_to_next,
1951                                            bi0, next0);
1952         }
1953
1954       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1955     }
1956
1957   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1958                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1959                                pkts_processed);
1960   return frame->n_vectors;
1961 }
1962
1963 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1964   .function = snat_det_out2in_node_fn,
1965   .name = "nat44-det-out2in",
1966   .vector_size = sizeof (u32),
1967   .format_trace = format_snat_out2in_trace,
1968   .type = VLIB_NODE_TYPE_INTERNAL,
1969
1970   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1971   .error_strings = snat_out2in_error_strings,
1972
1973   .runtime_data_bytes = sizeof (snat_runtime_t),
1974
1975   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1976
1977   /* edit / add dispositions here */
1978   .next_nodes = {
1979     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1980     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1981     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1982   },
1983 };
1984 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1985
1986 /**
1987  * Get address and port values to be used for ICMP packet translation
1988  * and create session if needed
1989  *
1990  * @param[in,out] sm             NAT main
1991  * @param[in,out] node           NAT node runtime
1992  * @param[in] thread_index       thread index
1993  * @param[in,out] b0             buffer containing packet to be translated
1994  * @param[out] p_proto           protocol used for matching
1995  * @param[out] p_value           address and port after NAT translation
1996  * @param[out] p_dont_translate  if packet should not be translated
1997  * @param d                      optional parameter
1998  * @param e                      optional parameter
1999  */
2000 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2001                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2002                           snat_session_key_t *p_value,
2003                           u8 *p_dont_translate, void *d, void *e)
2004 {
2005   ip4_header_t *ip0;
2006   icmp46_header_t *icmp0;
2007   u32 sw_if_index0;
2008   u8 protocol;
2009   snat_det_out_key_t key0;
2010   u8 dont_translate = 0;
2011   u32 next0 = ~0;
2012   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2013   ip4_header_t *inner_ip0;
2014   void *l4_header = 0;
2015   icmp46_header_t *inner_icmp0;
2016   snat_det_map_t * dm0 = 0;
2017   ip4_address_t new_addr0 = {{0}};
2018   snat_det_session_t * ses0 = 0;
2019   ip4_address_t out_addr;
2020
2021   ip0 = vlib_buffer_get_current (b0);
2022   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2023   echo0 = (icmp_echo_header_t *)(icmp0+1);
2024   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2025
2026   if (!icmp_is_error_message (icmp0))
2027     {
2028       protocol = SNAT_PROTOCOL_ICMP;
2029       key0.ext_host_addr = ip0->src_address;
2030       key0.ext_host_port = 0;
2031       key0.out_port = echo0->identifier;
2032       out_addr = ip0->dst_address;
2033     }
2034   else
2035     {
2036       inner_ip0 = (ip4_header_t *)(echo0+1);
2037       l4_header = ip4_next_header (inner_ip0);
2038       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2039       key0.ext_host_addr = inner_ip0->dst_address;
2040       out_addr = inner_ip0->src_address;
2041       switch (protocol)
2042         {
2043         case SNAT_PROTOCOL_ICMP:
2044           inner_icmp0 = (icmp46_header_t*)l4_header;
2045           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2046           key0.ext_host_port = 0;
2047           key0.out_port = inner_echo0->identifier;
2048           break;
2049         case SNAT_PROTOCOL_UDP:
2050         case SNAT_PROTOCOL_TCP:
2051           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2052           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2053           break;
2054         default:
2055           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2056           next0 = SNAT_OUT2IN_NEXT_DROP;
2057           goto out;
2058         }
2059     }
2060
2061   dm0 = snat_det_map_by_out(sm, &out_addr);
2062   if (PREDICT_FALSE(!dm0))
2063     {
2064       /* Don't NAT packet aimed at the intfc address */
2065       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2066                                           ip0->dst_address.as_u32)))
2067         {
2068           dont_translate = 1;
2069           goto out;
2070         }
2071       clib_warning("unknown dst address:  %U",
2072                    format_ip4_address, &ip0->dst_address);
2073       goto out;
2074     }
2075
2076   snat_det_reverse(dm0, &ip0->dst_address,
2077                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2078
2079   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2080   if (PREDICT_FALSE(!ses0))
2081     {
2082       /* Don't NAT packet aimed at the intfc address */
2083       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2084                                           ip0->dst_address.as_u32)))
2085         {
2086           dont_translate = 1;
2087           goto out;
2088         }
2089       clib_warning("no match src %U:%d dst %U:%d for user %U",
2090                    format_ip4_address, &key0.ext_host_addr,
2091                    clib_net_to_host_u16 (key0.ext_host_port),
2092                    format_ip4_address, &out_addr,
2093                    clib_net_to_host_u16 (key0.out_port),
2094                    format_ip4_address, &new_addr0);
2095       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2096       next0 = SNAT_OUT2IN_NEXT_DROP;
2097       goto out;
2098     }
2099
2100   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2101                     !icmp_is_error_message (icmp0)))
2102     {
2103       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2104       next0 = SNAT_OUT2IN_NEXT_DROP;
2105       goto out;
2106     }
2107
2108   goto out;
2109
2110 out:
2111   *p_proto = protocol;
2112   if (ses0)
2113     {
2114       p_value->addr = new_addr0;
2115       p_value->fib_index = sm->inside_fib_index;
2116       p_value->port = ses0->in_port;
2117     }
2118   *p_dont_translate = dont_translate;
2119   if (d)
2120     *(snat_det_session_t**)d = ses0;
2121   if (e)
2122     *(snat_det_map_t**)e = dm0;
2123   return next0;
2124 }
2125
2126 /**********************/
2127 /*** worker handoff ***/
2128 /**********************/
2129 static uword
2130 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2131                                vlib_node_runtime_t * node,
2132                                vlib_frame_t * frame)
2133 {
2134   snat_main_t *sm = &snat_main;
2135   vlib_thread_main_t *tm = vlib_get_thread_main ();
2136   u32 n_left_from, *from, *to_next = 0;
2137   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2138   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2139     = 0;
2140   vlib_frame_queue_elt_t *hf = 0;
2141   vlib_frame_t *f = 0;
2142   int i;
2143   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2144   u32 next_worker_index = 0;
2145   u32 current_worker_index = ~0;
2146   u32 thread_index = vlib_get_thread_index ();
2147
2148   ASSERT (vec_len (sm->workers));
2149
2150   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2151     {
2152       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2153
2154       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2155                                sm->first_worker_index + sm->num_workers - 1,
2156                                (vlib_frame_queue_t *) (~0));
2157     }
2158
2159   from = vlib_frame_vector_args (frame);
2160   n_left_from = frame->n_vectors;
2161
2162   while (n_left_from > 0)
2163     {
2164       u32 bi0;
2165       vlib_buffer_t *b0;
2166       u32 sw_if_index0;
2167       u32 rx_fib_index0;
2168       ip4_header_t * ip0;
2169       u8 do_handoff;
2170
2171       bi0 = from[0];
2172       from += 1;
2173       n_left_from -= 1;
2174
2175       b0 = vlib_get_buffer (vm, bi0);
2176
2177       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2178       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2179
2180       ip0 = vlib_buffer_get_current (b0);
2181
2182       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2183
2184       if (PREDICT_FALSE (next_worker_index != thread_index))
2185         {
2186           do_handoff = 1;
2187
2188           if (next_worker_index != current_worker_index)
2189             {
2190               if (hf)
2191                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2192
2193               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2194                                                       next_worker_index,
2195                                                       handoff_queue_elt_by_worker_index);
2196
2197               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2198               to_next_worker = &hf->buffer_index[hf->n_vectors];
2199               current_worker_index = next_worker_index;
2200             }
2201
2202           /* enqueue to correct worker thread */
2203           to_next_worker[0] = bi0;
2204           to_next_worker++;
2205           n_left_to_next_worker--;
2206
2207           if (n_left_to_next_worker == 0)
2208             {
2209               hf->n_vectors = VLIB_FRAME_SIZE;
2210               vlib_put_frame_queue_elt (hf);
2211               current_worker_index = ~0;
2212               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2213               hf = 0;
2214             }
2215         }
2216       else
2217         {
2218           do_handoff = 0;
2219           /* if this is 1st frame */
2220           if (!f)
2221             {
2222               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2223               to_next = vlib_frame_vector_args (f);
2224             }
2225
2226           to_next[0] = bi0;
2227           to_next += 1;
2228           f->n_vectors++;
2229         }
2230
2231       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2232                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2233         {
2234           snat_out2in_worker_handoff_trace_t *t =
2235             vlib_add_trace (vm, node, b0, sizeof (*t));
2236           t->next_worker_index = next_worker_index;
2237           t->do_handoff = do_handoff;
2238         }
2239     }
2240
2241   if (f)
2242     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2243
2244   if (hf)
2245     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2246
2247   /* Ship frames to the worker nodes */
2248   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2249     {
2250       if (handoff_queue_elt_by_worker_index[i])
2251         {
2252           hf = handoff_queue_elt_by_worker_index[i];
2253           /*
2254            * It works better to let the handoff node
2255            * rate-adapt, always ship the handoff queue element.
2256            */
2257           if (1 || hf->n_vectors == hf->last_n_vectors)
2258             {
2259               vlib_put_frame_queue_elt (hf);
2260               handoff_queue_elt_by_worker_index[i] = 0;
2261             }
2262           else
2263             hf->last_n_vectors = hf->n_vectors;
2264         }
2265       congested_handoff_queue_by_worker_index[i] =
2266         (vlib_frame_queue_t *) (~0);
2267     }
2268   hf = 0;
2269   current_worker_index = ~0;
2270   return frame->n_vectors;
2271 }
2272
2273 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2274   .function = snat_out2in_worker_handoff_fn,
2275   .name = "nat44-out2in-worker-handoff",
2276   .vector_size = sizeof (u32),
2277   .format_trace = format_snat_out2in_worker_handoff_trace,
2278   .type = VLIB_NODE_TYPE_INTERNAL,
2279
2280   .n_next_nodes = 1,
2281
2282   .next_nodes = {
2283     [0] = "error-drop",
2284   },
2285 };
2286
2287 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2288
2289 static uword
2290 snat_out2in_fast_node_fn (vlib_main_t * vm,
2291                           vlib_node_runtime_t * node,
2292                           vlib_frame_t * frame)
2293 {
2294   u32 n_left_from, * from, * to_next;
2295   snat_out2in_next_t next_index;
2296   u32 pkts_processed = 0;
2297   snat_main_t * sm = &snat_main;
2298
2299   from = vlib_frame_vector_args (frame);
2300   n_left_from = frame->n_vectors;
2301   next_index = node->cached_next_index;
2302
2303   while (n_left_from > 0)
2304     {
2305       u32 n_left_to_next;
2306
2307       vlib_get_next_frame (vm, node, next_index,
2308                            to_next, n_left_to_next);
2309
2310       while (n_left_from > 0 && n_left_to_next > 0)
2311         {
2312           u32 bi0;
2313           vlib_buffer_t * b0;
2314           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2315           u32 sw_if_index0;
2316           ip4_header_t * ip0;
2317           ip_csum_t sum0;
2318           u32 new_addr0, old_addr0;
2319           u16 new_port0, old_port0;
2320           udp_header_t * udp0;
2321           tcp_header_t * tcp0;
2322           icmp46_header_t * icmp0;
2323           snat_session_key_t key0, sm0;
2324           u32 proto0;
2325           u32 rx_fib_index0;
2326
2327           /* speculatively enqueue b0 to the current next frame */
2328           bi0 = from[0];
2329           to_next[0] = bi0;
2330           from += 1;
2331           to_next += 1;
2332           n_left_from -= 1;
2333           n_left_to_next -= 1;
2334
2335           b0 = vlib_get_buffer (vm, bi0);
2336
2337           ip0 = vlib_buffer_get_current (b0);
2338           udp0 = ip4_next_header (ip0);
2339           tcp0 = (tcp_header_t *) udp0;
2340           icmp0 = (icmp46_header_t *) udp0;
2341
2342           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2343           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2344
2345           vnet_feature_next (sw_if_index0, &next0, b0);
2346
2347           if (PREDICT_FALSE(ip0->ttl == 1))
2348             {
2349               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2350               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2351                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2352                                            0);
2353               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2354               goto trace00;
2355             }
2356
2357           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2358
2359           if (PREDICT_FALSE (proto0 == ~0))
2360               goto trace00;
2361
2362           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2363             {
2364               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2365                                   rx_fib_index0, node, next0, ~0, 0, 0);
2366               goto trace00;
2367             }
2368
2369           key0.addr = ip0->dst_address;
2370           key0.port = udp0->dst_port;
2371           key0.fib_index = rx_fib_index0;
2372
2373           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2374             {
2375               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2376               goto trace00;
2377             }
2378
2379           new_addr0 = sm0.addr.as_u32;
2380           new_port0 = sm0.port;
2381           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2382           old_addr0 = ip0->dst_address.as_u32;
2383           ip0->dst_address.as_u32 = new_addr0;
2384
2385           sum0 = ip0->checksum;
2386           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2387                                  ip4_header_t,
2388                                  dst_address /* changed member */);
2389           ip0->checksum = ip_csum_fold (sum0);
2390
2391           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2392             {
2393                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2394                 {
2395                   old_port0 = tcp0->dst_port;
2396                   tcp0->dst_port = new_port0;
2397
2398                   sum0 = tcp0->checksum;
2399                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2400                                          ip4_header_t,
2401                                          dst_address /* changed member */);
2402
2403                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2404                                          ip4_header_t /* cheat */,
2405                                          length /* changed member */);
2406                   tcp0->checksum = ip_csum_fold(sum0);
2407                 }
2408               else
2409                 {
2410                   old_port0 = udp0->dst_port;
2411                   udp0->dst_port = new_port0;
2412                   udp0->checksum = 0;
2413                 }
2414             }
2415           else
2416             {
2417               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2418                 {
2419                   sum0 = tcp0->checksum;
2420                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2421                                          ip4_header_t,
2422                                          dst_address /* changed member */);
2423
2424                   tcp0->checksum = ip_csum_fold(sum0);
2425                 }
2426             }
2427
2428         trace00:
2429
2430           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2431                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2432             {
2433               snat_out2in_trace_t *t =
2434                  vlib_add_trace (vm, node, b0, sizeof (*t));
2435               t->sw_if_index = sw_if_index0;
2436               t->next_index = next0;
2437             }
2438
2439           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2440
2441           /* verify speculative enqueue, maybe switch current next frame */
2442           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2443                                            to_next, n_left_to_next,
2444                                            bi0, next0);
2445         }
2446
2447       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2448     }
2449
2450   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2451                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2452                                pkts_processed);
2453   return frame->n_vectors;
2454 }
2455
2456 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2457   .function = snat_out2in_fast_node_fn,
2458   .name = "nat44-out2in-fast",
2459   .vector_size = sizeof (u32),
2460   .format_trace = format_snat_out2in_fast_trace,
2461   .type = VLIB_NODE_TYPE_INTERNAL,
2462
2463   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2464   .error_strings = snat_out2in_error_strings,
2465
2466   .runtime_data_bytes = sizeof (snat_runtime_t),
2467
2468   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2469
2470   /* edit / add dispositions here */
2471   .next_nodes = {
2472     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2473     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2474     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2475   },
2476 };
2477 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);