src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/udp/udp.h>
  23 #include <vnet/ethernet/ethernet.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <nat/nat.h>
  26 #include <nat/nat_ipfix_logging.h>
  27 #include <nat/nat_det.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37 } snat_out2in_trace_t;
  38
  39 typedef struct {
  40   u32 next_worker_index;
  41   u8 do_handoff;
  42 } snat_out2in_worker_handoff_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
  46 {
  47   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  48   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  49   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  50
  51   s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  52               t->sw_if_index, t->next_index, t->session_index);
  53   return s;
  54 }
  55
  56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
  57 {
  58   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  59   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  60   snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
  61
  62   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  63               t->sw_if_index, t->next_index);
  64   return s;
  65 }
  66
  67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
  68 {
  69   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  70   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  71   snat_out2in_worker_handoff_trace_t * t =
  72     va_arg (*args, snat_out2in_worker_handoff_trace_t *);
  73   char * m;
  74
  75   m = t->do_handoff ? "next worker" : "same worker";
  76   s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  77
  78   return s;
  79 }
  80
  81 vlib_node_registration_t snat_out2in_node;
  82 vlib_node_registration_t snat_out2in_fast_node;
  83 vlib_node_registration_t snat_out2in_worker_handoff_node;
  84 vlib_node_registration_t snat_det_out2in_node;
  85
  86 #define foreach_snat_out2in_error                       \
  87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  88 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  89 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  90 _(NO_TRANSLATION, "No translation")                     \
  91 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")
  92
  93 typedef enum {
  94 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  95   foreach_snat_out2in_error
  96 #undef _
  97   SNAT_OUT2IN_N_ERROR,
  98 } snat_out2in_error_t;
  99
 100 static char * snat_out2in_error_strings[] = {
 101 #define _(sym,string) string,
 102   foreach_snat_out2in_error
 103 #undef _
 104 };
 105
 106 typedef enum {
 107   SNAT_OUT2IN_NEXT_DROP,
 108   SNAT_OUT2IN_NEXT_LOOKUP,
 109   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 110   SNAT_OUT2IN_N_NEXT,
 111 } snat_out2in_next_t;
 112
 113 /**
 114  * @brief Create session for static mapping.
 115  *
 116  * Create NAT session initiated by host from external network with static
 117  * mapping.
 118  *
 119  * @param sm     NAT main.
 120  * @param b0     Vlib buffer.
 121  * @param in2out In2out NAT44 session key.
 122  * @param out2in Out2in NAT44 session key.
 123  * @param node   Vlib node.
 124  *
 125  * @returns SNAT session if successfully created otherwise 0.
 126  */
 127 static inline snat_session_t *
 128 create_session_for_static_mapping (snat_main_t *sm,
 129                                    vlib_buffer_t *b0,
 130                                    snat_session_key_t in2out,
 131                                    snat_session_key_t out2in,
 132                                    vlib_node_runtime_t * node,
 133                                    u32 thread_index)
 134 {
 135   snat_user_t *u;
 136   snat_user_key_t user_key;
 137   snat_session_t *s;
 138   clib_bihash_kv_8_8_t kv0, value0;
 139   dlist_elt_t * per_user_translation_list_elt;
 140   dlist_elt_t * per_user_list_head_elt;
 141   ip4_header_t *ip0;
 142
 143   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 144     {
 145       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 146       return 0;
 147     }
 148
 149   ip0 = vlib_buffer_get_current (b0);
 150
 151   user_key.addr = in2out.addr;
 152   user_key.fib_index = in2out.fib_index;
 153   kv0.key = user_key.as_u64;
 154
 155   /* Ever heard of the "user" = inside ip4 address before? */
 156   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
 157                               &kv0, &value0))
 158     {
 159       /* no, make a new one */
 160       pool_get (sm->per_thread_data[thread_index].users, u);
 161       memset (u, 0, sizeof (*u));
 162       u->addr = in2out.addr;
 163       u->fib_index = in2out.fib_index;
 164
 165       pool_get (sm->per_thread_data[thread_index].list_pool,
 166                 per_user_list_head_elt);
 167
 168       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 169         sm->per_thread_data[thread_index].list_pool;
 170
 171       clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 172                        u->sessions_per_user_list_head_index);
 173
 174       kv0.value = u - sm->per_thread_data[thread_index].users;
 175
 176       /* add user */
 177       clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
 178                                &kv0, 1 /* is_add */);
 179     }
 180   else
 181     {
 182       u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
 183                              value0.value);
 184     }
 185
 186   pool_get (sm->per_thread_data[thread_index].sessions, s);
 187   memset (s, 0, sizeof (*s));
 188
 189   s->outside_address_index = ~0;
 190   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 191   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 192   u->nstaticsessions++;
 193
 194   /* Create list elts */
 195   pool_get (sm->per_thread_data[thread_index].list_pool,
 196             per_user_translation_list_elt);
 197   clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
 198                    per_user_translation_list_elt -
 199                    sm->per_thread_data[thread_index].list_pool);
 200
 201   per_user_translation_list_elt->value =
 202     s - sm->per_thread_data[thread_index].sessions;
 203   s->per_user_index =
 204     per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
 205   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 206
 207   clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 208                       s->per_user_list_head_index,
 209                       per_user_translation_list_elt -
 210                       sm->per_thread_data[thread_index].list_pool);
 211
 212   s->in2out = in2out;
 213   s->out2in = out2in;
 214   s->in2out.protocol = out2in.protocol;
 215
 216   /* Add to translation hashes */
 217   kv0.key = s->in2out.as_u64;
 218   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 219   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 220                                1 /* is_add */))
 221       clib_warning ("in2out key add failed");
 222
 223   kv0.key = s->out2in.as_u64;
 224   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 225
 226   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 227                                1 /* is_add */))
 228       clib_warning ("out2in key add failed");
 229
 230   /* log NAT event */
 231   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 232                                       s->out2in.addr.as_u32,
 233                                       s->in2out.protocol,
 234                                       s->in2out.port,
 235                                       s->out2in.port,
 236                                       s->in2out.fib_index);
 237    return s;
 238 }
 239
 240 static_always_inline
 241 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
 242                                  snat_session_key_t *p_key0)
 243 {
 244   icmp46_header_t *icmp0;
 245   snat_session_key_t key0;
 246   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 247   ip4_header_t *inner_ip0;
 248   void *l4_header = 0;
 249   icmp46_header_t *inner_icmp0;
 250
 251   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 252   echo0 = (icmp_echo_header_t *)(icmp0+1);
 253
 254   if (!icmp_is_error_message (icmp0))
 255     {
 256       key0.protocol = SNAT_PROTOCOL_ICMP;
 257       key0.addr = ip0->dst_address;
 258       key0.port = echo0->identifier;
 259     }
 260   else
 261     {
 262       inner_ip0 = (ip4_header_t *)(echo0+1);
 263       l4_header = ip4_next_header (inner_ip0);
 264       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 265       key0.addr = inner_ip0->src_address;
 266       switch (key0.protocol)
 267         {
 268         case SNAT_PROTOCOL_ICMP:
 269           inner_icmp0 = (icmp46_header_t*)l4_header;
 270           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 271           key0.port = inner_echo0->identifier;
 272           break;
 273         case SNAT_PROTOCOL_UDP:
 274         case SNAT_PROTOCOL_TCP:
 275           key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
 276           break;
 277         default:
 278           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 279         }
 280     }
 281   *p_key0 = key0;
 282   return -1; /* success */
 283 }
 284
 285 /**
 286  * Get address and port values to be used for ICMP packet translation
 287  * and create session if needed
 288  *
 289  * @param[in,out] sm             NAT main
 290  * @param[in,out] node           NAT node runtime
 291  * @param[in] thread_index       thread index
 292  * @param[in,out] b0             buffer containing packet to be translated
 293  * @param[out] p_proto           protocol used for matching
 294  * @param[out] p_value           address and port after NAT translation
 295  * @param[out] p_dont_translate  if packet should not be translated
 296  * @param d                      optional parameter
 297  * @param e                      optional parameter
 298  */
 299 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 300                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 301                            snat_session_key_t *p_value,
 302                            u8 *p_dont_translate, void *d, void *e)
 303 {
 304   ip4_header_t *ip0;
 305   icmp46_header_t *icmp0;
 306   u32 sw_if_index0;
 307   u32 rx_fib_index0;
 308   snat_session_key_t key0;
 309   snat_session_key_t sm0;
 310   snat_session_t *s0 = 0;
 311   u8 dont_translate = 0;
 312   clib_bihash_kv_8_8_t kv0, value0;
 313   u8 is_addr_only;
 314   u32 next0 = ~0;
 315   int err;
 316
 317   ip0 = vlib_buffer_get_current (b0);
 318   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 319   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 320   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 321
 322   key0.protocol = 0;
 323
 324   err = icmp_get_key (ip0, &key0);
 325   if (err != -1)
 326     {
 327       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 328       next0 = SNAT_OUT2IN_NEXT_DROP;
 329       goto out;
 330     }
 331   key0.fib_index = rx_fib_index0;
 332
 333   kv0.key = key0.as_u64;
 334
 335   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 336                               &value0))
 337     {
 338       /* Try to match static mapping by external address and port,
 339          destination address and port in packet */
 340       if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 341         {
 342           /* Don't NAT packet aimed at the intfc address */
 343           if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 344                                               ip0->dst_address.as_u32)))
 345             {
 346               dont_translate = 1;
 347               goto out;
 348             }
 349           b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 350           next0 = SNAT_OUT2IN_NEXT_DROP;
 351           goto out;
 352         }
 353
 354       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 355                         (icmp0->type != ICMP4_echo_request || !is_addr_only)))
 356         {
 357           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 358           next0 = SNAT_OUT2IN_NEXT_DROP;
 359           goto out;
 360         }
 361
 362       /* Create session initiated by host from external network */
 363       s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
 364                                              node, thread_index);
 365
 366       if (!s0)
 367         {
 368           next0 = SNAT_OUT2IN_NEXT_DROP;
 369           goto out;
 370         }
 371     }
 372   else
 373     {
 374       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 375                         icmp0->type != ICMP4_echo_request &&
 376                         !icmp_is_error_message (icmp0)))
 377         {
 378           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 379           next0 = SNAT_OUT2IN_NEXT_DROP;
 380           goto out;
 381         }
 382
 383       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 384                               value0.value);
 385     }
 386
 387 out:
 388   *p_proto = key0.protocol;
 389   if (s0)
 390     *p_value = s0->in2out;
 391   *p_dont_translate = dont_translate;
 392   if (d)
 393     *(snat_session_t**)d = s0;
 394   return next0;
 395 }
 396
 397 /**
 398  * Get address and port values to be used for ICMP packet translation
 399  *
 400  * @param[in] sm                 NAT main
 401  * @param[in,out] node           NAT node runtime
 402  * @param[in] thread_index       thread index
 403  * @param[in,out] b0             buffer containing packet to be translated
 404  * @param[out] p_proto           protocol used for matching
 405  * @param[out] p_value           address and port after NAT translation
 406  * @param[out] p_dont_translate  if packet should not be translated
 407  * @param d                      optional parameter
 408  * @param e                      optional parameter
 409  */
 410 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 411                            u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
 412                            snat_session_key_t *p_value,
 413                            u8 *p_dont_translate, void *d, void *e)
 414 {
 415   ip4_header_t *ip0;
 416   icmp46_header_t *icmp0;
 417   u32 sw_if_index0;
 418   u32 rx_fib_index0;
 419   snat_session_key_t key0;
 420   snat_session_key_t sm0;
 421   u8 dont_translate = 0;
 422   u8 is_addr_only;
 423   u32 next0 = ~0;
 424   int err;
 425
 426   ip0 = vlib_buffer_get_current (b0);
 427   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 428   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 429   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 430
 431   err = icmp_get_key (ip0, &key0);
 432   if (err != -1)
 433     {
 434       b0->error = node->errors[err];
 435       next0 = SNAT_OUT2IN_NEXT_DROP;
 436       goto out2;
 437     }
 438   key0.fib_index = rx_fib_index0;
 439
 440   if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
 441     {
 442       /* Don't NAT packet aimed at the intfc address */
 443       if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
 444         {
 445           dont_translate = 1;
 446           goto out;
 447         }
 448       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 449       next0 = SNAT_OUT2IN_NEXT_DROP;
 450       goto out;
 451     }
 452
 453   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
 454                     (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 455                     !icmp_is_error_message (icmp0)))
 456     {
 457       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 458       next0 = SNAT_OUT2IN_NEXT_DROP;
 459       goto out;
 460     }
 461
 462 out:
 463   *p_value = sm0;
 464 out2:
 465   *p_proto = key0.protocol;
 466   *p_dont_translate = dont_translate;
 467   return next0;
 468 }
 469
 470 static inline u32 icmp_out2in (snat_main_t *sm,
 471                                vlib_buffer_t * b0,
 472                                ip4_header_t * ip0,
 473                                icmp46_header_t * icmp0,
 474                                u32 sw_if_index0,
 475                                u32 rx_fib_index0,
 476                                vlib_node_runtime_t * node,
 477                                u32 next0,
 478                                u32 thread_index,
 479                                void *d,
 480                                void *e)
 481 {
 482   snat_session_key_t sm0;
 483   u8 protocol;
 484   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 485   ip4_header_t *inner_ip0 = 0;
 486   void *l4_header = 0;
 487   icmp46_header_t *inner_icmp0;
 488   u8 dont_translate;
 489   u32 new_addr0, old_addr0;
 490   u16 old_id0, new_id0;
 491   ip_csum_t sum0;
 492   u16 checksum0;
 493   u32 next0_tmp;
 494
 495   echo0 = (icmp_echo_header_t *)(icmp0+1);
 496
 497   next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
 498                                        &protocol, &sm0, &dont_translate, d, e);
 499   if (next0_tmp != ~0)
 500     next0 = next0_tmp;
 501   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 502     goto out;
 503
 504   sum0 = ip_incremental_checksum (0, icmp0,
 505                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 506   checksum0 = ~ip_csum_fold (sum0);
 507   if (checksum0 != 0 && checksum0 != 0xffff)
 508     {
 509       next0 = SNAT_OUT2IN_NEXT_DROP;
 510       goto out;
 511     }
 512
 513   old_addr0 = ip0->dst_address.as_u32;
 514   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 515   vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 516
 517   sum0 = ip0->checksum;
 518   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 519                          dst_address /* changed member */);
 520   ip0->checksum = ip_csum_fold (sum0);
 521
 522   if (!icmp_is_error_message (icmp0))
 523     {
 524       new_id0 = sm0.port;
 525       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 526         {
 527           old_id0 = echo0->identifier;
 528           new_id0 = sm0.port;
 529           echo0->identifier = new_id0;
 530
 531           sum0 = icmp0->checksum;
 532           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 533                                  identifier /* changed member */);
 534           icmp0->checksum = ip_csum_fold (sum0);
 535         }
 536     }
 537   else
 538     {
 539       inner_ip0 = (ip4_header_t *)(echo0+1);
 540       l4_header = ip4_next_header (inner_ip0);
 541
 542       if (!ip4_header_checksum_is_valid (inner_ip0))
 543         {
 544           next0 = SNAT_OUT2IN_NEXT_DROP;
 545           goto out;
 546         }
 547
 548       old_addr0 = inner_ip0->src_address.as_u32;
 549       inner_ip0->src_address = sm0.addr;
 550       new_addr0 = inner_ip0->src_address.as_u32;
 551
 552       sum0 = icmp0->checksum;
 553       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 554                              src_address /* changed member */);
 555       icmp0->checksum = ip_csum_fold (sum0);
 556
 557       switch (protocol)
 558         {
 559         case SNAT_PROTOCOL_ICMP:
 560           inner_icmp0 = (icmp46_header_t*)l4_header;
 561           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 562
 563           old_id0 = inner_echo0->identifier;
 564           new_id0 = sm0.port;
 565           inner_echo0->identifier = new_id0;
 566
 567           sum0 = icmp0->checksum;
 568           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 569                                  identifier);
 570           icmp0->checksum = ip_csum_fold (sum0);
 571           break;
 572         case SNAT_PROTOCOL_UDP:
 573         case SNAT_PROTOCOL_TCP:
 574           old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
 575           new_id0 = sm0.port;
 576           ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
 577
 578           sum0 = icmp0->checksum;
 579           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 580                                  src_port);
 581           icmp0->checksum = ip_csum_fold (sum0);
 582           break;
 583         default:
 584           ASSERT(0);
 585         }
 586     }
 587
 588 out:
 589   return next0;
 590 }
 591
 592
 593 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
 594                                          vlib_buffer_t * b0,
 595                                          ip4_header_t * ip0,
 596                                          icmp46_header_t * icmp0,
 597                                          u32 sw_if_index0,
 598                                          u32 rx_fib_index0,
 599                                          vlib_node_runtime_t * node,
 600                                          u32 next0, f64 now,
 601                                          u32 thread_index,
 602                                          snat_session_t ** p_s0)
 603 {
 604   next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 605                       next0, thread_index, p_s0, 0);
 606   snat_session_t * s0 = *p_s0;
 607   if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 608     {
 609       /* Accounting */
 610       s0->last_heard = now;
 611       s0->total_pkts++;
 612       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
 613       /* Per-user LRU list maintenance for dynamic translation */
 614       if (!snat_is_session_static (s0))
 615         {
 616           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
 617                              s0->per_user_index);
 618           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
 619                               s0->per_user_list_head_index,
 620                               s0->per_user_index);
 621         }
 622     }
 623   return next0;
 624 }
 625
 626 static snat_session_t *
 627 snat_out2in_unknown_proto (snat_main_t *sm,
 628                            vlib_buffer_t * b,
 629                            ip4_header_t * ip,
 630                            u32 rx_fib_index,
 631                            u32 thread_index,
 632                            f64 now,
 633                            vlib_main_t * vm,
 634                            vlib_node_runtime_t * node)
 635 {
 636   clib_bihash_kv_8_8_t kv, value;
 637   clib_bihash_kv_16_8_t s_kv, s_value;
 638   snat_static_mapping_t *m;
 639   snat_session_key_t m_key;
 640   u32 old_addr, new_addr;
 641   ip_csum_t sum;
 642   nat_ed_ses_key_t key;
 643   snat_session_t * s;
 644   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 645   snat_user_key_t u_key;
 646   snat_user_t *u;
 647   dlist_elt_t *head, *elt;
 648
 649   old_addr = ip->dst_address.as_u32;
 650
 651   key.l_addr = ip->dst_address;
 652   key.r_addr = ip->src_address;
 653   key.fib_index = rx_fib_index;
 654   key.proto = ip->protocol;
 655   key.rsvd = 0;
 656   key.l_port = 0;
 657   s_kv.key[0] = key.as_u64[0];
 658   s_kv.key[1] = key.as_u64[1];
 659
 660   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 661     {
 662       s = pool_elt_at_index (tsm->sessions, s_value.value);
 663       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 664     }
 665   else
 666     {
 667       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 668         {
 669           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 670           return 0;
 671         }
 672
 673       m_key.addr = ip->dst_address;
 674       m_key.port = 0;
 675       m_key.protocol = 0;
 676       m_key.fib_index = rx_fib_index;
 677       kv.key = m_key.as_u64;
 678       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 679         {
 680           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 681           return 0;
 682         }
 683
 684       m = pool_elt_at_index (sm->static_mappings, value.value);
 685
 686       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 687
 688       u_key.addr = ip->src_address;
 689       u_key.fib_index = m->fib_index;
 690       kv.key = u_key.as_u64;
 691
 692       /* Ever heard of the "user" = src ip4 address before? */
 693       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 694         {
 695           /* no, make a new one */
 696           pool_get (tsm->users, u);
 697           memset (u, 0, sizeof (*u));
 698           u->addr = ip->src_address;
 699           u->fib_index = rx_fib_index;
 700
 701           pool_get (tsm->list_pool, head);
 702           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 703
 704           clib_dlist_init (tsm->list_pool,
 705                            u->sessions_per_user_list_head_index);
 706
 707           kv.value = u - tsm->users;
 708
 709           /* add user */
 710           clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
 711         }
 712       else
 713         {
 714           u = pool_elt_at_index (tsm->users, value.value);
 715         }
 716
 717       /* Create a new session */
 718       pool_get (tsm->sessions, s);
 719       memset (s, 0, sizeof (*s));
 720
 721       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 722       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 723       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 724       s->outside_address_index = ~0;
 725       s->out2in.addr.as_u32 = old_addr;
 726       s->out2in.fib_index = rx_fib_index;
 727       s->in2out.addr.as_u32 = new_addr;
 728       s->in2out.fib_index = m->fib_index;
 729       s->in2out.port = s->out2in.port = ip->protocol;
 730       u->nstaticsessions++;
 731
 732       /* Create list elts */
 733       pool_get (tsm->list_pool, elt);
 734       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 735       elt->value = s - tsm->sessions;
 736       s->per_user_index = elt - tsm->list_pool;
 737       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 738       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 739                           s->per_user_index);
 740
 741       /* Add to lookup tables */
 742       s_kv.value = s - tsm->sessions;
 743       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 744         clib_warning ("out2in key add failed");
 745
 746       key.l_addr = ip->dst_address;
 747       key.fib_index = m->fib_index;
 748       s_kv.key[0] = key.as_u64[0];
 749       s_kv.key[1] = key.as_u64[1];
 750       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 751         clib_warning ("in2out key add failed");
 752    }
 753
 754   /* Update IP checksum */
 755   sum = ip->checksum;
 756   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 757   ip->checksum = ip_csum_fold (sum);
 758
 759   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 760
 761   /* Accounting */
 762   s->last_heard = now;
 763   s->total_pkts++;
 764   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 765   /* Per-user LRU list maintenance */
 766   clib_dlist_remove (tsm->list_pool, s->per_user_index);
 767   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 768                       s->per_user_index);
 769
 770   return s;
 771 }
 772
 773 static snat_session_t *
 774 snat_out2in_lb (snat_main_t *sm,
 775                 vlib_buffer_t * b,
 776                 ip4_header_t * ip,
 777                 u32 rx_fib_index,
 778                 u32 thread_index,
 779                 f64 now,
 780                 vlib_main_t * vm,
 781                 vlib_node_runtime_t * node)
 782 {
 783   nat_ed_ses_key_t key;
 784   clib_bihash_kv_16_8_t s_kv, s_value;
 785   udp_header_t *udp = ip4_next_header (ip);
 786   tcp_header_t *tcp = (tcp_header_t *) udp;
 787   snat_session_t *s = 0;
 788   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 789   snat_session_key_t e_key, l_key;
 790   clib_bihash_kv_8_8_t kv, value;
 791   u32 old_addr, new_addr;
 792   u32 proto = ip_proto_to_snat_proto (ip->protocol);
 793   u16 new_port, old_port;
 794   ip_csum_t sum;
 795   snat_user_key_t u_key;
 796   snat_user_t *u;
 797   dlist_elt_t *head, *elt;
 798
 799   old_addr = ip->dst_address.as_u32;
 800
 801   key.l_addr = ip->dst_address;
 802   key.r_addr = ip->src_address;
 803   key.fib_index = rx_fib_index;
 804   key.proto = ip->protocol;
 805   key.rsvd = 0;
 806   key.l_port = udp->dst_port;
 807   s_kv.key[0] = key.as_u64[0];
 808   s_kv.key[1] = key.as_u64[1];
 809
 810   if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 811     {
 812       s = pool_elt_at_index (tsm->sessions, s_value.value);
 813     }
 814   else
 815     {
 816       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 817         {
 818           b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 819           return 0;
 820         }
 821
 822       e_key.addr = ip->dst_address;
 823       e_key.port = udp->dst_port;
 824       e_key.protocol = proto;
 825       e_key.fib_index = rx_fib_index;
 826       if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
 827         return 0;
 828
 829       u_key.addr = l_key.addr;
 830       u_key.fib_index = l_key.fib_index;
 831       kv.key = u_key.as_u64;
 832
 833       /* Ever heard of the "user" = src ip4 address before? */
 834       if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 835         {
 836           /* no, make a new one */
 837           pool_get (tsm->users, u);
 838           memset (u, 0, sizeof (*u));
 839           u->addr = l_key.addr;
 840           u->fib_index = l_key.fib_index;
 841
 842           pool_get (tsm->list_pool, head);
 843           u->sessions_per_user_list_head_index = head - tsm->list_pool;
 844
 845           clib_dlist_init (tsm->list_pool,
 846                            u->sessions_per_user_list_head_index);
 847
 848           kv.value = u - tsm->users;
 849
 850           /* add user */
 851           if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 852             clib_warning ("user key add failed");
 853         }
 854       else
 855         {
 856           u = pool_elt_at_index (tsm->users, value.value);
 857         }
 858
 859       /* Create a new session */
 860       pool_get (tsm->sessions, s);
 861       memset (s, 0, sizeof (*s));
 862
 863       s->ext_host_addr.as_u32 = ip->src_address.as_u32;
 864       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 865       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 866       s->outside_address_index = ~0;
 867       s->out2in = e_key;
 868       s->in2out = l_key;
 869       u->nstaticsessions++;
 870
 871       /* Create list elts */
 872       pool_get (tsm->list_pool, elt);
 873       clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
 874       elt->value = s - tsm->sessions;
 875       s->per_user_index = elt - tsm->list_pool;
 876       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 877       clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 878                           s->per_user_index);
 879
 880       /* Add to lookup tables */
 881       s_kv.value = s - tsm->sessions;
 882       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
 883         clib_warning ("out2in-ed key add failed");
 884
 885       key.l_addr = l_key.addr;
 886       key.fib_index = l_key.fib_index;
 887       key.l_port = l_key.port;
 888       s_kv.key[0] = key.as_u64[0];
 889       s_kv.key[1] = key.as_u64[1];
 890       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
 891         clib_warning ("in2out-ed key add failed");
 892     }
 893
 894   new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 895
 896   /* Update IP checksum */
 897   sum = ip->checksum;
 898   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 899   ip->checksum = ip_csum_fold (sum);
 900
 901   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
 902     {
 903       old_port = tcp->dst_port;
 904       tcp->dst_port = s->in2out.port;
 905       new_port = tcp->dst_port;
 906
 907       sum = tcp->checksum;
 908       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 909       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
 910       tcp->checksum = ip_csum_fold(sum);
 911     }
 912   else
 913     {
 914       udp->dst_port = s->in2out.port;
 915       udp->checksum = 0;
 916     }
 917
 918   vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 919
 920   /* Accounting */
 921   s->last_heard = now;
 922   s->total_pkts++;
 923   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
 924   return s;
 925 }
 926
 927 static uword
 928 snat_out2in_node_fn (vlib_main_t * vm,
 929                   vlib_node_runtime_t * node,
 930                   vlib_frame_t * frame)
 931 {
 932   u32 n_left_from, * from, * to_next;
 933   snat_out2in_next_t next_index;
 934   u32 pkts_processed = 0;
 935   snat_main_t * sm = &snat_main;
 936   f64 now = vlib_time_now (vm);
 937   u32 thread_index = vlib_get_thread_index ();
 938
 939   from = vlib_frame_vector_args (frame);
 940   n_left_from = frame->n_vectors;
 941   next_index = node->cached_next_index;
 942
 943   while (n_left_from > 0)
 944     {
 945       u32 n_left_to_next;
 946
 947       vlib_get_next_frame (vm, node, next_index,
 948                            to_next, n_left_to_next);
 949
 950       while (n_left_from >= 4 && n_left_to_next >= 2)
 951         {
 952           u32 bi0, bi1;
 953           vlib_buffer_t * b0, * b1;
 954           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 955           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 956           u32 sw_if_index0, sw_if_index1;
 957           ip4_header_t * ip0, *ip1;
 958           ip_csum_t sum0, sum1;
 959           u32 new_addr0, old_addr0;
 960           u16 new_port0, old_port0;
 961           u32 new_addr1, old_addr1;
 962           u16 new_port1, old_port1;
 963           udp_header_t * udp0, * udp1;
 964           tcp_header_t * tcp0, * tcp1;
 965           icmp46_header_t * icmp0, * icmp1;
 966           snat_session_key_t key0, key1, sm0, sm1;
 967           u32 rx_fib_index0, rx_fib_index1;
 968           u32 proto0, proto1;
 969           snat_session_t * s0 = 0, * s1 = 0;
 970           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 971
 972           /* Prefetch next iteration. */
 973           {
 974             vlib_buffer_t * p2, * p3;
 975
 976             p2 = vlib_get_buffer (vm, from[2]);
 977             p3 = vlib_get_buffer (vm, from[3]);
 978
 979             vlib_prefetch_buffer_header (p2, LOAD);
 980             vlib_prefetch_buffer_header (p3, LOAD);
 981
 982             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 983             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 984           }
 985
 986           /* speculatively enqueue b0 and b1 to the current next frame */
 987           to_next[0] = bi0 = from[0];
 988           to_next[1] = bi1 = from[1];
 989           from += 2;
 990           to_next += 2;
 991           n_left_from -= 2;
 992           n_left_to_next -= 2;
 993
 994           b0 = vlib_get_buffer (vm, bi0);
 995           b1 = vlib_get_buffer (vm, bi1);
 996
 997           vnet_buffer (b0)->snat.flags = 0;
 998           vnet_buffer (b1)->snat.flags = 0;
 999
1000           ip0 = vlib_buffer_get_current (b0);
1001           udp0 = ip4_next_header (ip0);
1002           tcp0 = (tcp_header_t *) udp0;
1003           icmp0 = (icmp46_header_t *) udp0;
1004
1005           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1006           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1007                                    sw_if_index0);
1008
1009           if (PREDICT_FALSE(ip0->ttl == 1))
1010             {
1011               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1012               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1013                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1014                                            0);
1015               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1016               goto trace0;
1017             }
1018
1019           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1020
1021           if (PREDICT_FALSE (proto0 == ~0))
1022             {
1023               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1024                                              thread_index, now, vm, node);
1025               if (!s0)
1026                 next0 = SNAT_OUT2IN_NEXT_DROP;
1027               goto trace0;
1028             }
1029
1030           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1031             {
1032               next0 = icmp_out2in_slow_path
1033                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1034                  next0, now, thread_index, &s0);
1035               goto trace0;
1036             }
1037
1038           key0.addr = ip0->dst_address;
1039           key0.port = udp0->dst_port;
1040           key0.protocol = proto0;
1041           key0.fib_index = rx_fib_index0;
1042
1043           kv0.key = key0.as_u64;
1044
1045           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1046                                       &kv0, &value0))
1047             {
1048               /* Try to match static mapping by external address and port,
1049                  destination address and port in packet */
1050               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1051                 {
1052                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1053                   /*
1054                    * Send DHCP packets to the ipv4 stack, or we won't
1055                    * be able to use dhcp client on the outside interface
1056                    */
1057                   if (proto0 != SNAT_PROTOCOL_UDP
1058                       || (udp0->dst_port
1059                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1060                     next0 = SNAT_OUT2IN_NEXT_DROP;
1061                   goto trace0;
1062                 }
1063
1064               /* Create session initiated by host from external network */
1065               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1066                                                      thread_index);
1067               if (!s0)
1068                 {
1069                   next0 = SNAT_OUT2IN_NEXT_DROP;
1070                   goto trace0;
1071                 }
1072             }
1073           else
1074             {
1075               if (PREDICT_FALSE (value0.value == ~0ULL))
1076                 {
1077                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1078                                       now, vm, node);
1079                   if (!s0)
1080                     next0 = SNAT_OUT2IN_NEXT_DROP;
1081                   goto trace0;
1082                 }
1083               else
1084                 {
1085                   s0 = pool_elt_at_index (
1086                     sm->per_thread_data[thread_index].sessions,
1087                     value0.value);
1088                 }
1089             }
1090
1091           old_addr0 = ip0->dst_address.as_u32;
1092           ip0->dst_address = s0->in2out.addr;
1093           new_addr0 = ip0->dst_address.as_u32;
1094           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1095
1096           sum0 = ip0->checksum;
1097           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1098                                  ip4_header_t,
1099                                  dst_address /* changed member */);
1100           ip0->checksum = ip_csum_fold (sum0);
1101
1102           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1103             {
1104               old_port0 = tcp0->dst_port;
1105               tcp0->dst_port = s0->in2out.port;
1106               new_port0 = tcp0->dst_port;
1107
1108               sum0 = tcp0->checksum;
1109               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1110                                      ip4_header_t,
1111                                      dst_address /* changed member */);
1112
1113               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1114                                      ip4_header_t /* cheat */,
1115                                      length /* changed member */);
1116               tcp0->checksum = ip_csum_fold(sum0);
1117             }
1118           else
1119             {
1120               old_port0 = udp0->dst_port;
1121               udp0->dst_port = s0->in2out.port;
1122               udp0->checksum = 0;
1123             }
1124
1125           /* Accounting */
1126           s0->last_heard = now;
1127           s0->total_pkts++;
1128           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1129           /* Per-user LRU list maintenance for dynamic translation */
1130           if (!snat_is_session_static (s0))
1131             {
1132               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1133                                  s0->per_user_index);
1134               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1135                                   s0->per_user_list_head_index,
1136                                   s0->per_user_index);
1137             }
1138         trace0:
1139
1140           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1141                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1142             {
1143               snat_out2in_trace_t *t =
1144                  vlib_add_trace (vm, node, b0, sizeof (*t));
1145               t->sw_if_index = sw_if_index0;
1146               t->next_index = next0;
1147               t->session_index = ~0;
1148               if (s0)
1149                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1150             }
1151
1152           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1153
1154
1155           ip1 = vlib_buffer_get_current (b1);
1156           udp1 = ip4_next_header (ip1);
1157           tcp1 = (tcp_header_t *) udp1;
1158           icmp1 = (icmp46_header_t *) udp1;
1159
1160           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1161           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1162                                    sw_if_index1);
1163
1164           if (PREDICT_FALSE(ip1->ttl == 1))
1165             {
1166               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1167               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1168                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1169                                            0);
1170               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1171               goto trace1;
1172             }
1173
1174           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1175
1176           if (PREDICT_FALSE (proto1 == ~0))
1177             {
1178               s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1179                                              thread_index, now, vm, node);
1180               if (!s1)
1181                 next1 = SNAT_OUT2IN_NEXT_DROP;
1182               goto trace1;
1183             }
1184
1185           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1186             {
1187               next1 = icmp_out2in_slow_path
1188                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1189                  next1, now, thread_index, &s1);
1190               goto trace1;
1191             }
1192
1193           key1.addr = ip1->dst_address;
1194           key1.port = udp1->dst_port;
1195           key1.protocol = proto1;
1196           key1.fib_index = rx_fib_index1;
1197
1198           kv1.key = key1.as_u64;
1199
1200           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1201                                       &kv1, &value1))
1202             {
1203               /* Try to match static mapping by external address and port,
1204                  destination address and port in packet */
1205               if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1206                 {
1207                   b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1208                   /*
1209                    * Send DHCP packets to the ipv4 stack, or we won't
1210                    * be able to use dhcp client on the outside interface
1211                    */
1212                   if (proto1 != SNAT_PROTOCOL_UDP
1213                       || (udp1->dst_port
1214                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1215                     next1 = SNAT_OUT2IN_NEXT_DROP;
1216                   goto trace1;
1217                 }
1218
1219               /* Create session initiated by host from external network */
1220               s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1221                                                      thread_index);
1222               if (!s1)
1223                 {
1224                   next1 = SNAT_OUT2IN_NEXT_DROP;
1225                   goto trace1;
1226                 }
1227             }
1228           else
1229             {
1230               if (PREDICT_FALSE (value1.value == ~0ULL))
1231                 {
1232                   s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1233                                       now, vm, node);
1234                   if (!s1)
1235                     next1 = SNAT_OUT2IN_NEXT_DROP;
1236                   goto trace1;
1237                 }
1238               else
1239                 {
1240                   s1 = pool_elt_at_index (
1241                     sm->per_thread_data[thread_index].sessions,
1242                     value1.value);
1243                 }
1244             }
1245
1246           old_addr1 = ip1->dst_address.as_u32;
1247           ip1->dst_address = s1->in2out.addr;
1248           new_addr1 = ip1->dst_address.as_u32;
1249           vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1250
1251           sum1 = ip1->checksum;
1252           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1253                                  ip4_header_t,
1254                                  dst_address /* changed member */);
1255           ip1->checksum = ip_csum_fold (sum1);
1256
1257           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1258             {
1259               old_port1 = tcp1->dst_port;
1260               tcp1->dst_port = s1->in2out.port;
1261               new_port1 = tcp1->dst_port;
1262
1263               sum1 = tcp1->checksum;
1264               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1265                                      ip4_header_t,
1266                                      dst_address /* changed member */);
1267
1268               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1269                                      ip4_header_t /* cheat */,
1270                                      length /* changed member */);
1271               tcp1->checksum = ip_csum_fold(sum1);
1272             }
1273           else
1274             {
1275               old_port1 = udp1->dst_port;
1276               udp1->dst_port = s1->in2out.port;
1277               udp1->checksum = 0;
1278             }
1279
1280           /* Accounting */
1281           s1->last_heard = now;
1282           s1->total_pkts++;
1283           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1284           /* Per-user LRU list maintenance for dynamic translation */
1285           if (!snat_is_session_static (s1))
1286             {
1287               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1288                                  s1->per_user_index);
1289               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1290                                   s1->per_user_list_head_index,
1291                                   s1->per_user_index);
1292             }
1293         trace1:
1294
1295           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1296                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1297             {
1298               snat_out2in_trace_t *t =
1299                  vlib_add_trace (vm, node, b1, sizeof (*t));
1300               t->sw_if_index = sw_if_index1;
1301               t->next_index = next1;
1302               t->session_index = ~0;
1303               if (s1)
1304                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1305             }
1306
1307           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1308
1309           /* verify speculative enqueues, maybe switch current next frame */
1310           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1311                                            to_next, n_left_to_next,
1312                                            bi0, bi1, next0, next1);
1313         }
1314
1315       while (n_left_from > 0 && n_left_to_next > 0)
1316         {
1317           u32 bi0;
1318           vlib_buffer_t * b0;
1319           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1320           u32 sw_if_index0;
1321           ip4_header_t * ip0;
1322           ip_csum_t sum0;
1323           u32 new_addr0, old_addr0;
1324           u16 new_port0, old_port0;
1325           udp_header_t * udp0;
1326           tcp_header_t * tcp0;
1327           icmp46_header_t * icmp0;
1328           snat_session_key_t key0, sm0;
1329           u32 rx_fib_index0;
1330           u32 proto0;
1331           snat_session_t * s0 = 0;
1332           clib_bihash_kv_8_8_t kv0, value0;
1333
1334           /* speculatively enqueue b0 to the current next frame */
1335           bi0 = from[0];
1336           to_next[0] = bi0;
1337           from += 1;
1338           to_next += 1;
1339           n_left_from -= 1;
1340           n_left_to_next -= 1;
1341
1342           b0 = vlib_get_buffer (vm, bi0);
1343
1344           vnet_buffer (b0)->snat.flags = 0;
1345
1346           ip0 = vlib_buffer_get_current (b0);
1347           udp0 = ip4_next_header (ip0);
1348           tcp0 = (tcp_header_t *) udp0;
1349           icmp0 = (icmp46_header_t *) udp0;
1350
1351           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1352           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1353                                    sw_if_index0);
1354
1355           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1356
1357           if (PREDICT_FALSE (proto0 == ~0))
1358             {
1359               s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1360                                              thread_index, now, vm, node);
1361               if (!s0)
1362                 next0 = SNAT_OUT2IN_NEXT_DROP;
1363               goto trace00;
1364             }
1365
1366           if (PREDICT_FALSE(ip0->ttl == 1))
1367             {
1368               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1369               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1370                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1371                                            0);
1372               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1373               goto trace00;
1374             }
1375
1376           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1377             {
1378               next0 = icmp_out2in_slow_path
1379                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1380                  next0, now, thread_index, &s0);
1381               goto trace00;
1382             }
1383
1384           key0.addr = ip0->dst_address;
1385           key0.port = udp0->dst_port;
1386           key0.protocol = proto0;
1387           key0.fib_index = rx_fib_index0;
1388
1389           kv0.key = key0.as_u64;
1390
1391           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1392                                       &kv0, &value0))
1393             {
1394               /* Try to match static mapping by external address and port,
1395                  destination address and port in packet */
1396               if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1397                 {
1398                   b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1399                   /*
1400                    * Send DHCP packets to the ipv4 stack, or we won't
1401                    * be able to use dhcp client on the outside interface
1402                    */
1403                   if (proto0 != SNAT_PROTOCOL_UDP
1404                       || (udp0->dst_port
1405                           != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1406
1407                     next0 = SNAT_OUT2IN_NEXT_DROP;
1408                   goto trace00;
1409                 }
1410
1411               /* Create session initiated by host from external network */
1412               s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1413                                                      thread_index);
1414               if (!s0)
1415                 {
1416                   next0 = SNAT_OUT2IN_NEXT_DROP;
1417                   goto trace00;
1418                 }
1419             }
1420           else
1421             {
1422               if (PREDICT_FALSE (value0.value == ~0ULL))
1423                 {
1424                   s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1425                                       now, vm, node);
1426                   if (!s0)
1427                     next0 = SNAT_OUT2IN_NEXT_DROP;
1428                   goto trace00;
1429                 }
1430               else
1431                 {
1432                   s0 = pool_elt_at_index (
1433                     sm->per_thread_data[thread_index].sessions,
1434                     value0.value);
1435                 }
1436             }
1437
1438           old_addr0 = ip0->dst_address.as_u32;
1439           ip0->dst_address = s0->in2out.addr;
1440           new_addr0 = ip0->dst_address.as_u32;
1441           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1442
1443           sum0 = ip0->checksum;
1444           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1445                                  ip4_header_t,
1446                                  dst_address /* changed member */);
1447           ip0->checksum = ip_csum_fold (sum0);
1448
1449           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1450             {
1451               old_port0 = tcp0->dst_port;
1452               tcp0->dst_port = s0->in2out.port;
1453               new_port0 = tcp0->dst_port;
1454
1455               sum0 = tcp0->checksum;
1456               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1457                                      ip4_header_t,
1458                                      dst_address /* changed member */);
1459
1460               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1461                                      ip4_header_t /* cheat */,
1462                                      length /* changed member */);
1463               tcp0->checksum = ip_csum_fold(sum0);
1464             }
1465           else
1466             {
1467               old_port0 = udp0->dst_port;
1468               udp0->dst_port = s0->in2out.port;
1469               udp0->checksum = 0;
1470             }
1471
1472           /* Accounting */
1473           s0->last_heard = now;
1474           s0->total_pkts++;
1475           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1476           /* Per-user LRU list maintenance for dynamic translation */
1477           if (!snat_is_session_static (s0))
1478             {
1479               clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1480                                  s0->per_user_index);
1481               clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1482                                   s0->per_user_list_head_index,
1483                                   s0->per_user_index);
1484             }
1485         trace00:
1486
1487           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1488                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1489             {
1490               snat_out2in_trace_t *t =
1491                  vlib_add_trace (vm, node, b0, sizeof (*t));
1492               t->sw_if_index = sw_if_index0;
1493               t->next_index = next0;
1494               t->session_index = ~0;
1495               if (s0)
1496                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1497             }
1498
1499           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1500
1501           /* verify speculative enqueue, maybe switch current next frame */
1502           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1503                                            to_next, n_left_to_next,
1504                                            bi0, next0);
1505         }
1506
1507       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1508     }
1509
1510   vlib_node_increment_counter (vm, snat_out2in_node.index,
1511                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1512                                pkts_processed);
1513   return frame->n_vectors;
1514 }
1515
1516 VLIB_REGISTER_NODE (snat_out2in_node) = {
1517   .function = snat_out2in_node_fn,
1518   .name = "nat44-out2in",
1519   .vector_size = sizeof (u32),
1520   .format_trace = format_snat_out2in_trace,
1521   .type = VLIB_NODE_TYPE_INTERNAL,
1522
1523   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1524   .error_strings = snat_out2in_error_strings,
1525
1526   .runtime_data_bytes = sizeof (snat_runtime_t),
1527
1528   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1529
1530   /* edit / add dispositions here */
1531   .next_nodes = {
1532     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1533     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1534     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1535   },
1536 };
1537 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1538
1539 /**************************/
1540 /*** deterministic mode ***/
1541 /**************************/
1542 static uword
1543 snat_det_out2in_node_fn (vlib_main_t * vm,
1544                          vlib_node_runtime_t * node,
1545                          vlib_frame_t * frame)
1546 {
1547   u32 n_left_from, * from, * to_next;
1548   snat_out2in_next_t next_index;
1549   u32 pkts_processed = 0;
1550   snat_main_t * sm = &snat_main;
1551   u32 thread_index = vlib_get_thread_index ();
1552
1553   from = vlib_frame_vector_args (frame);
1554   n_left_from = frame->n_vectors;
1555   next_index = node->cached_next_index;
1556
1557   while (n_left_from > 0)
1558     {
1559       u32 n_left_to_next;
1560
1561       vlib_get_next_frame (vm, node, next_index,
1562                            to_next, n_left_to_next);
1563
1564       while (n_left_from >= 4 && n_left_to_next >= 2)
1565         {
1566           u32 bi0, bi1;
1567           vlib_buffer_t * b0, * b1;
1568           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1569           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1570           u32 sw_if_index0, sw_if_index1;
1571           ip4_header_t * ip0, * ip1;
1572           ip_csum_t sum0, sum1;
1573           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1574           u16 new_port0, old_port0, old_port1, new_port1;
1575           udp_header_t * udp0, * udp1;
1576           tcp_header_t * tcp0, * tcp1;
1577           u32 proto0, proto1;
1578           snat_det_out_key_t key0, key1;
1579           snat_det_map_t * dm0, * dm1;
1580           snat_det_session_t * ses0 = 0, * ses1 = 0;
1581           u32 rx_fib_index0, rx_fib_index1;
1582           icmp46_header_t * icmp0, * icmp1;
1583
1584           /* Prefetch next iteration. */
1585           {
1586             vlib_buffer_t * p2, * p3;
1587
1588             p2 = vlib_get_buffer (vm, from[2]);
1589             p3 = vlib_get_buffer (vm, from[3]);
1590
1591             vlib_prefetch_buffer_header (p2, LOAD);
1592             vlib_prefetch_buffer_header (p3, LOAD);
1593
1594             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1595             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1596           }
1597
1598           /* speculatively enqueue b0 and b1 to the current next frame */
1599           to_next[0] = bi0 = from[0];
1600           to_next[1] = bi1 = from[1];
1601           from += 2;
1602           to_next += 2;
1603           n_left_from -= 2;
1604           n_left_to_next -= 2;
1605
1606           b0 = vlib_get_buffer (vm, bi0);
1607           b1 = vlib_get_buffer (vm, bi1);
1608
1609           ip0 = vlib_buffer_get_current (b0);
1610           udp0 = ip4_next_header (ip0);
1611           tcp0 = (tcp_header_t *) udp0;
1612
1613           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1614
1615           if (PREDICT_FALSE(ip0->ttl == 1))
1616             {
1617               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1618               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1619                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1620                                            0);
1621               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1622               goto trace0;
1623             }
1624
1625           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1626
1627           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1628             {
1629               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1630               icmp0 = (icmp46_header_t *) udp0;
1631
1632               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1633                                   rx_fib_index0, node, next0, thread_index,
1634                                   &ses0, &dm0);
1635               goto trace0;
1636             }
1637
1638           key0.ext_host_addr = ip0->src_address;
1639           key0.ext_host_port = tcp0->src;
1640           key0.out_port = tcp0->dst;
1641
1642           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1643           if (PREDICT_FALSE(!dm0))
1644             {
1645               clib_warning("unknown dst address:  %U",
1646                            format_ip4_address, &ip0->dst_address);
1647               next0 = SNAT_OUT2IN_NEXT_DROP;
1648               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1649               goto trace0;
1650             }
1651
1652           snat_det_reverse(dm0, &ip0->dst_address,
1653                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1654
1655           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1656           if (PREDICT_FALSE(!ses0))
1657             {
1658               clib_warning("no match src %U:%d dst %U:%d for user %U",
1659                            format_ip4_address, &ip0->src_address,
1660                            clib_net_to_host_u16 (tcp0->src),
1661                            format_ip4_address, &ip0->dst_address,
1662                            clib_net_to_host_u16 (tcp0->dst),
1663                            format_ip4_address, &new_addr0);
1664               next0 = SNAT_OUT2IN_NEXT_DROP;
1665               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1666               goto trace0;
1667             }
1668           new_port0 = ses0->in_port;
1669
1670           old_addr0 = ip0->dst_address;
1671           ip0->dst_address = new_addr0;
1672           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1673
1674           sum0 = ip0->checksum;
1675           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1676                                  ip4_header_t,
1677                                  dst_address /* changed member */);
1678           ip0->checksum = ip_csum_fold (sum0);
1679
1680           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1681             {
1682               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1683                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1684               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1685                 snat_det_ses_close(dm0, ses0);
1686
1687               old_port0 = tcp0->dst;
1688               tcp0->dst = new_port0;
1689
1690               sum0 = tcp0->checksum;
1691               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1692                                      ip4_header_t,
1693                                      dst_address /* changed member */);
1694
1695               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1696                                      ip4_header_t /* cheat */,
1697                                      length /* changed member */);
1698               tcp0->checksum = ip_csum_fold(sum0);
1699             }
1700           else
1701             {
1702               old_port0 = udp0->dst_port;
1703               udp0->dst_port = new_port0;
1704               udp0->checksum = 0;
1705             }
1706
1707         trace0:
1708
1709           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1710                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1711             {
1712               snat_out2in_trace_t *t =
1713                  vlib_add_trace (vm, node, b0, sizeof (*t));
1714               t->sw_if_index = sw_if_index0;
1715               t->next_index = next0;
1716               t->session_index = ~0;
1717               if (ses0)
1718                 t->session_index = ses0 - dm0->sessions;
1719             }
1720
1721           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1722
1723           b1 = vlib_get_buffer (vm, bi1);
1724
1725           ip1 = vlib_buffer_get_current (b1);
1726           udp1 = ip4_next_header (ip1);
1727           tcp1 = (tcp_header_t *) udp1;
1728
1729           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1730
1731           if (PREDICT_FALSE(ip1->ttl == 1))
1732             {
1733               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1734               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1735                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1736                                            0);
1737               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1738               goto trace1;
1739             }
1740
1741           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1742
1743           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1744             {
1745               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1746               icmp1 = (icmp46_header_t *) udp1;
1747
1748               next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1749                                   rx_fib_index1, node, next1, thread_index,
1750                                   &ses1, &dm1);
1751               goto trace1;
1752             }
1753
1754           key1.ext_host_addr = ip1->src_address;
1755           key1.ext_host_port = tcp1->src;
1756           key1.out_port = tcp1->dst;
1757
1758           dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1759           if (PREDICT_FALSE(!dm1))
1760             {
1761               clib_warning("unknown dst address:  %U",
1762                            format_ip4_address, &ip1->dst_address);
1763               next1 = SNAT_OUT2IN_NEXT_DROP;
1764               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1765               goto trace1;
1766             }
1767
1768           snat_det_reverse(dm1, &ip1->dst_address,
1769                            clib_net_to_host_u16(tcp1->dst), &new_addr1);
1770
1771           ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1772           if (PREDICT_FALSE(!ses1))
1773             {
1774               clib_warning("no match src %U:%d dst %U:%d for user %U",
1775                            format_ip4_address, &ip1->src_address,
1776                            clib_net_to_host_u16 (tcp1->src),
1777                            format_ip4_address, &ip1->dst_address,
1778                            clib_net_to_host_u16 (tcp1->dst),
1779                            format_ip4_address, &new_addr1);
1780               next1 = SNAT_OUT2IN_NEXT_DROP;
1781               b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1782               goto trace1;
1783             }
1784           new_port1 = ses1->in_port;
1785
1786           old_addr1 = ip1->dst_address;
1787           ip1->dst_address = new_addr1;
1788           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1789
1790           sum1 = ip1->checksum;
1791           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1792                                  ip4_header_t,
1793                                  dst_address /* changed member */);
1794           ip1->checksum = ip_csum_fold (sum1);
1795
1796           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1797             {
1798               if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1799                 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1800               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1801                 snat_det_ses_close(dm1, ses1);
1802
1803               old_port1 = tcp1->dst;
1804               tcp1->dst = new_port1;
1805
1806               sum1 = tcp1->checksum;
1807               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1808                                      ip4_header_t,
1809                                      dst_address /* changed member */);
1810
1811               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1812                                      ip4_header_t /* cheat */,
1813                                      length /* changed member */);
1814               tcp1->checksum = ip_csum_fold(sum1);
1815             }
1816           else
1817             {
1818               old_port1 = udp1->dst_port;
1819               udp1->dst_port = new_port1;
1820               udp1->checksum = 0;
1821             }
1822
1823         trace1:
1824
1825           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1826                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1827             {
1828               snat_out2in_trace_t *t =
1829                  vlib_add_trace (vm, node, b1, sizeof (*t));
1830               t->sw_if_index = sw_if_index1;
1831               t->next_index = next1;
1832               t->session_index = ~0;
1833               if (ses1)
1834                 t->session_index = ses1 - dm1->sessions;
1835             }
1836
1837           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1838
1839           /* verify speculative enqueues, maybe switch current next frame */
1840           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1841                                            to_next, n_left_to_next,
1842                                            bi0, bi1, next0, next1);
1843          }
1844
1845       while (n_left_from > 0 && n_left_to_next > 0)
1846         {
1847           u32 bi0;
1848           vlib_buffer_t * b0;
1849           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1850           u32 sw_if_index0;
1851           ip4_header_t * ip0;
1852           ip_csum_t sum0;
1853           ip4_address_t new_addr0, old_addr0;
1854           u16 new_port0, old_port0;
1855           udp_header_t * udp0;
1856           tcp_header_t * tcp0;
1857           u32 proto0;
1858           snat_det_out_key_t key0;
1859           snat_det_map_t * dm0;
1860           snat_det_session_t * ses0 = 0;
1861           u32 rx_fib_index0;
1862           icmp46_header_t * icmp0;
1863
1864           /* speculatively enqueue b0 to the current next frame */
1865           bi0 = from[0];
1866           to_next[0] = bi0;
1867           from += 1;
1868           to_next += 1;
1869           n_left_from -= 1;
1870           n_left_to_next -= 1;
1871
1872           b0 = vlib_get_buffer (vm, bi0);
1873
1874           ip0 = vlib_buffer_get_current (b0);
1875           udp0 = ip4_next_header (ip0);
1876           tcp0 = (tcp_header_t *) udp0;
1877
1878           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1879
1880           if (PREDICT_FALSE(ip0->ttl == 1))
1881             {
1882               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1883               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1884                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1885                                            0);
1886               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1887               goto trace00;
1888             }
1889
1890           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1891
1892           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1893             {
1894               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1895               icmp0 = (icmp46_header_t *) udp0;
1896
1897               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1898                                   rx_fib_index0, node, next0, thread_index,
1899                                   &ses0, &dm0);
1900               goto trace00;
1901             }
1902
1903           key0.ext_host_addr = ip0->src_address;
1904           key0.ext_host_port = tcp0->src;
1905           key0.out_port = tcp0->dst;
1906
1907           dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1908           if (PREDICT_FALSE(!dm0))
1909             {
1910               clib_warning("unknown dst address:  %U",
1911                            format_ip4_address, &ip0->dst_address);
1912               next0 = SNAT_OUT2IN_NEXT_DROP;
1913               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1914               goto trace00;
1915             }
1916
1917           snat_det_reverse(dm0, &ip0->dst_address,
1918                            clib_net_to_host_u16(tcp0->dst), &new_addr0);
1919
1920           ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1921           if (PREDICT_FALSE(!ses0))
1922             {
1923               clib_warning("no match src %U:%d dst %U:%d for user %U",
1924                            format_ip4_address, &ip0->src_address,
1925                            clib_net_to_host_u16 (tcp0->src),
1926                            format_ip4_address, &ip0->dst_address,
1927                            clib_net_to_host_u16 (tcp0->dst),
1928                            format_ip4_address, &new_addr0);
1929               next0 = SNAT_OUT2IN_NEXT_DROP;
1930               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1931               goto trace00;
1932             }
1933           new_port0 = ses0->in_port;
1934
1935           old_addr0 = ip0->dst_address;
1936           ip0->dst_address = new_addr0;
1937           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1938
1939           sum0 = ip0->checksum;
1940           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1941                                  ip4_header_t,
1942                                  dst_address /* changed member */);
1943           ip0->checksum = ip_csum_fold (sum0);
1944
1945           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1946             {
1947               if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1948                 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1949               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1950                 snat_det_ses_close(dm0, ses0);
1951
1952               old_port0 = tcp0->dst;
1953               tcp0->dst = new_port0;
1954
1955               sum0 = tcp0->checksum;
1956               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1957                                      ip4_header_t,
1958                                      dst_address /* changed member */);
1959
1960               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1961                                      ip4_header_t /* cheat */,
1962                                      length /* changed member */);
1963               tcp0->checksum = ip_csum_fold(sum0);
1964             }
1965           else
1966             {
1967               old_port0 = udp0->dst_port;
1968               udp0->dst_port = new_port0;
1969               udp0->checksum = 0;
1970             }
1971
1972         trace00:
1973
1974           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1975                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1976             {
1977               snat_out2in_trace_t *t =
1978                  vlib_add_trace (vm, node, b0, sizeof (*t));
1979               t->sw_if_index = sw_if_index0;
1980               t->next_index = next0;
1981               t->session_index = ~0;
1982               if (ses0)
1983                 t->session_index = ses0 - dm0->sessions;
1984             }
1985
1986           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1987
1988           /* verify speculative enqueue, maybe switch current next frame */
1989           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1990                                            to_next, n_left_to_next,
1991                                            bi0, next0);
1992         }
1993
1994       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1995     }
1996
1997   vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1998                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1999                                pkts_processed);
2000   return frame->n_vectors;
2001 }
2002
2003 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2004   .function = snat_det_out2in_node_fn,
2005   .name = "nat44-det-out2in",
2006   .vector_size = sizeof (u32),
2007   .format_trace = format_snat_out2in_trace,
2008   .type = VLIB_NODE_TYPE_INTERNAL,
2009
2010   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2011   .error_strings = snat_out2in_error_strings,
2012
2013   .runtime_data_bytes = sizeof (snat_runtime_t),
2014
2015   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2016
2017   /* edit / add dispositions here */
2018   .next_nodes = {
2019     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2020     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2021     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2022   },
2023 };
2024 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2025
2026 /**
2027  * Get address and port values to be used for ICMP packet translation
2028  * and create session if needed
2029  *
2030  * @param[in,out] sm             NAT main
2031  * @param[in,out] node           NAT node runtime
2032  * @param[in] thread_index       thread index
2033  * @param[in,out] b0             buffer containing packet to be translated
2034  * @param[out] p_proto           protocol used for matching
2035  * @param[out] p_value           address and port after NAT translation
2036  * @param[out] p_dont_translate  if packet should not be translated
2037  * @param d                      optional parameter
2038  * @param e                      optional parameter
2039  */
2040 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2041                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2042                           snat_session_key_t *p_value,
2043                           u8 *p_dont_translate, void *d, void *e)
2044 {
2045   ip4_header_t *ip0;
2046   icmp46_header_t *icmp0;
2047   u32 sw_if_index0;
2048   u8 protocol;
2049   snat_det_out_key_t key0;
2050   u8 dont_translate = 0;
2051   u32 next0 = ~0;
2052   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2053   ip4_header_t *inner_ip0;
2054   void *l4_header = 0;
2055   icmp46_header_t *inner_icmp0;
2056   snat_det_map_t * dm0 = 0;
2057   ip4_address_t new_addr0 = {{0}};
2058   snat_det_session_t * ses0 = 0;
2059   ip4_address_t out_addr;
2060
2061   ip0 = vlib_buffer_get_current (b0);
2062   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2063   echo0 = (icmp_echo_header_t *)(icmp0+1);
2064   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2065
2066   if (!icmp_is_error_message (icmp0))
2067     {
2068       protocol = SNAT_PROTOCOL_ICMP;
2069       key0.ext_host_addr = ip0->src_address;
2070       key0.ext_host_port = 0;
2071       key0.out_port = echo0->identifier;
2072       out_addr = ip0->dst_address;
2073     }
2074   else
2075     {
2076       inner_ip0 = (ip4_header_t *)(echo0+1);
2077       l4_header = ip4_next_header (inner_ip0);
2078       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2079       key0.ext_host_addr = inner_ip0->dst_address;
2080       out_addr = inner_ip0->src_address;
2081       switch (protocol)
2082         {
2083         case SNAT_PROTOCOL_ICMP:
2084           inner_icmp0 = (icmp46_header_t*)l4_header;
2085           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2086           key0.ext_host_port = 0;
2087           key0.out_port = inner_echo0->identifier;
2088           break;
2089         case SNAT_PROTOCOL_UDP:
2090         case SNAT_PROTOCOL_TCP:
2091           key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2092           key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2093           break;
2094         default:
2095           b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2096           next0 = SNAT_OUT2IN_NEXT_DROP;
2097           goto out;
2098         }
2099     }
2100
2101   dm0 = snat_det_map_by_out(sm, &out_addr);
2102   if (PREDICT_FALSE(!dm0))
2103     {
2104       /* Don't NAT packet aimed at the intfc address */
2105       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2106                                           ip0->dst_address.as_u32)))
2107         {
2108           dont_translate = 1;
2109           goto out;
2110         }
2111       clib_warning("unknown dst address:  %U",
2112                    format_ip4_address, &ip0->dst_address);
2113       goto out;
2114     }
2115
2116   snat_det_reverse(dm0, &ip0->dst_address,
2117                    clib_net_to_host_u16(key0.out_port), &new_addr0);
2118
2119   ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2120   if (PREDICT_FALSE(!ses0))
2121     {
2122       /* Don't NAT packet aimed at the intfc address */
2123       if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2124                                           ip0->dst_address.as_u32)))
2125         {
2126           dont_translate = 1;
2127           goto out;
2128         }
2129       clib_warning("no match src %U:%d dst %U:%d for user %U",
2130                    format_ip4_address, &key0.ext_host_addr,
2131                    clib_net_to_host_u16 (key0.ext_host_port),
2132                    format_ip4_address, &out_addr,
2133                    clib_net_to_host_u16 (key0.out_port),
2134                    format_ip4_address, &new_addr0);
2135       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2136       next0 = SNAT_OUT2IN_NEXT_DROP;
2137       goto out;
2138     }
2139
2140   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2141                     !icmp_is_error_message (icmp0)))
2142     {
2143       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2144       next0 = SNAT_OUT2IN_NEXT_DROP;
2145       goto out;
2146     }
2147
2148   goto out;
2149
2150 out:
2151   *p_proto = protocol;
2152   if (ses0)
2153     {
2154       p_value->addr = new_addr0;
2155       p_value->fib_index = sm->inside_fib_index;
2156       p_value->port = ses0->in_port;
2157     }
2158   *p_dont_translate = dont_translate;
2159   if (d)
2160     *(snat_det_session_t**)d = ses0;
2161   if (e)
2162     *(snat_det_map_t**)e = dm0;
2163   return next0;
2164 }
2165
2166 /**********************/
2167 /*** worker handoff ***/
2168 /**********************/
2169 static uword
2170 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2171                                vlib_node_runtime_t * node,
2172                                vlib_frame_t * frame)
2173 {
2174   snat_main_t *sm = &snat_main;
2175   vlib_thread_main_t *tm = vlib_get_thread_main ();
2176   u32 n_left_from, *from, *to_next = 0;
2177   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2178   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2179     = 0;
2180   vlib_frame_queue_elt_t *hf = 0;
2181   vlib_frame_t *f = 0;
2182   int i;
2183   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2184   u32 next_worker_index = 0;
2185   u32 current_worker_index = ~0;
2186   u32 thread_index = vlib_get_thread_index ();
2187
2188   ASSERT (vec_len (sm->workers));
2189
2190   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2191     {
2192       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2193
2194       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2195                                sm->first_worker_index + sm->num_workers - 1,
2196                                (vlib_frame_queue_t *) (~0));
2197     }
2198
2199   from = vlib_frame_vector_args (frame);
2200   n_left_from = frame->n_vectors;
2201
2202   while (n_left_from > 0)
2203     {
2204       u32 bi0;
2205       vlib_buffer_t *b0;
2206       u32 sw_if_index0;
2207       u32 rx_fib_index0;
2208       ip4_header_t * ip0;
2209       u8 do_handoff;
2210
2211       bi0 = from[0];
2212       from += 1;
2213       n_left_from -= 1;
2214
2215       b0 = vlib_get_buffer (vm, bi0);
2216
2217       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2218       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2219
2220       ip0 = vlib_buffer_get_current (b0);
2221
2222       next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2223
2224       if (PREDICT_FALSE (next_worker_index != thread_index))
2225         {
2226           do_handoff = 1;
2227
2228           if (next_worker_index != current_worker_index)
2229             {
2230               if (hf)
2231                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2232
2233               hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2234                                                       next_worker_index,
2235                                                       handoff_queue_elt_by_worker_index);
2236
2237               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2238               to_next_worker = &hf->buffer_index[hf->n_vectors];
2239               current_worker_index = next_worker_index;
2240             }
2241
2242           /* enqueue to correct worker thread */
2243           to_next_worker[0] = bi0;
2244           to_next_worker++;
2245           n_left_to_next_worker--;
2246
2247           if (n_left_to_next_worker == 0)
2248             {
2249               hf->n_vectors = VLIB_FRAME_SIZE;
2250               vlib_put_frame_queue_elt (hf);
2251               current_worker_index = ~0;
2252               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2253               hf = 0;
2254             }
2255         }
2256       else
2257         {
2258           do_handoff = 0;
2259           /* if this is 1st frame */
2260           if (!f)
2261             {
2262               f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2263               to_next = vlib_frame_vector_args (f);
2264             }
2265
2266           to_next[0] = bi0;
2267           to_next += 1;
2268           f->n_vectors++;
2269         }
2270
2271       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2272                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2273         {
2274           snat_out2in_worker_handoff_trace_t *t =
2275             vlib_add_trace (vm, node, b0, sizeof (*t));
2276           t->next_worker_index = next_worker_index;
2277           t->do_handoff = do_handoff;
2278         }
2279     }
2280
2281   if (f)
2282     vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2283
2284   if (hf)
2285     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2286
2287   /* Ship frames to the worker nodes */
2288   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2289     {
2290       if (handoff_queue_elt_by_worker_index[i])
2291         {
2292           hf = handoff_queue_elt_by_worker_index[i];
2293           /*
2294            * It works better to let the handoff node
2295            * rate-adapt, always ship the handoff queue element.
2296            */
2297           if (1 || hf->n_vectors == hf->last_n_vectors)
2298             {
2299               vlib_put_frame_queue_elt (hf);
2300               handoff_queue_elt_by_worker_index[i] = 0;
2301             }
2302           else
2303             hf->last_n_vectors = hf->n_vectors;
2304         }
2305       congested_handoff_queue_by_worker_index[i] =
2306         (vlib_frame_queue_t *) (~0);
2307     }
2308   hf = 0;
2309   current_worker_index = ~0;
2310   return frame->n_vectors;
2311 }
2312
2313 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2314   .function = snat_out2in_worker_handoff_fn,
2315   .name = "nat44-out2in-worker-handoff",
2316   .vector_size = sizeof (u32),
2317   .format_trace = format_snat_out2in_worker_handoff_trace,
2318   .type = VLIB_NODE_TYPE_INTERNAL,
2319
2320   .n_next_nodes = 1,
2321
2322   .next_nodes = {
2323     [0] = "error-drop",
2324   },
2325 };
2326
2327 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2328
2329 static uword
2330 snat_out2in_fast_node_fn (vlib_main_t * vm,
2331                           vlib_node_runtime_t * node,
2332                           vlib_frame_t * frame)
2333 {
2334   u32 n_left_from, * from, * to_next;
2335   snat_out2in_next_t next_index;
2336   u32 pkts_processed = 0;
2337   snat_main_t * sm = &snat_main;
2338
2339   from = vlib_frame_vector_args (frame);
2340   n_left_from = frame->n_vectors;
2341   next_index = node->cached_next_index;
2342
2343   while (n_left_from > 0)
2344     {
2345       u32 n_left_to_next;
2346
2347       vlib_get_next_frame (vm, node, next_index,
2348                            to_next, n_left_to_next);
2349
2350       while (n_left_from > 0 && n_left_to_next > 0)
2351         {
2352           u32 bi0;
2353           vlib_buffer_t * b0;
2354           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2355           u32 sw_if_index0;
2356           ip4_header_t * ip0;
2357           ip_csum_t sum0;
2358           u32 new_addr0, old_addr0;
2359           u16 new_port0, old_port0;
2360           udp_header_t * udp0;
2361           tcp_header_t * tcp0;
2362           icmp46_header_t * icmp0;
2363           snat_session_key_t key0, sm0;
2364           u32 proto0;
2365           u32 rx_fib_index0;
2366
2367           /* speculatively enqueue b0 to the current next frame */
2368           bi0 = from[0];
2369           to_next[0] = bi0;
2370           from += 1;
2371           to_next += 1;
2372           n_left_from -= 1;
2373           n_left_to_next -= 1;
2374
2375           b0 = vlib_get_buffer (vm, bi0);
2376
2377           ip0 = vlib_buffer_get_current (b0);
2378           udp0 = ip4_next_header (ip0);
2379           tcp0 = (tcp_header_t *) udp0;
2380           icmp0 = (icmp46_header_t *) udp0;
2381
2382           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2383           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2384
2385           vnet_feature_next (sw_if_index0, &next0, b0);
2386
2387           if (PREDICT_FALSE(ip0->ttl == 1))
2388             {
2389               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2390               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2391                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2392                                            0);
2393               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2394               goto trace00;
2395             }
2396
2397           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2398
2399           if (PREDICT_FALSE (proto0 == ~0))
2400               goto trace00;
2401
2402           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2403             {
2404               next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2405                                   rx_fib_index0, node, next0, ~0, 0, 0);
2406               goto trace00;
2407             }
2408
2409           key0.addr = ip0->dst_address;
2410           key0.port = udp0->dst_port;
2411           key0.fib_index = rx_fib_index0;
2412
2413           if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2414             {
2415               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2416               goto trace00;
2417             }
2418
2419           new_addr0 = sm0.addr.as_u32;
2420           new_port0 = sm0.port;
2421           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2422           old_addr0 = ip0->dst_address.as_u32;
2423           ip0->dst_address.as_u32 = new_addr0;
2424
2425           sum0 = ip0->checksum;
2426           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2427                                  ip4_header_t,
2428                                  dst_address /* changed member */);
2429           ip0->checksum = ip_csum_fold (sum0);
2430
2431           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2432             {
2433                if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2434                 {
2435                   old_port0 = tcp0->dst_port;
2436                   tcp0->dst_port = new_port0;
2437
2438                   sum0 = tcp0->checksum;
2439                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2440                                          ip4_header_t,
2441                                          dst_address /* changed member */);
2442
2443                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2444                                          ip4_header_t /* cheat */,
2445                                          length /* changed member */);
2446                   tcp0->checksum = ip_csum_fold(sum0);
2447                 }
2448               else
2449                 {
2450                   old_port0 = udp0->dst_port;
2451                   udp0->dst_port = new_port0;
2452                   udp0->checksum = 0;
2453                 }
2454             }
2455           else
2456             {
2457               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2458                 {
2459                   sum0 = tcp0->checksum;
2460                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2461                                          ip4_header_t,
2462                                          dst_address /* changed member */);
2463
2464                   tcp0->checksum = ip_csum_fold(sum0);
2465                 }
2466             }
2467
2468         trace00:
2469
2470           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2471                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2472             {
2473               snat_out2in_trace_t *t =
2474                  vlib_add_trace (vm, node, b0, sizeof (*t));
2475               t->sw_if_index = sw_if_index0;
2476               t->next_index = next0;
2477             }
2478
2479           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2480
2481           /* verify speculative enqueue, maybe switch current next frame */
2482           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2483                                            to_next, n_left_to_next,
2484                                            bi0, next0);
2485         }
2486
2487       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2488     }
2489
2490   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2491                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2492                                pkts_processed);
2493   return frame->n_vectors;
2494 }
2495
2496 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2497   .function = snat_out2in_fast_node_fn,
2498   .name = "nat44-out2in-fast",
2499   .vector_size = sizeof (u32),
2500   .format_trace = format_snat_out2in_fast_trace,
2501   .type = VLIB_NODE_TYPE_INTERNAL,
2502
2503   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2504   .error_strings = snat_out2in_error_strings,
2505
2506   .runtime_data_bytes = sizeof (snat_runtime_t),
2507
2508   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2509
2510   /* edit / add dispositions here */
2511   .next_nodes = {
2512     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2513     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2514     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2515   },
2516 };
2517 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);