src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/udp/udp.h>
  26 #include <vnet/ethernet/ethernet.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_reass.h>
  31 #include <nat/nat_inlines.h>
  32
  33 #include <vppinfra/hash.h>
  34 #include <vppinfra/error.h>
  35 #include <vppinfra/elog.h>
  36
  37 typedef struct
  38 {
  39   u32 sw_if_index;
  40   u32 next_index;
  41   u32 session_index;
  42 } snat_out2in_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 *
  46 format_snat_out2in_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
  51
  52   s =
  53     format (s,
  54             "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  55             t->sw_if_index, t->next_index, t->session_index);
  56   return s;
  57 }
  58
  59 static u8 *
  60 format_snat_out2in_fast_trace (u8 * s, va_list * args)
  61 {
  62   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  63   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  64   snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
  65
  66   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  67               t->sw_if_index, t->next_index);
  68   return s;
  69 }
  70
  71 vlib_node_registration_t snat_out2in_node;
  72 vlib_node_registration_t snat_out2in_fast_node;
  73 vlib_node_registration_t nat44_out2in_reass_node;
  74
  75 #define foreach_snat_out2in_error                       \
  76 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  77 _(OUT2IN_PACKETS, "Good out2in packets processed")      \
  78 _(OUT_OF_PORTS, "Out of ports")                         \
  79 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  80 _(NO_TRANSLATION, "No translation")                     \
  81 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  82 _(DROP_FRAGMENT, "Drop fragment")                       \
  83 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  84 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  85
  86 typedef enum
  87 {
  88 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  89   foreach_snat_out2in_error
  90 #undef _
  91     SNAT_OUT2IN_N_ERROR,
  92 } snat_out2in_error_t;
  93
  94 static char *snat_out2in_error_strings[] = {
  95 #define _(sym,string) string,
  96   foreach_snat_out2in_error
  97 #undef _
  98 };
  99
 100 typedef enum
 101 {
 102   SNAT_OUT2IN_NEXT_DROP,
 103   SNAT_OUT2IN_NEXT_LOOKUP,
 104   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 105   SNAT_OUT2IN_NEXT_REASS,
 106   SNAT_OUT2IN_N_NEXT,
 107 } snat_out2in_next_t;
 108
 109 int
 110 nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 111 {
 112   snat_main_t *sm = &snat_main;
 113   nat44_is_idle_session_ctx_t *ctx = arg;
 114   snat_session_t *s;
 115   u64 sess_timeout_time;
 116   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 117                                                        ctx->thread_index);
 118   clib_bihash_kv_8_8_t s_kv;
 119
 120   s = pool_elt_at_index (tsm->sessions, kv->value);
 121   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 122   if (ctx->now >= sess_timeout_time)
 123     {
 124       s_kv.key = s->in2out.as_u64;
 125       if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
 126         nat_log_warn ("out2in key del failed");
 127
 128       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 129                                            s->out2in.addr.as_u32,
 130                                            s->in2out.protocol,
 131                                            s->in2out.port,
 132                                            s->out2in.port,
 133                                            s->in2out.fib_index);
 134
 135       if (!snat_is_session_static (s))
 136         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 137                                             &s->out2in);
 138
 139       nat44_delete_session (sm, s, ctx->thread_index);
 140       return 1;
 141     }
 142
 143   return 0;
 144 }
 145
 146 /**
 147  * @brief Create session for static mapping.
 148  *
 149  * Create NAT session initiated by host from external network with static
 150  * mapping.
 151  *
 152  * @param sm     NAT main.
 153  * @param b0     Vlib buffer.
 154  * @param in2out In2out NAT44 session key.
 155  * @param out2in Out2in NAT44 session key.
 156  * @param node   Vlib node.
 157  *
 158  * @returns SNAT session if successfully created otherwise 0.
 159  */
 160 static inline snat_session_t *
 161 create_session_for_static_mapping (snat_main_t * sm,
 162                                    vlib_buffer_t * b0,
 163                                    snat_session_key_t in2out,
 164                                    snat_session_key_t out2in,
 165                                    vlib_node_runtime_t * node,
 166                                    u32 thread_index, f64 now)
 167 {
 168   snat_user_t *u;
 169   snat_session_t *s;
 170   clib_bihash_kv_8_8_t kv0;
 171   ip4_header_t *ip0;
 172   udp_header_t *udp0;
 173   nat44_is_idle_session_ctx_t ctx0;
 174
 175   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 176     {
 177       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 178       nat_log_notice ("maximum sessions exceeded");
 179       return 0;
 180     }
 181
 182   ip0 = vlib_buffer_get_current (b0);
 183   udp0 = ip4_next_header (ip0);
 184
 185   u =
 186     nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
 187   if (!u)
 188     {
 189       nat_log_warn ("create NAT user failed");
 190       return 0;
 191     }
 192
 193   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 194   if (!s)
 195     {
 196       nat44_delete_user_with_no_session (sm, u, thread_index);
 197       nat_log_warn ("create NAT session failed");
 198       return 0;
 199     }
 200
 201   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 202   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 203   s->ext_host_port = udp0->src_port;
 204   user_session_increment (sm, u, 1 /* static */ );
 205   s->in2out = in2out;
 206   s->out2in = out2in;
 207   s->in2out.protocol = out2in.protocol;
 208
 209   /* Add to translation hashes */
 210   ctx0.now = now;
 211   ctx0.thread_index = thread_index;
 212   kv0.key = s->in2out.as_u64;
 213   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 214   if (clib_bihash_add_or_overwrite_stale_8_8
 215       (&sm->per_thread_data[thread_index].in2out, &kv0,
 216        nat44_i2o_is_idle_session_cb, &ctx0))
 217     nat_log_notice ("in2out key add failed");
 218
 219   kv0.key = s->out2in.as_u64;
 220
 221   if (clib_bihash_add_or_overwrite_stale_8_8
 222       (&sm->per_thread_data[thread_index].out2in, &kv0,
 223        nat44_o2i_is_idle_session_cb, &ctx0))
 224     nat_log_notice ("out2in key add failed");
 225
 226   /* log NAT event */
 227   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 228                                        s->out2in.addr.as_u32,
 229                                        s->in2out.protocol,
 230                                        s->in2out.port,
 231                                        s->out2in.port, s->in2out.fib_index);
 232   return s;
 233 }
 234
 235 static_always_inline
 236   snat_out2in_error_t icmp_get_key (ip4_header_t * ip0,
 237                                     snat_session_key_t * p_key0)
 238 {
 239   icmp46_header_t *icmp0;
 240   snat_session_key_t key0;
 241   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 242   ip4_header_t *inner_ip0;
 243   void *l4_header = 0;
 244   icmp46_header_t *inner_icmp0;
 245
 246   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 247   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 248
 249   if (!icmp_is_error_message (icmp0))
 250     {
 251       key0.protocol = SNAT_PROTOCOL_ICMP;
 252       key0.addr = ip0->dst_address;
 253       key0.port = echo0->identifier;
 254     }
 255   else
 256     {
 257       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 258       l4_header = ip4_next_header (inner_ip0);
 259       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 260       key0.addr = inner_ip0->src_address;
 261       switch (key0.protocol)
 262         {
 263         case SNAT_PROTOCOL_ICMP:
 264           inner_icmp0 = (icmp46_header_t *) l4_header;
 265           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 266           key0.port = inner_echo0->identifier;
 267           break;
 268         case SNAT_PROTOCOL_UDP:
 269         case SNAT_PROTOCOL_TCP:
 270           key0.port = ((tcp_udp_header_t *) l4_header)->src_port;
 271           break;
 272         default:
 273           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 274         }
 275     }
 276   *p_key0 = key0;
 277   return -1;                    /* success */
 278 }
 279
 280 /**
 281  * Get address and port values to be used for ICMP packet translation
 282  * and create session if needed
 283  *
 284  * @param[in,out] sm             NAT main
 285  * @param[in,out] node           NAT node runtime
 286  * @param[in] thread_index       thread index
 287  * @param[in,out] b0             buffer containing packet to be translated
 288  * @param[out] p_proto           protocol used for matching
 289  * @param[out] p_value           address and port after NAT translation
 290  * @param[out] p_dont_translate  if packet should not be translated
 291  * @param d                      optional parameter
 292  * @param e                      optional parameter
 293  */
 294 u32
 295 icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 296                         u32 thread_index, vlib_buffer_t * b0,
 297                         ip4_header_t * ip0, u8 * p_proto,
 298                         snat_session_key_t * p_value,
 299                         u8 * p_dont_translate, void *d, void *e)
 300 {
 301   icmp46_header_t *icmp0;
 302   u32 sw_if_index0;
 303   u32 rx_fib_index0;
 304   snat_session_key_t key0;
 305   snat_session_key_t sm0;
 306   snat_session_t *s0 = 0;
 307   u8 dont_translate = 0;
 308   clib_bihash_kv_8_8_t kv0, value0;
 309   u8 is_addr_only;
 310   u32 next0 = ~0;
 311   int err;
 312
 313   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 314   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 315   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 316
 317   key0.protocol = 0;
 318
 319   err = icmp_get_key (ip0, &key0);
 320   if (err != -1)
 321     {
 322       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 323       next0 = SNAT_OUT2IN_NEXT_DROP;
 324       goto out;
 325     }
 326   key0.fib_index = rx_fib_index0;
 327
 328   kv0.key = key0.as_u64;
 329
 330   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 331                               &value0))
 332     {
 333       /* Try to match static mapping by external address and port,
 334          destination address and port in packet */
 335       if (snat_static_mapping_match
 336           (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0))
 337         {
 338           if (!sm->forwarding_enabled)
 339             {
 340               /* Don't NAT packet aimed at the intfc address */
 341               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index0,
 342                                                     ip0->dst_address.as_u32)))
 343                 {
 344                   dont_translate = 1;
 345                   goto out;
 346                 }
 347               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 348               next0 = SNAT_OUT2IN_NEXT_DROP;
 349               goto out;
 350             }
 351           else
 352             {
 353               dont_translate = 1;
 354               goto out;
 355             }
 356         }
 357
 358       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_reply &&
 359                          (icmp0->type != ICMP4_echo_request
 360                           || !is_addr_only)))
 361         {
 362           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 363           next0 = SNAT_OUT2IN_NEXT_DROP;
 364           goto out;
 365         }
 366
 367       /* Create session initiated by host from external network */
 368       s0 = create_session_for_static_mapping (sm, b0, sm0, key0,
 369                                               node, thread_index,
 370                                               vlib_time_now (sm->vlib_main));
 371
 372       if (!s0)
 373         {
 374           next0 = SNAT_OUT2IN_NEXT_DROP;
 375           goto out;
 376         }
 377     }
 378   else
 379     {
 380       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_reply &&
 381                          icmp0->type != ICMP4_echo_request &&
 382                          !icmp_is_error_message (icmp0)))
 383         {
 384           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 385           next0 = SNAT_OUT2IN_NEXT_DROP;
 386           goto out;
 387         }
 388
 389       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 390                               value0.value);
 391     }
 392
 393 out:
 394   *p_proto = key0.protocol;
 395   if (s0)
 396     *p_value = s0->in2out;
 397   *p_dont_translate = dont_translate;
 398   if (d)
 399     *(snat_session_t **) d = s0;
 400   return next0;
 401 }
 402
 403 /**
 404  * Get address and port values to be used for ICMP packet translation
 405  *
 406  * @param[in] sm                 NAT main
 407  * @param[in,out] node           NAT node runtime
 408  * @param[in] thread_index       thread index
 409  * @param[in,out] b0             buffer containing packet to be translated
 410  * @param[out] p_proto           protocol used for matching
 411  * @param[out] p_value           address and port after NAT translation
 412  * @param[out] p_dont_translate  if packet should not be translated
 413  * @param d                      optional parameter
 414  * @param e                      optional parameter
 415  */
 416 u32
 417 icmp_match_out2in_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 418                         u32 thread_index, vlib_buffer_t * b0,
 419                         ip4_header_t * ip0, u8 * p_proto,
 420                         snat_session_key_t * p_value,
 421                         u8 * p_dont_translate, void *d, void *e)
 422 {
 423   icmp46_header_t *icmp0;
 424   u32 sw_if_index0;
 425   u32 rx_fib_index0;
 426   snat_session_key_t key0;
 427   snat_session_key_t sm0;
 428   u8 dont_translate = 0;
 429   u8 is_addr_only;
 430   u32 next0 = ~0;
 431   int err;
 432
 433   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 434   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 435   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 436
 437   err = icmp_get_key (ip0, &key0);
 438   if (err != -1)
 439     {
 440       b0->error = node->errors[err];
 441       next0 = SNAT_OUT2IN_NEXT_DROP;
 442       goto out2;
 443     }
 444   key0.fib_index = rx_fib_index0;
 445
 446   if (snat_static_mapping_match (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0))
 447     {
 448       /* Don't NAT packet aimed at the intfc address */
 449       if (is_interface_addr (sm, node, sw_if_index0, ip0->dst_address.as_u32))
 450         {
 451           dont_translate = 1;
 452           goto out;
 453         }
 454       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 455       next0 = SNAT_OUT2IN_NEXT_DROP;
 456       goto out;
 457     }
 458
 459   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_reply &&
 460                      (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
 461                      !icmp_is_error_message (icmp0)))
 462     {
 463       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 464       next0 = SNAT_OUT2IN_NEXT_DROP;
 465       goto out;
 466     }
 467
 468 out:
 469   *p_value = sm0;
 470 out2:
 471   *p_proto = key0.protocol;
 472   *p_dont_translate = dont_translate;
 473   return next0;
 474 }
 475
 476 u32
 477 icmp_out2in (snat_main_t * sm,
 478              vlib_buffer_t * b0,
 479              ip4_header_t * ip0,
 480              icmp46_header_t * icmp0,
 481              u32 sw_if_index0,
 482              u32 rx_fib_index0,
 483              vlib_node_runtime_t * node,
 484              u32 next0, u32 thread_index, void *d, void *e)
 485 {
 486   snat_session_key_t sm0;
 487   u8 protocol;
 488   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 489   ip4_header_t *inner_ip0 = 0;
 490   void *l4_header = 0;
 491   icmp46_header_t *inner_icmp0;
 492   u8 dont_translate;
 493   u32 new_addr0, old_addr0;
 494   u16 old_id0, new_id0;
 495   ip_csum_t sum0;
 496   u16 checksum0;
 497   u32 next0_tmp;
 498
 499   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 500
 501   next0_tmp = sm->icmp_match_out2in_cb (sm, node, thread_index, b0, ip0,
 502                                         &protocol, &sm0, &dont_translate, d,
 503                                         e);
 504   if (next0_tmp != ~0)
 505     next0 = next0_tmp;
 506   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 507     goto out;
 508
 509   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 510     {
 511       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 512                                              (u8 *)
 513                                              vlib_buffer_get_current (b0),
 514                                              ntohs (ip0->length) -
 515                                              ip4_header_bytes (ip0), 0);
 516       checksum0 = ~ip_csum_fold (sum0);
 517       if (checksum0 != 0 && checksum0 != 0xffff)
 518         {
 519           next0 = SNAT_OUT2IN_NEXT_DROP;
 520           goto out;
 521         }
 522     }
 523
 524   old_addr0 = ip0->dst_address.as_u32;
 525   new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
 526   vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 527
 528   sum0 = ip0->checksum;
 529   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 530                          dst_address /* changed member */ );
 531   ip0->checksum = ip_csum_fold (sum0);
 532
 533   if (icmp0->checksum == 0)
 534     icmp0->checksum = 0xffff;
 535
 536   if (!icmp_is_error_message (icmp0))
 537     {
 538       new_id0 = sm0.port;
 539       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 540         {
 541           old_id0 = echo0->identifier;
 542           new_id0 = sm0.port;
 543           echo0->identifier = new_id0;
 544
 545           sum0 = icmp0->checksum;
 546           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 547                                  identifier /* changed member */ );
 548           icmp0->checksum = ip_csum_fold (sum0);
 549         }
 550     }
 551   else
 552     {
 553       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 554       l4_header = ip4_next_header (inner_ip0);
 555
 556       if (!ip4_header_checksum_is_valid (inner_ip0))
 557         {
 558           next0 = SNAT_OUT2IN_NEXT_DROP;
 559           goto out;
 560         }
 561
 562       old_addr0 = inner_ip0->src_address.as_u32;
 563       inner_ip0->src_address = sm0.addr;
 564       new_addr0 = inner_ip0->src_address.as_u32;
 565
 566       sum0 = icmp0->checksum;
 567       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 568                              src_address /* changed member */ );
 569       icmp0->checksum = ip_csum_fold (sum0);
 570
 571       switch (protocol)
 572         {
 573         case SNAT_PROTOCOL_ICMP:
 574           inner_icmp0 = (icmp46_header_t *) l4_header;
 575           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 576
 577           old_id0 = inner_echo0->identifier;
 578           new_id0 = sm0.port;
 579           inner_echo0->identifier = new_id0;
 580
 581           sum0 = icmp0->checksum;
 582           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 583                                  identifier);
 584           icmp0->checksum = ip_csum_fold (sum0);
 585           break;
 586         case SNAT_PROTOCOL_UDP:
 587         case SNAT_PROTOCOL_TCP:
 588           old_id0 = ((tcp_udp_header_t *) l4_header)->src_port;
 589           new_id0 = sm0.port;
 590           ((tcp_udp_header_t *) l4_header)->src_port = new_id0;
 591
 592           sum0 = icmp0->checksum;
 593           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 594                                  src_port);
 595           icmp0->checksum = ip_csum_fold (sum0);
 596           break;
 597         default:
 598           ASSERT (0);
 599         }
 600     }
 601
 602 out:
 603   return next0;
 604 }
 605
 606
 607 static inline u32
 608 icmp_out2in_slow_path (snat_main_t * sm,
 609                        vlib_buffer_t * b0,
 610                        ip4_header_t * ip0,
 611                        icmp46_header_t * icmp0,
 612                        u32 sw_if_index0,
 613                        u32 rx_fib_index0,
 614                        vlib_node_runtime_t * node,
 615                        u32 next0, f64 now,
 616                        u32 thread_index, snat_session_t ** p_s0)
 617 {
 618   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 619                        next0, thread_index, p_s0, 0);
 620   snat_session_t *s0 = *p_s0;
 621   if (PREDICT_TRUE (next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 622     {
 623       /* Accounting */
 624       nat44_session_update_counters (s0, now,
 625                                      vlib_buffer_length_in_chain
 626                                      (sm->vlib_main, b0));
 627       /* Per-user LRU list maintenance */
 628       nat44_session_update_lru (sm, s0, thread_index);
 629     }
 630   return next0;
 631 }
 632
 633 static int
 634 nat_out2in_sm_unknown_proto (snat_main_t * sm,
 635                              vlib_buffer_t * b,
 636                              ip4_header_t * ip, u32 rx_fib_index)
 637 {
 638   clib_bihash_kv_8_8_t kv, value;
 639   snat_static_mapping_t *m;
 640   snat_session_key_t m_key;
 641   u32 old_addr, new_addr;
 642   ip_csum_t sum;
 643
 644   m_key.addr = ip->dst_address;
 645   m_key.port = 0;
 646   m_key.protocol = 0;
 647   m_key.fib_index = 0;
 648   kv.key = m_key.as_u64;
 649   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 650     return 1;
 651
 652   m = pool_elt_at_index (sm->static_mappings, value.value);
 653
 654   old_addr = ip->dst_address.as_u32;
 655   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 656   sum = ip->checksum;
 657   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 658   ip->checksum = ip_csum_fold (sum);
 659
 660   vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 661   return 0;
 662 }
 663
 664 static uword
 665 snat_out2in_node_fn (vlib_main_t * vm,
 666                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 667 {
 668   u32 n_left_from, *from, *to_next;
 669   snat_out2in_next_t next_index;
 670   u32 pkts_processed = 0;
 671   snat_main_t *sm = &snat_main;
 672   f64 now = vlib_time_now (vm);
 673   u32 thread_index = vm->thread_index;
 674
 675   from = vlib_frame_vector_args (frame);
 676   n_left_from = frame->n_vectors;
 677   next_index = node->cached_next_index;
 678
 679   while (n_left_from > 0)
 680     {
 681       u32 n_left_to_next;
 682
 683       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 684
 685       while (n_left_from >= 4 && n_left_to_next >= 2)
 686         {
 687           u32 bi0, bi1;
 688           vlib_buffer_t *b0, *b1;
 689           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 690           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 691           u32 sw_if_index0, sw_if_index1;
 692           ip4_header_t *ip0, *ip1;
 693           ip_csum_t sum0, sum1;
 694           u32 new_addr0, old_addr0;
 695           u16 new_port0, old_port0;
 696           u32 new_addr1, old_addr1;
 697           u16 new_port1, old_port1;
 698           udp_header_t *udp0, *udp1;
 699           tcp_header_t *tcp0, *tcp1;
 700           icmp46_header_t *icmp0, *icmp1;
 701           snat_session_key_t key0, key1, sm0, sm1;
 702           u32 rx_fib_index0, rx_fib_index1;
 703           u32 proto0, proto1;
 704           snat_session_t *s0 = 0, *s1 = 0;
 705           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 706
 707           /* Prefetch next iteration. */
 708           {
 709             vlib_buffer_t *p2, *p3;
 710
 711             p2 = vlib_get_buffer (vm, from[2]);
 712             p3 = vlib_get_buffer (vm, from[3]);
 713
 714             vlib_prefetch_buffer_header (p2, LOAD);
 715             vlib_prefetch_buffer_header (p3, LOAD);
 716
 717             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 718             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 719           }
 720
 721           /* speculatively enqueue b0 and b1 to the current next frame */
 722           to_next[0] = bi0 = from[0];
 723           to_next[1] = bi1 = from[1];
 724           from += 2;
 725           to_next += 2;
 726           n_left_from -= 2;
 727           n_left_to_next -= 2;
 728
 729           b0 = vlib_get_buffer (vm, bi0);
 730           b1 = vlib_get_buffer (vm, bi1);
 731
 732           vnet_buffer (b0)->snat.flags = 0;
 733           vnet_buffer (b1)->snat.flags = 0;
 734
 735           ip0 = vlib_buffer_get_current (b0);
 736           udp0 = ip4_next_header (ip0);
 737           tcp0 = (tcp_header_t *) udp0;
 738           icmp0 = (icmp46_header_t *) udp0;
 739
 740           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 741           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 742                                    sw_if_index0);
 743
 744           if (PREDICT_FALSE (ip0->ttl == 1))
 745             {
 746               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 747               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 748                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 749                                            0);
 750               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 751               goto trace0;
 752             }
 753
 754           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 755
 756           if (PREDICT_FALSE (proto0 == ~0))
 757             {
 758               if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
 759                 {
 760                   if (!sm->forwarding_enabled)
 761                     {
 762                       b0->error =
 763                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 764                       next0 = SNAT_OUT2IN_NEXT_DROP;
 765                     }
 766                 }
 767               goto trace0;
 768             }
 769
 770           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 771             {
 772               next0 = icmp_out2in_slow_path
 773                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 774                  next0, now, thread_index, &s0);
 775               goto trace0;
 776             }
 777
 778           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
 779             {
 780               next0 = SNAT_OUT2IN_NEXT_REASS;
 781               goto trace0;
 782             }
 783
 784           key0.addr = ip0->dst_address;
 785           key0.port = udp0->dst_port;
 786           key0.protocol = proto0;
 787           key0.fib_index = rx_fib_index0;
 788
 789           kv0.key = key0.as_u64;
 790
 791           if (clib_bihash_search_8_8
 792               (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 793             {
 794               /* Try to match static mapping by external address and port,
 795                  destination address and port in packet */
 796               if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
 797                 {
 798                   /*
 799                    * Send DHCP packets to the ipv4 stack, or we won't
 800                    * be able to use dhcp client on the outside interface
 801                    */
 802                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
 803                                      && (udp0->dst_port ==
 804                                          clib_host_to_net_u16
 805                                          (UDP_DST_PORT_dhcp_to_client))))
 806                     {
 807                       vnet_feature_next (&next0, b0);
 808                       goto trace0;
 809                     }
 810
 811                   if (!sm->forwarding_enabled)
 812                     {
 813                       b0->error =
 814                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 815                       next0 = SNAT_OUT2IN_NEXT_DROP;
 816                     }
 817                   goto trace0;
 818                 }
 819
 820               /* Create session initiated by host from external network */
 821               s0 = create_session_for_static_mapping (sm, b0, sm0, key0, node,
 822                                                       thread_index, now);
 823               if (!s0)
 824                 {
 825                   next0 = SNAT_OUT2IN_NEXT_DROP;
 826                   goto trace0;
 827                 }
 828             }
 829           else
 830             s0 =
 831               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 832                                  value0.value);
 833
 834           old_addr0 = ip0->dst_address.as_u32;
 835           ip0->dst_address = s0->in2out.addr;
 836           new_addr0 = ip0->dst_address.as_u32;
 837           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 838
 839           sum0 = ip0->checksum;
 840           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 841                                  ip4_header_t,
 842                                  dst_address /* changed member */ );
 843           ip0->checksum = ip_csum_fold (sum0);
 844
 845           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 846             {
 847               old_port0 = tcp0->dst_port;
 848               tcp0->dst_port = s0->in2out.port;
 849               new_port0 = tcp0->dst_port;
 850
 851               sum0 = tcp0->checksum;
 852               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 853                                      ip4_header_t,
 854                                      dst_address /* changed member */ );
 855
 856               sum0 = ip_csum_update (sum0, old_port0, new_port0,
 857                                      ip4_header_t /* cheat */ ,
 858                                      length /* changed member */ );
 859               tcp0->checksum = ip_csum_fold (sum0);
 860             }
 861           else
 862             {
 863               old_port0 = udp0->dst_port;
 864               udp0->dst_port = s0->in2out.port;
 865               udp0->checksum = 0;
 866             }
 867
 868           /* Accounting */
 869           nat44_session_update_counters (s0, now,
 870                                          vlib_buffer_length_in_chain (vm,
 871                                                                       b0));
 872           /* Per-user LRU list maintenance */
 873           nat44_session_update_lru (sm, s0, thread_index);
 874         trace0:
 875
 876           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 877                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 878             {
 879               snat_out2in_trace_t *t =
 880                 vlib_add_trace (vm, node, b0, sizeof (*t));
 881               t->sw_if_index = sw_if_index0;
 882               t->next_index = next0;
 883               t->session_index = ~0;
 884               if (s0)
 885                 t->session_index =
 886                   s0 - sm->per_thread_data[thread_index].sessions;
 887             }
 888
 889           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
 890
 891
 892           ip1 = vlib_buffer_get_current (b1);
 893           udp1 = ip4_next_header (ip1);
 894           tcp1 = (tcp_header_t *) udp1;
 895           icmp1 = (icmp46_header_t *) udp1;
 896
 897           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 898           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 899                                    sw_if_index1);
 900
 901           if (PREDICT_FALSE (ip1->ttl == 1))
 902             {
 903               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 904               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 905                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 906                                            0);
 907               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 908               goto trace1;
 909             }
 910
 911           proto1 = ip_proto_to_snat_proto (ip1->protocol);
 912
 913           if (PREDICT_FALSE (proto1 == ~0))
 914             {
 915               if (nat_out2in_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
 916                 {
 917                   if (!sm->forwarding_enabled)
 918                     {
 919                       b1->error =
 920                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 921                       next1 = SNAT_OUT2IN_NEXT_DROP;
 922                     }
 923                 }
 924               goto trace1;
 925             }
 926
 927           if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
 928             {
 929               next1 = icmp_out2in_slow_path
 930                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
 931                  next1, now, thread_index, &s1);
 932               goto trace1;
 933             }
 934
 935           if (PREDICT_FALSE (ip4_is_fragment (ip1)))
 936             {
 937               next1 = SNAT_OUT2IN_NEXT_REASS;
 938               goto trace1;
 939             }
 940
 941           key1.addr = ip1->dst_address;
 942           key1.port = udp1->dst_port;
 943           key1.protocol = proto1;
 944           key1.fib_index = rx_fib_index1;
 945
 946           kv1.key = key1.as_u64;
 947
 948           if (clib_bihash_search_8_8
 949               (&sm->per_thread_data[thread_index].out2in, &kv1, &value1))
 950             {
 951               /* Try to match static mapping by external address and port,
 952                  destination address and port in packet */
 953               if (snat_static_mapping_match (sm, key1, &sm1, 1, 0, 0, 0, 0))
 954                 {
 955                   /*
 956                    * Send DHCP packets to the ipv4 stack, or we won't
 957                    * be able to use dhcp client on the outside interface
 958                    */
 959                   if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
 960                                      && (udp1->dst_port ==
 961                                          clib_host_to_net_u16
 962                                          (UDP_DST_PORT_dhcp_to_client))))
 963                     {
 964                       vnet_feature_next (&next1, b1);
 965                       goto trace1;
 966                     }
 967
 968                   if (!sm->forwarding_enabled)
 969                     {
 970                       b1->error =
 971                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 972                       next1 = SNAT_OUT2IN_NEXT_DROP;
 973                     }
 974                   goto trace1;
 975                 }
 976
 977               /* Create session initiated by host from external network */
 978               s1 = create_session_for_static_mapping (sm, b1, sm1, key1, node,
 979                                                       thread_index, now);
 980               if (!s1)
 981                 {
 982                   next1 = SNAT_OUT2IN_NEXT_DROP;
 983                   goto trace1;
 984                 }
 985             }
 986           else
 987             s1 =
 988               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 989                                  value1.value);
 990
 991           old_addr1 = ip1->dst_address.as_u32;
 992           ip1->dst_address = s1->in2out.addr;
 993           new_addr1 = ip1->dst_address.as_u32;
 994           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
 995
 996           sum1 = ip1->checksum;
 997           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
 998                                  ip4_header_t,
 999                                  dst_address /* changed member */ );
1000           ip1->checksum = ip_csum_fold (sum1);
1001
1002           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1003             {
1004               old_port1 = tcp1->dst_port;
1005               tcp1->dst_port = s1->in2out.port;
1006               new_port1 = tcp1->dst_port;
1007
1008               sum1 = tcp1->checksum;
1009               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1010                                      ip4_header_t,
1011                                      dst_address /* changed member */ );
1012
1013               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1014                                      ip4_header_t /* cheat */ ,
1015                                      length /* changed member */ );
1016               tcp1->checksum = ip_csum_fold (sum1);
1017             }
1018           else
1019             {
1020               old_port1 = udp1->dst_port;
1021               udp1->dst_port = s1->in2out.port;
1022               udp1->checksum = 0;
1023             }
1024
1025           /* Accounting */
1026           nat44_session_update_counters (s1, now,
1027                                          vlib_buffer_length_in_chain (vm,
1028                                                                       b1));
1029           /* Per-user LRU list maintenance */
1030           nat44_session_update_lru (sm, s1, thread_index);
1031         trace1:
1032
1033           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1034                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1035             {
1036               snat_out2in_trace_t *t =
1037                 vlib_add_trace (vm, node, b1, sizeof (*t));
1038               t->sw_if_index = sw_if_index1;
1039               t->next_index = next1;
1040               t->session_index = ~0;
1041               if (s1)
1042                 t->session_index =
1043                   s1 - sm->per_thread_data[thread_index].sessions;
1044             }
1045
1046           pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1047
1048           /* verify speculative enqueues, maybe switch current next frame */
1049           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1050                                            to_next, n_left_to_next,
1051                                            bi0, bi1, next0, next1);
1052         }
1053
1054       while (n_left_from > 0 && n_left_to_next > 0)
1055         {
1056           u32 bi0;
1057           vlib_buffer_t *b0;
1058           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1059           u32 sw_if_index0;
1060           ip4_header_t *ip0;
1061           ip_csum_t sum0;
1062           u32 new_addr0, old_addr0;
1063           u16 new_port0, old_port0;
1064           udp_header_t *udp0;
1065           tcp_header_t *tcp0;
1066           icmp46_header_t *icmp0;
1067           snat_session_key_t key0, sm0;
1068           u32 rx_fib_index0;
1069           u32 proto0;
1070           snat_session_t *s0 = 0;
1071           clib_bihash_kv_8_8_t kv0, value0;
1072
1073           /* speculatively enqueue b0 to the current next frame */
1074           bi0 = from[0];
1075           to_next[0] = bi0;
1076           from += 1;
1077           to_next += 1;
1078           n_left_from -= 1;
1079           n_left_to_next -= 1;
1080
1081           b0 = vlib_get_buffer (vm, bi0);
1082
1083           vnet_buffer (b0)->snat.flags = 0;
1084
1085           ip0 = vlib_buffer_get_current (b0);
1086           udp0 = ip4_next_header (ip0);
1087           tcp0 = (tcp_header_t *) udp0;
1088           icmp0 = (icmp46_header_t *) udp0;
1089
1090           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1091           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1092                                    sw_if_index0);
1093
1094           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1095
1096           if (PREDICT_FALSE (proto0 == ~0))
1097             {
1098               if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1099                 {
1100                   if (!sm->forwarding_enabled)
1101                     {
1102                       b0->error =
1103                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1104                       next0 = SNAT_OUT2IN_NEXT_DROP;
1105                     }
1106                 }
1107               goto trace00;
1108             }
1109
1110           if (PREDICT_FALSE (ip0->ttl == 1))
1111             {
1112               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1113               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1114                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1115                                            0);
1116               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1117               goto trace00;
1118             }
1119
1120           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1121             {
1122               next0 = icmp_out2in_slow_path
1123                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1124                  next0, now, thread_index, &s0);
1125               goto trace00;
1126             }
1127
1128           if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1129             {
1130               next0 = SNAT_OUT2IN_NEXT_REASS;
1131               goto trace00;
1132             }
1133
1134           key0.addr = ip0->dst_address;
1135           key0.port = udp0->dst_port;
1136           key0.protocol = proto0;
1137           key0.fib_index = rx_fib_index0;
1138
1139           kv0.key = key0.as_u64;
1140
1141           if (clib_bihash_search_8_8
1142               (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
1143             {
1144               /* Try to match static mapping by external address and port,
1145                  destination address and port in packet */
1146               if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
1147                 {
1148                   /*
1149                    * Send DHCP packets to the ipv4 stack, or we won't
1150                    * be able to use dhcp client on the outside interface
1151                    */
1152                   if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1153                                      && (udp0->dst_port ==
1154                                          clib_host_to_net_u16
1155                                          (UDP_DST_PORT_dhcp_to_client))))
1156                     {
1157                       vnet_feature_next (&next0, b0);
1158                       goto trace00;
1159                     }
1160
1161                   if (!sm->forwarding_enabled)
1162                     {
1163                       b0->error =
1164                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1165                       next0 = SNAT_OUT2IN_NEXT_DROP;
1166                     }
1167                   goto trace00;
1168                 }
1169
1170               /* Create session initiated by host from external network */
1171               s0 = create_session_for_static_mapping (sm, b0, sm0, key0, node,
1172                                                       thread_index, now);
1173               if (!s0)
1174                 {
1175                   next0 = SNAT_OUT2IN_NEXT_DROP;
1176                   goto trace00;
1177                 }
1178             }
1179           else
1180             s0 =
1181               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1182                                  value0.value);
1183
1184           old_addr0 = ip0->dst_address.as_u32;
1185           ip0->dst_address = s0->in2out.addr;
1186           new_addr0 = ip0->dst_address.as_u32;
1187           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1188
1189           sum0 = ip0->checksum;
1190           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1191                                  ip4_header_t,
1192                                  dst_address /* changed member */ );
1193           ip0->checksum = ip_csum_fold (sum0);
1194
1195           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1196             {
1197               old_port0 = tcp0->dst_port;
1198               tcp0->dst_port = s0->in2out.port;
1199               new_port0 = tcp0->dst_port;
1200
1201               sum0 = tcp0->checksum;
1202               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1203                                      ip4_header_t,
1204                                      dst_address /* changed member */ );
1205
1206               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1207                                      ip4_header_t /* cheat */ ,
1208                                      length /* changed member */ );
1209               tcp0->checksum = ip_csum_fold (sum0);
1210             }
1211           else
1212             {
1213               old_port0 = udp0->dst_port;
1214               udp0->dst_port = s0->in2out.port;
1215               udp0->checksum = 0;
1216             }
1217
1218           /* Accounting */
1219           nat44_session_update_counters (s0, now,
1220                                          vlib_buffer_length_in_chain (vm,
1221                                                                       b0));
1222           /* Per-user LRU list maintenance */
1223           nat44_session_update_lru (sm, s0, thread_index);
1224         trace00:
1225
1226           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1227                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1228             {
1229               snat_out2in_trace_t *t =
1230                 vlib_add_trace (vm, node, b0, sizeof (*t));
1231               t->sw_if_index = sw_if_index0;
1232               t->next_index = next0;
1233               t->session_index = ~0;
1234               if (s0)
1235                 t->session_index =
1236                   s0 - sm->per_thread_data[thread_index].sessions;
1237             }
1238
1239           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1240
1241           /* verify speculative enqueue, maybe switch current next frame */
1242           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1243                                            to_next, n_left_to_next,
1244                                            bi0, next0);
1245         }
1246
1247       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1248     }
1249
1250   vlib_node_increment_counter (vm, snat_out2in_node.index,
1251                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1252                                pkts_processed);
1253   return frame->n_vectors;
1254 }
1255
1256 /* *INDENT-OFF* */
1257 VLIB_REGISTER_NODE (snat_out2in_node) = {
1258   .function = snat_out2in_node_fn,
1259   .name = "nat44-out2in",
1260   .vector_size = sizeof (u32),
1261   .format_trace = format_snat_out2in_trace,
1262   .type = VLIB_NODE_TYPE_INTERNAL,
1263
1264   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1265   .error_strings = snat_out2in_error_strings,
1266
1267   .runtime_data_bytes = sizeof (snat_runtime_t),
1268
1269   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1270
1271   /* edit / add dispositions here */
1272   .next_nodes = {
1273     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1274     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1275     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1276     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1277   },
1278 };
1279 /* *INDENT-ON* */
1280
1281 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1282
1283 static uword
1284 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1285                             vlib_node_runtime_t * node, vlib_frame_t * frame)
1286 {
1287   u32 n_left_from, *from, *to_next;
1288   snat_out2in_next_t next_index;
1289   u32 pkts_processed = 0;
1290   snat_main_t *sm = &snat_main;
1291   f64 now = vlib_time_now (vm);
1292   u32 thread_index = vm->thread_index;
1293   snat_main_per_thread_data_t *per_thread_data =
1294     &sm->per_thread_data[thread_index];
1295   u32 *fragments_to_drop = 0;
1296   u32 *fragments_to_loopback = 0;
1297
1298   from = vlib_frame_vector_args (frame);
1299   n_left_from = frame->n_vectors;
1300   next_index = node->cached_next_index;
1301
1302   while (n_left_from > 0)
1303     {
1304       u32 n_left_to_next;
1305
1306       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1307
1308       while (n_left_from > 0 && n_left_to_next > 0)
1309         {
1310           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1311           vlib_buffer_t *b0;
1312           u32 next0;
1313           u8 cached0 = 0;
1314           ip4_header_t *ip0;
1315           nat_reass_ip4_t *reass0;
1316           udp_header_t *udp0;
1317           tcp_header_t *tcp0;
1318           snat_session_key_t key0, sm0;
1319           clib_bihash_kv_8_8_t kv0, value0;
1320           snat_session_t *s0 = 0;
1321           u16 old_port0, new_port0;
1322           ip_csum_t sum0;
1323
1324           /* speculatively enqueue b0 to the current next frame */
1325           bi0 = from[0];
1326           to_next[0] = bi0;
1327           from += 1;
1328           to_next += 1;
1329           n_left_from -= 1;
1330           n_left_to_next -= 1;
1331
1332           b0 = vlib_get_buffer (vm, bi0);
1333           next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1334
1335           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1336           rx_fib_index0 =
1337             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1338                                                  sw_if_index0);
1339
1340           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1341             {
1342               next0 = SNAT_OUT2IN_NEXT_DROP;
1343               b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1344               goto trace0;
1345             }
1346
1347           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1348           udp0 = ip4_next_header (ip0);
1349           tcp0 = (tcp_header_t *) udp0;
1350           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1351
1352           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1353                                                  ip0->dst_address,
1354                                                  ip0->fragment_id,
1355                                                  ip0->protocol,
1356                                                  1, &fragments_to_drop);
1357
1358           if (PREDICT_FALSE (!reass0))
1359             {
1360               next0 = SNAT_OUT2IN_NEXT_DROP;
1361               b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1362               nat_log_notice ("maximum reassemblies exceeded");
1363               goto trace0;
1364             }
1365
1366           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1367             {
1368               key0.addr = ip0->dst_address;
1369               key0.port = udp0->dst_port;
1370               key0.protocol = proto0;
1371               key0.fib_index = rx_fib_index0;
1372               kv0.key = key0.as_u64;
1373
1374               if (clib_bihash_search_8_8
1375                   (&per_thread_data->out2in, &kv0, &value0))
1376                 {
1377                   /* Try to match static mapping by external address and port,
1378                      destination address and port in packet */
1379                   if (snat_static_mapping_match
1380                       (sm, key0, &sm0, 1, 0, 0, 0, 0))
1381                     {
1382                       /*
1383                        * Send DHCP packets to the ipv4 stack, or we won't
1384                        * be able to use dhcp client on the outside interface
1385                        */
1386                       if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1387                                          && (udp0->dst_port
1388                                              ==
1389                                              clib_host_to_net_u16
1390                                              (UDP_DST_PORT_dhcp_to_client))))
1391                         {
1392                           vnet_feature_next (&next0, b0);
1393                           goto trace0;
1394                         }
1395
1396                       if (!sm->forwarding_enabled)
1397                         {
1398                           b0->error =
1399                             node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1400                           next0 = SNAT_OUT2IN_NEXT_DROP;
1401                         }
1402                       goto trace0;
1403                     }
1404
1405                   /* Create session initiated by host from external network */
1406                   s0 =
1407                     create_session_for_static_mapping (sm, b0, sm0, key0,
1408                                                        node, thread_index,
1409                                                        now);
1410                   if (!s0)
1411                     {
1412                       b0->error =
1413                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1414                       next0 = SNAT_OUT2IN_NEXT_DROP;
1415                       goto trace0;
1416                     }
1417                   reass0->sess_index = s0 - per_thread_data->sessions;
1418                   reass0->thread_index = thread_index;
1419                 }
1420               else
1421                 {
1422                   s0 = pool_elt_at_index (per_thread_data->sessions,
1423                                           value0.value);
1424                   reass0->sess_index = value0.value;
1425                 }
1426               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1427             }
1428           else
1429             {
1430               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1431                 {
1432                   if (nat_ip4_reass_add_fragment
1433                       (reass0, bi0, &fragments_to_drop))
1434                     {
1435                       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1436                       nat_log_notice
1437                         ("maximum fragments per reassembly exceeded");
1438                       next0 = SNAT_OUT2IN_NEXT_DROP;
1439                       goto trace0;
1440                     }
1441                   cached0 = 1;
1442                   goto trace0;
1443                 }
1444               s0 = pool_elt_at_index (per_thread_data->sessions,
1445                                       reass0->sess_index);
1446             }
1447
1448           old_addr0 = ip0->dst_address.as_u32;
1449           ip0->dst_address = s0->in2out.addr;
1450           new_addr0 = ip0->dst_address.as_u32;
1451           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1452
1453           sum0 = ip0->checksum;
1454           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1455                                  ip4_header_t,
1456                                  dst_address /* changed member */ );
1457           ip0->checksum = ip_csum_fold (sum0);
1458
1459           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1460             {
1461               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1462                 {
1463                   old_port0 = tcp0->dst_port;
1464                   tcp0->dst_port = s0->in2out.port;
1465                   new_port0 = tcp0->dst_port;
1466
1467                   sum0 = tcp0->checksum;
1468                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1469                                          ip4_header_t,
1470                                          dst_address /* changed member */ );
1471
1472                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1473                                          ip4_header_t /* cheat */ ,
1474                                          length /* changed member */ );
1475                   tcp0->checksum = ip_csum_fold (sum0);
1476                 }
1477               else
1478                 {
1479                   old_port0 = udp0->dst_port;
1480                   udp0->dst_port = s0->in2out.port;
1481                   udp0->checksum = 0;
1482                 }
1483             }
1484
1485           /* Accounting */
1486           nat44_session_update_counters (s0, now,
1487                                          vlib_buffer_length_in_chain (vm,
1488                                                                       b0));
1489           /* Per-user LRU list maintenance */
1490           nat44_session_update_lru (sm, s0, thread_index);
1491
1492         trace0:
1493           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1494                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1495             {
1496               nat44_reass_trace_t *t =
1497                 vlib_add_trace (vm, node, b0, sizeof (*t));
1498               t->cached = cached0;
1499               t->sw_if_index = sw_if_index0;
1500               t->next_index = next0;
1501             }
1502
1503           if (cached0)
1504             {
1505               n_left_to_next++;
1506               to_next--;
1507             }
1508           else
1509             {
1510               pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1511
1512               /* verify speculative enqueue, maybe switch current next frame */
1513               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1514                                                to_next, n_left_to_next,
1515                                                bi0, next0);
1516             }
1517
1518           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1519             {
1520               from = vlib_frame_vector_args (frame);
1521               u32 len = vec_len (fragments_to_loopback);
1522               if (len <= VLIB_FRAME_SIZE)
1523                 {
1524                   clib_memcpy (from, fragments_to_loopback,
1525                                sizeof (u32) * len);
1526                   n_left_from = len;
1527                   vec_reset_length (fragments_to_loopback);
1528                 }
1529               else
1530                 {
1531                   clib_memcpy (from,
1532                                fragments_to_loopback + (len -
1533                                                         VLIB_FRAME_SIZE),
1534                                sizeof (u32) * VLIB_FRAME_SIZE);
1535                   n_left_from = VLIB_FRAME_SIZE;
1536                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1537                 }
1538             }
1539         }
1540
1541       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1542     }
1543
1544   vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1545                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1546                                pkts_processed);
1547
1548   nat_send_all_to_node (vm, fragments_to_drop, node,
1549                         &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1550                         SNAT_OUT2IN_NEXT_DROP);
1551
1552   vec_free (fragments_to_drop);
1553   vec_free (fragments_to_loopback);
1554   return frame->n_vectors;
1555 }
1556
1557 /* *INDENT-OFF* */
1558 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1559   .function = nat44_out2in_reass_node_fn,
1560   .name = "nat44-out2in-reass",
1561   .vector_size = sizeof (u32),
1562   .format_trace = format_nat44_reass_trace,
1563   .type = VLIB_NODE_TYPE_INTERNAL,
1564
1565   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1566   .error_strings = snat_out2in_error_strings,
1567
1568   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1569
1570   /* edit / add dispositions here */
1571   .next_nodes = {
1572     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1573     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1574     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1575     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1576   },
1577 };
1578 /* *INDENT-ON* */
1579
1580 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1581                               nat44_out2in_reass_node_fn);
1582
1583 static uword
1584 snat_out2in_fast_node_fn (vlib_main_t * vm,
1585                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1586 {
1587   u32 n_left_from, *from, *to_next;
1588   snat_out2in_next_t next_index;
1589   u32 pkts_processed = 0;
1590   snat_main_t *sm = &snat_main;
1591
1592   from = vlib_frame_vector_args (frame);
1593   n_left_from = frame->n_vectors;
1594   next_index = node->cached_next_index;
1595
1596   while (n_left_from > 0)
1597     {
1598       u32 n_left_to_next;
1599
1600       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1601
1602       while (n_left_from > 0 && n_left_to_next > 0)
1603         {
1604           u32 bi0;
1605           vlib_buffer_t *b0;
1606           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1607           u32 sw_if_index0;
1608           ip4_header_t *ip0;
1609           ip_csum_t sum0;
1610           u32 new_addr0, old_addr0;
1611           u16 new_port0, old_port0;
1612           udp_header_t *udp0;
1613           tcp_header_t *tcp0;
1614           icmp46_header_t *icmp0;
1615           snat_session_key_t key0, sm0;
1616           u32 proto0;
1617           u32 rx_fib_index0;
1618
1619           /* speculatively enqueue b0 to the current next frame */
1620           bi0 = from[0];
1621           to_next[0] = bi0;
1622           from += 1;
1623           to_next += 1;
1624           n_left_from -= 1;
1625           n_left_to_next -= 1;
1626
1627           b0 = vlib_get_buffer (vm, bi0);
1628
1629           ip0 = vlib_buffer_get_current (b0);
1630           udp0 = ip4_next_header (ip0);
1631           tcp0 = (tcp_header_t *) udp0;
1632           icmp0 = (icmp46_header_t *) udp0;
1633
1634           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1635           rx_fib_index0 =
1636             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1637
1638           vnet_feature_next (&next0, b0);
1639
1640           if (PREDICT_FALSE (ip0->ttl == 1))
1641             {
1642               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1643               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1644                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1645                                            0);
1646               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1647               goto trace00;
1648             }
1649
1650           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1651
1652           if (PREDICT_FALSE (proto0 == ~0))
1653             goto trace00;
1654
1655           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1656             {
1657               next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0,
1658                                    rx_fib_index0, node, next0, ~0, 0, 0);
1659               goto trace00;
1660             }
1661
1662           key0.addr = ip0->dst_address;
1663           key0.port = udp0->dst_port;
1664           key0.fib_index = rx_fib_index0;
1665
1666           if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
1667             {
1668               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1669               goto trace00;
1670             }
1671
1672           new_addr0 = sm0.addr.as_u32;
1673           new_port0 = sm0.port;
1674           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1675           old_addr0 = ip0->dst_address.as_u32;
1676           ip0->dst_address.as_u32 = new_addr0;
1677
1678           sum0 = ip0->checksum;
1679           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1680                                  ip4_header_t,
1681                                  dst_address /* changed member */ );
1682           ip0->checksum = ip_csum_fold (sum0);
1683
1684           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1685             {
1686               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1687                 {
1688                   old_port0 = tcp0->dst_port;
1689                   tcp0->dst_port = new_port0;
1690
1691                   sum0 = tcp0->checksum;
1692                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1693                                          ip4_header_t,
1694                                          dst_address /* changed member */ );
1695
1696                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1697                                          ip4_header_t /* cheat */ ,
1698                                          length /* changed member */ );
1699                   tcp0->checksum = ip_csum_fold (sum0);
1700                 }
1701               else
1702                 {
1703                   old_port0 = udp0->dst_port;
1704                   udp0->dst_port = new_port0;
1705                   udp0->checksum = 0;
1706                 }
1707             }
1708           else
1709             {
1710               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1711                 {
1712                   sum0 = tcp0->checksum;
1713                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1714                                          ip4_header_t,
1715                                          dst_address /* changed member */ );
1716
1717                   tcp0->checksum = ip_csum_fold (sum0);
1718                 }
1719             }
1720
1721         trace00:
1722
1723           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1724                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1725             {
1726               snat_out2in_trace_t *t =
1727                 vlib_add_trace (vm, node, b0, sizeof (*t));
1728               t->sw_if_index = sw_if_index0;
1729               t->next_index = next0;
1730             }
1731
1732           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1733
1734           /* verify speculative enqueue, maybe switch current next frame */
1735           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1736                                            to_next, n_left_to_next,
1737                                            bi0, next0);
1738         }
1739
1740       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1741     }
1742
1743   vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
1744                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1745                                pkts_processed);
1746   return frame->n_vectors;
1747 }
1748
1749 /* *INDENT-OFF* */
1750 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1751   .function = snat_out2in_fast_node_fn,
1752   .name = "nat44-out2in-fast",
1753   .vector_size = sizeof (u32),
1754   .format_trace = format_snat_out2in_fast_trace,
1755   .type = VLIB_NODE_TYPE_INTERNAL,
1756
1757   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1758   .error_strings = snat_out2in_error_strings,
1759
1760   .runtime_data_bytes = sizeof (snat_runtime_t),
1761
1762   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1763
1764   /* edit / add dispositions here */
1765   .next_nodes = {
1766     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1767     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1768     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1769     [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1770   },
1771 };
1772 /* *INDENT-ON* */
1773
1774 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node,
1775                               snat_out2in_fast_node_fn);
1776
1777 /*
1778  * fd.io coding-style-patch-verification: ON
1779  *
1780  * Local Variables:
1781  * eval: (c-set-style "gnu")
1782  * End:
1783  */