src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #include <vppinfra/hash.h>
  33 #include <vppinfra/error.h>
  34 #include <vppinfra/elog.h>
  35
  36 typedef struct
  37 {
  38   u32 sw_if_index;
  39   u32 next_index;
  40   u32 session_index;
  41   u32 is_slow_path;
  42 } snat_in2out_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 *
  46 format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  51   char *tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 *
  62 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 vlib_node_registration_t snat_in2out_node;
  75 vlib_node_registration_t snat_in2out_slowpath_node;
  76 vlib_node_registration_t snat_in2out_fast_node;
  77 vlib_node_registration_t snat_in2out_output_node;
  78 vlib_node_registration_t snat_in2out_output_slowpath_node;
  79 vlib_node_registration_t nat44_in2out_reass_node;
  80
  81 #define foreach_snat_in2out_error                       \
  82 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  83 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  84 _(OUT_OF_PORTS, "Out of ports")                         \
  85 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  86 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  87 _(NO_TRANSLATION, "No translation")                     \
  88 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  89 _(DROP_FRAGMENT, "Drop fragment")                       \
  90 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  91 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  92
  93 typedef enum
  94 {
  95 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  96   foreach_snat_in2out_error
  97 #undef _
  98     SNAT_IN2OUT_N_ERROR,
  99 } snat_in2out_error_t;
 100
 101 static char *snat_in2out_error_strings[] = {
 102 #define _(sym,string) string,
 103   foreach_snat_in2out_error
 104 #undef _
 105 };
 106
 107 typedef enum
 108 {
 109   SNAT_IN2OUT_NEXT_LOOKUP,
 110   SNAT_IN2OUT_NEXT_DROP,
 111   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 112   SNAT_IN2OUT_NEXT_SLOW_PATH,
 113   SNAT_IN2OUT_NEXT_REASS,
 114   SNAT_IN2OUT_N_NEXT,
 115 } snat_in2out_next_t;
 116
 117 static inline int
 118 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 119                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 120                     u32 rx_fib_index0, u32 thread_index)
 121 {
 122   udp_header_t *udp0 = ip4_next_header (ip0);
 123   snat_session_key_t key0, sm0;
 124   clib_bihash_kv_8_8_t kv0, value0;
 125
 126   key0.addr = ip0->dst_address;
 127   key0.port = udp0->dst_port;
 128   key0.protocol = proto0;
 129   key0.fib_index = sm->outside_fib_index;
 130   kv0.key = key0.as_u64;
 131
 132   /* NAT packet aimed at external address if */
 133   /* has active sessions */
 134   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 135                               &value0))
 136     {
 137       /* or is static mappings */
 138       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 139         return 0;
 140     }
 141   else
 142     return 0;
 143
 144   if (sm->forwarding_enabled)
 145     return 1;
 146
 147   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 148                                   rx_fib_index0);
 149 }
 150
 151 static inline int
 152 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 153                                   u32 proto0, u16 src_port, u16 dst_port,
 154                                   u32 thread_index, u32 sw_if_index)
 155 {
 156   snat_session_key_t key0;
 157   clib_bihash_kv_8_8_t kv0, value0;
 158   snat_interface_t *i;
 159
 160   /* src NAT check */
 161   key0.addr = ip0->src_address;
 162   key0.port = src_port;
 163   key0.protocol = proto0;
 164   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 165   kv0.key = key0.as_u64;
 166
 167   if (!clib_bihash_search_8_8
 168       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 169     return 1;
 170
 171   /* dst NAT check */
 172   key0.addr = ip0->dst_address;
 173   key0.port = dst_port;
 174   key0.protocol = proto0;
 175   kv0.key = key0.as_u64;
 176   if (!clib_bihash_search_8_8
 177       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 178     {
 179       /* hairpinning */
 180     /* *INDENT-OFF* */
 181     pool_foreach (i, sm->output_feature_interfaces,
 182     ({
 183       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 184         return 0;
 185     }));
 186     /* *INDENT-ON* */
 187       return 1;
 188     }
 189
 190   return 0;
 191 }
 192
 193 int
 194 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 195 {
 196   snat_main_t *sm = &snat_main;
 197   nat44_is_idle_session_ctx_t *ctx = arg;
 198   snat_session_t *s;
 199   u64 sess_timeout_time;
 200   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 201                                                        ctx->thread_index);
 202   clib_bihash_kv_8_8_t s_kv;
 203
 204   s = pool_elt_at_index (tsm->sessions, kv->value);
 205   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 206   if (ctx->now >= sess_timeout_time)
 207     {
 208       s_kv.key = s->out2in.as_u64;
 209       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 210         nat_log_warn ("out2in key del failed");
 211
 212       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 213                                            s->out2in.addr.as_u32,
 214                                            s->in2out.protocol,
 215                                            s->in2out.port,
 216                                            s->out2in.port,
 217                                            s->in2out.fib_index);
 218
 219       if (!snat_is_session_static (s))
 220         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 221                                             &s->out2in);
 222
 223       nat44_delete_session (sm, s, ctx->thread_index);
 224       return 1;
 225     }
 226
 227   return 0;
 228 }
 229
 230 static u32
 231 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 232            ip4_header_t * ip0,
 233            u32 rx_fib_index0,
 234            snat_session_key_t * key0,
 235            snat_session_t ** sessionp,
 236            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 237 {
 238   snat_user_t *u;
 239   snat_session_t *s = 0;
 240   clib_bihash_kv_8_8_t kv0;
 241   snat_session_key_t key1;
 242   udp_header_t *udp0 = ip4_next_header (ip0);
 243   u8 is_sm = 0;
 244   nat_outside_fib_t *outside_fib;
 245   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 246   u8 identity_nat;
 247   fib_prefix_t pfx = {
 248     .fp_proto = FIB_PROTOCOL_IP4,
 249     .fp_len = 32,
 250     .fp_addr = {
 251                 .ip4.as_u32 = ip0->dst_address.as_u32,
 252                 },
 253   };
 254   nat44_is_idle_session_ctx_t ctx0;
 255
 256   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 257     {
 258       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 259       nat_ipfix_logging_max_sessions (sm->max_translations);
 260       nat_log_notice ("maximum sessions exceeded");
 261       return SNAT_IN2OUT_NEXT_DROP;
 262     }
 263
 264   key1.protocol = key0->protocol;
 265
 266   /* First try to match static mapping by local address and port */
 267   if (snat_static_mapping_match
 268       (sm, *key0, &key1, 0, 0, 0, 0, 0, &identity_nat))
 269     {
 270       /* Try to create dynamic translation */
 271       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 272                                                thread_index, &key1,
 273                                                sm->port_per_thread,
 274                                                sm->per_thread_data
 275                                                [thread_index].snat_thread_index))
 276         {
 277           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 278           return SNAT_IN2OUT_NEXT_DROP;
 279         }
 280     }
 281   else
 282     {
 283       if (PREDICT_FALSE (identity_nat))
 284         {
 285           *sessionp = s;
 286           return next0;
 287         }
 288
 289       is_sm = 1;
 290     }
 291
 292   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 293                               thread_index);
 294   if (!u)
 295     {
 296       nat_log_warn ("create NAT user failed");
 297       return SNAT_IN2OUT_NEXT_DROP;
 298     }
 299
 300   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 301   if (!s)
 302     {
 303       nat44_delete_user_with_no_session (sm, u, thread_index);
 304       nat_log_warn ("create NAT session failed");
 305       return SNAT_IN2OUT_NEXT_DROP;
 306     }
 307
 308   if (is_sm)
 309     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 310   user_session_increment (sm, u, is_sm);
 311   s->in2out = *key0;
 312   s->out2in = key1;
 313   s->out2in.protocol = key0->protocol;
 314   s->out2in.fib_index = sm->outside_fib_index;
 315   switch (vec_len (sm->outside_fibs))
 316     {
 317     case 0:
 318       s->out2in.fib_index = sm->outside_fib_index;
 319       break;
 320     case 1:
 321       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 322       break;
 323     default:
 324       /* *INDENT-OFF* */
 325       vec_foreach (outside_fib, sm->outside_fibs)
 326         {
 327           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 328           if (FIB_NODE_INDEX_INVALID != fei)
 329             {
 330               if (fib_entry_get_resolving_interface (fei) != ~0)
 331                 {
 332                   s->out2in.fib_index = outside_fib->fib_index;
 333                   break;
 334                 }
 335             }
 336         }
 337       /* *INDENT-ON* */
 338       break;
 339     }
 340   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 341   s->ext_host_port = udp0->dst_port;
 342   *sessionp = s;
 343
 344   /* Add to translation hashes */
 345   ctx0.now = now;
 346   ctx0.thread_index = thread_index;
 347   kv0.key = s->in2out.as_u64;
 348   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 349   if (clib_bihash_add_or_overwrite_stale_8_8
 350       (&sm->per_thread_data[thread_index].in2out, &kv0,
 351        nat44_i2o_is_idle_session_cb, &ctx0))
 352     nat_log_notice ("in2out key add failed");
 353
 354   kv0.key = s->out2in.as_u64;
 355   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 356
 357   if (clib_bihash_add_or_overwrite_stale_8_8
 358       (&sm->per_thread_data[thread_index].out2in, &kv0,
 359        nat44_o2i_is_idle_session_cb, &ctx0))
 360     nat_log_notice ("out2in key add failed");
 361
 362   /* log NAT event */
 363   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 364                                        s->out2in.addr.as_u32,
 365                                        s->in2out.protocol,
 366                                        s->in2out.port,
 367                                        s->out2in.port, s->in2out.fib_index);
 368   return next0;
 369 }
 370
 371 static_always_inline
 372   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 373                                     snat_session_key_t * p_key0)
 374 {
 375   icmp46_header_t *icmp0;
 376   snat_session_key_t key0;
 377   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 378   ip4_header_t *inner_ip0 = 0;
 379   void *l4_header = 0;
 380   icmp46_header_t *inner_icmp0;
 381
 382   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 383   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 384
 385   if (!icmp_is_error_message (icmp0))
 386     {
 387       key0.protocol = SNAT_PROTOCOL_ICMP;
 388       key0.addr = ip0->src_address;
 389       key0.port = echo0->identifier;
 390     }
 391   else
 392     {
 393       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 394       l4_header = ip4_next_header (inner_ip0);
 395       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 396       key0.addr = inner_ip0->dst_address;
 397       switch (key0.protocol)
 398         {
 399         case SNAT_PROTOCOL_ICMP:
 400           inner_icmp0 = (icmp46_header_t *) l4_header;
 401           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 402           key0.port = inner_echo0->identifier;
 403           break;
 404         case SNAT_PROTOCOL_UDP:
 405         case SNAT_PROTOCOL_TCP:
 406           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 407           break;
 408         default:
 409           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 410         }
 411     }
 412   *p_key0 = key0;
 413   return -1;                    /* success */
 414 }
 415
 416 /**
 417  * Get address and port values to be used for ICMP packet translation
 418  * and create session if needed
 419  *
 420  * @param[in,out] sm             NAT main
 421  * @param[in,out] node           NAT node runtime
 422  * @param[in] thread_index       thread index
 423  * @param[in,out] b0             buffer containing packet to be translated
 424  * @param[out] p_proto           protocol used for matching
 425  * @param[out] p_value           address and port after NAT translation
 426  * @param[out] p_dont_translate  if packet should not be translated
 427  * @param d                      optional parameter
 428  * @param e                      optional parameter
 429  */
 430 u32
 431 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 432                         u32 thread_index, vlib_buffer_t * b0,
 433                         ip4_header_t * ip0, u8 * p_proto,
 434                         snat_session_key_t * p_value,
 435                         u8 * p_dont_translate, void *d, void *e)
 436 {
 437   icmp46_header_t *icmp0;
 438   u32 sw_if_index0;
 439   u32 rx_fib_index0;
 440   snat_session_key_t key0;
 441   snat_session_t *s0 = 0;
 442   u8 dont_translate = 0;
 443   clib_bihash_kv_8_8_t kv0, value0;
 444   u32 next0 = ~0;
 445   int err;
 446
 447   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 448   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 449   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 450
 451   err = icmp_get_key (ip0, &key0);
 452   if (err != -1)
 453     {
 454       b0->error = node->errors[err];
 455       next0 = SNAT_IN2OUT_NEXT_DROP;
 456       goto out;
 457     }
 458   key0.fib_index = rx_fib_index0;
 459
 460   kv0.key = key0.as_u64;
 461
 462   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 463                               &value0))
 464     {
 465       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 466         {
 467           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 468                                                                key0.protocol,
 469                                                                key0.port,
 470                                                                key0.port,
 471                                                                thread_index,
 472                                                                sw_if_index0)))
 473             {
 474               dont_translate = 1;
 475               goto out;
 476             }
 477         }
 478       else
 479         {
 480           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 481                                                  ip0, SNAT_PROTOCOL_ICMP,
 482                                                  rx_fib_index0,
 483                                                  thread_index)))
 484             {
 485               dont_translate = 1;
 486               goto out;
 487             }
 488         }
 489
 490       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 491         {
 492           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 493           next0 = SNAT_IN2OUT_NEXT_DROP;
 494           goto out;
 495         }
 496
 497       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 498                          thread_index, vlib_time_now (sm->vlib_main));
 499
 500       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 501         goto out;
 502
 503       if (!s0)
 504         {
 505           dont_translate = 1;
 506           goto out;
 507         }
 508     }
 509   else
 510     {
 511       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 512                          icmp0->type != ICMP4_echo_reply &&
 513                          !icmp_is_error_message (icmp0)))
 514         {
 515           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 516           next0 = SNAT_IN2OUT_NEXT_DROP;
 517           goto out;
 518         }
 519
 520       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 521                               value0.value);
 522     }
 523
 524 out:
 525   *p_proto = key0.protocol;
 526   if (s0)
 527     *p_value = s0->out2in;
 528   *p_dont_translate = dont_translate;
 529   if (d)
 530     *(snat_session_t **) d = s0;
 531   return next0;
 532 }
 533
 534 /**
 535  * Get address and port values to be used for ICMP packet translation
 536  *
 537  * @param[in] sm                 NAT main
 538  * @param[in,out] node           NAT node runtime
 539  * @param[in] thread_index       thread index
 540  * @param[in,out] b0             buffer containing packet to be translated
 541  * @param[out] p_proto           protocol used for matching
 542  * @param[out] p_value           address and port after NAT translation
 543  * @param[out] p_dont_translate  if packet should not be translated
 544  * @param d                      optional parameter
 545  * @param e                      optional parameter
 546  */
 547 u32
 548 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 549                         u32 thread_index, vlib_buffer_t * b0,
 550                         ip4_header_t * ip0, u8 * p_proto,
 551                         snat_session_key_t * p_value,
 552                         u8 * p_dont_translate, void *d, void *e)
 553 {
 554   icmp46_header_t *icmp0;
 555   u32 sw_if_index0;
 556   u32 rx_fib_index0;
 557   snat_session_key_t key0;
 558   snat_session_key_t sm0;
 559   u8 dont_translate = 0;
 560   u8 is_addr_only;
 561   u32 next0 = ~0;
 562   int err;
 563
 564   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 565   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 566   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 567
 568   err = icmp_get_key (ip0, &key0);
 569   if (err != -1)
 570     {
 571       b0->error = node->errors[err];
 572       next0 = SNAT_IN2OUT_NEXT_DROP;
 573       goto out2;
 574     }
 575   key0.fib_index = rx_fib_index0;
 576
 577   if (snat_static_mapping_match
 578       (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0, 0))
 579     {
 580       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 581                                                   IP_PROTOCOL_ICMP,
 582                                                   rx_fib_index0)))
 583         {
 584           dont_translate = 1;
 585           goto out;
 586         }
 587
 588       if (icmp_is_error_message (icmp0))
 589         {
 590           next0 = SNAT_IN2OUT_NEXT_DROP;
 591           goto out;
 592         }
 593
 594       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 595       next0 = SNAT_IN2OUT_NEXT_DROP;
 596       goto out;
 597     }
 598
 599   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 600                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 601                      !icmp_is_error_message (icmp0)))
 602     {
 603       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 604       next0 = SNAT_IN2OUT_NEXT_DROP;
 605       goto out;
 606     }
 607
 608 out:
 609   *p_value = sm0;
 610 out2:
 611   *p_proto = key0.protocol;
 612   *p_dont_translate = dont_translate;
 613   return next0;
 614 }
 615
 616 u32
 617 icmp_in2out (snat_main_t * sm,
 618              vlib_buffer_t * b0,
 619              ip4_header_t * ip0,
 620              icmp46_header_t * icmp0,
 621              u32 sw_if_index0,
 622              u32 rx_fib_index0,
 623              vlib_node_runtime_t * node,
 624              u32 next0, u32 thread_index, void *d, void *e)
 625 {
 626   snat_session_key_t sm0;
 627   u8 protocol;
 628   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 629   ip4_header_t *inner_ip0;
 630   void *l4_header = 0;
 631   icmp46_header_t *inner_icmp0;
 632   u8 dont_translate;
 633   u32 new_addr0, old_addr0;
 634   u16 old_id0, new_id0;
 635   u16 old_checksum0, new_checksum0;
 636   ip_csum_t sum0;
 637   u16 checksum0;
 638   u32 next0_tmp;
 639
 640   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 641
 642   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 643                                         &protocol, &sm0, &dont_translate, d,
 644                                         e);
 645   if (next0_tmp != ~0)
 646     next0 = next0_tmp;
 647   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 648     goto out;
 649
 650   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 651     {
 652       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 653                                              (u8 *)
 654                                              vlib_buffer_get_current (b0),
 655                                              ntohs (ip0->length) -
 656                                              ip4_header_bytes (ip0), 0);
 657       checksum0 = ~ip_csum_fold (sum0);
 658       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 659         {
 660           next0 = SNAT_IN2OUT_NEXT_DROP;
 661           goto out;
 662         }
 663     }
 664
 665   old_addr0 = ip0->src_address.as_u32;
 666   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 667
 668   sum0 = ip0->checksum;
 669   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 670                          src_address /* changed member */ );
 671   ip0->checksum = ip_csum_fold (sum0);
 672
 673   if (icmp0->checksum == 0)
 674     icmp0->checksum = 0xffff;
 675
 676   if (!icmp_is_error_message (icmp0))
 677     {
 678       new_id0 = sm0.port;
 679       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 680         {
 681           old_id0 = echo0->identifier;
 682           new_id0 = sm0.port;
 683           echo0->identifier = new_id0;
 684
 685           sum0 = icmp0->checksum;
 686           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 687                                  identifier);
 688           icmp0->checksum = ip_csum_fold (sum0);
 689         }
 690     }
 691   else
 692     {
 693       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 694       l4_header = ip4_next_header (inner_ip0);
 695
 696       if (!ip4_header_checksum_is_valid (inner_ip0))
 697         {
 698           next0 = SNAT_IN2OUT_NEXT_DROP;
 699           goto out;
 700         }
 701
 702       /* update inner destination IP address */
 703       old_addr0 = inner_ip0->dst_address.as_u32;
 704       inner_ip0->dst_address = sm0.addr;
 705       new_addr0 = inner_ip0->dst_address.as_u32;
 706       sum0 = icmp0->checksum;
 707       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 708                              dst_address /* changed member */ );
 709       icmp0->checksum = ip_csum_fold (sum0);
 710
 711       /* update inner IP header checksum */
 712       old_checksum0 = inner_ip0->checksum;
 713       sum0 = inner_ip0->checksum;
 714       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 715                              dst_address /* changed member */ );
 716       inner_ip0->checksum = ip_csum_fold (sum0);
 717       new_checksum0 = inner_ip0->checksum;
 718       sum0 = icmp0->checksum;
 719       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 720                              checksum);
 721       icmp0->checksum = ip_csum_fold (sum0);
 722
 723       switch (protocol)
 724         {
 725         case SNAT_PROTOCOL_ICMP:
 726           inner_icmp0 = (icmp46_header_t *) l4_header;
 727           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 728
 729           old_id0 = inner_echo0->identifier;
 730           new_id0 = sm0.port;
 731           inner_echo0->identifier = new_id0;
 732
 733           sum0 = icmp0->checksum;
 734           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 735                                  identifier);
 736           icmp0->checksum = ip_csum_fold (sum0);
 737           break;
 738         case SNAT_PROTOCOL_UDP:
 739         case SNAT_PROTOCOL_TCP:
 740           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 741           new_id0 = sm0.port;
 742           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 743
 744           sum0 = icmp0->checksum;
 745           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 746                                  dst_port);
 747           icmp0->checksum = ip_csum_fold (sum0);
 748           break;
 749         default:
 750           ASSERT (0);
 751         }
 752     }
 753
 754   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 755     {
 756       if (sm->deterministic ||
 757           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 758                                       sm->endpoint_dependent))
 759         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 760     }
 761
 762 out:
 763   return next0;
 764 }
 765
 766 static inline u32
 767 icmp_in2out_slow_path (snat_main_t * sm,
 768                        vlib_buffer_t * b0,
 769                        ip4_header_t * ip0,
 770                        icmp46_header_t * icmp0,
 771                        u32 sw_if_index0,
 772                        u32 rx_fib_index0,
 773                        vlib_node_runtime_t * node,
 774                        u32 next0,
 775                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 776 {
 777   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 778                        next0, thread_index, p_s0, 0);
 779   snat_session_t *s0 = *p_s0;
 780   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 781     {
 782       /* Accounting */
 783       nat44_session_update_counters (s0, now,
 784                                      vlib_buffer_length_in_chain
 785                                      (sm->vlib_main, b0));
 786       /* Per-user LRU list maintenance */
 787       nat44_session_update_lru (sm, s0, thread_index);
 788     }
 789   return next0;
 790 }
 791
 792 static int
 793 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 794                              vlib_buffer_t * b,
 795                              ip4_header_t * ip, u32 rx_fib_index)
 796 {
 797   clib_bihash_kv_8_8_t kv, value;
 798   snat_static_mapping_t *m;
 799   snat_session_key_t m_key;
 800   u32 old_addr, new_addr;
 801   ip_csum_t sum;
 802
 803   m_key.addr = ip->src_address;
 804   m_key.port = 0;
 805   m_key.protocol = 0;
 806   m_key.fib_index = rx_fib_index;
 807   kv.key = m_key.as_u64;
 808   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 809     return 1;
 810
 811   m = pool_elt_at_index (sm->static_mappings, value.value);
 812
 813   old_addr = ip->src_address.as_u32;
 814   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 815   sum = ip->checksum;
 816   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 817   ip->checksum = ip_csum_fold (sum);
 818
 819
 820   /* Hairpinning */
 821   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 822     {
 823       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 824       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 825     }
 826
 827   return 0;
 828 }
 829
 830 static inline uword
 831 snat_in2out_node_fn_inline (vlib_main_t * vm,
 832                             vlib_node_runtime_t * node,
 833                             vlib_frame_t * frame, int is_slow_path,
 834                             int is_output_feature)
 835 {
 836   u32 n_left_from, *from, *to_next;
 837   snat_in2out_next_t next_index;
 838   u32 pkts_processed = 0;
 839   snat_main_t *sm = &snat_main;
 840   f64 now = vlib_time_now (vm);
 841   u32 stats_node_index;
 842   u32 thread_index = vm->thread_index;
 843
 844   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 845     snat_in2out_node.index;
 846
 847   from = vlib_frame_vector_args (frame);
 848   n_left_from = frame->n_vectors;
 849   next_index = node->cached_next_index;
 850
 851   while (n_left_from > 0)
 852     {
 853       u32 n_left_to_next;
 854
 855       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 856
 857       while (n_left_from >= 4 && n_left_to_next >= 2)
 858         {
 859           u32 bi0, bi1;
 860           vlib_buffer_t *b0, *b1;
 861           u32 next0, next1;
 862           u32 sw_if_index0, sw_if_index1;
 863           ip4_header_t *ip0, *ip1;
 864           ip_csum_t sum0, sum1;
 865           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 866           u16 old_port0, new_port0, old_port1, new_port1;
 867           udp_header_t *udp0, *udp1;
 868           tcp_header_t *tcp0, *tcp1;
 869           icmp46_header_t *icmp0, *icmp1;
 870           snat_session_key_t key0, key1;
 871           u32 rx_fib_index0, rx_fib_index1;
 872           u32 proto0, proto1;
 873           snat_session_t *s0 = 0, *s1 = 0;
 874           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 875           u32 iph_offset0 = 0, iph_offset1 = 0;
 876
 877           /* Prefetch next iteration. */
 878           {
 879             vlib_buffer_t *p2, *p3;
 880
 881             p2 = vlib_get_buffer (vm, from[2]);
 882             p3 = vlib_get_buffer (vm, from[3]);
 883
 884             vlib_prefetch_buffer_header (p2, LOAD);
 885             vlib_prefetch_buffer_header (p3, LOAD);
 886
 887             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 888             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 889           }
 890
 891           /* speculatively enqueue b0 and b1 to the current next frame */
 892           to_next[0] = bi0 = from[0];
 893           to_next[1] = bi1 = from[1];
 894           from += 2;
 895           to_next += 2;
 896           n_left_from -= 2;
 897           n_left_to_next -= 2;
 898
 899           b0 = vlib_get_buffer (vm, bi0);
 900           b1 = vlib_get_buffer (vm, bi1);
 901
 902           if (is_output_feature)
 903             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 904
 905           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 906                                   iph_offset0);
 907
 908           udp0 = ip4_next_header (ip0);
 909           tcp0 = (tcp_header_t *) udp0;
 910           icmp0 = (icmp46_header_t *) udp0;
 911
 912           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 913           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 914                                    sw_if_index0);
 915
 916           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 917
 918           if (PREDICT_FALSE (ip0->ttl == 1))
 919             {
 920               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 921               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 922                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 923                                            0);
 924               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 925               goto trace00;
 926             }
 927
 928           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 929
 930           /* Next configured feature, probably ip4-lookup */
 931           if (is_slow_path)
 932             {
 933               if (PREDICT_FALSE (proto0 == ~0))
 934                 {
 935                   if (nat_in2out_sm_unknown_proto
 936                       (sm, b0, ip0, rx_fib_index0))
 937                     {
 938                       next0 = SNAT_IN2OUT_NEXT_DROP;
 939                       b0->error =
 940                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 941                     }
 942                   goto trace00;
 943                 }
 944
 945               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 946                 {
 947                   next0 = icmp_in2out_slow_path
 948                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 949                      node, next0, now, thread_index, &s0);
 950                   goto trace00;
 951                 }
 952             }
 953           else
 954             {
 955               if (PREDICT_FALSE
 956                   (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 957                 {
 958                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 959                   goto trace00;
 960                 }
 961
 962               if (ip4_is_fragment (ip0))
 963                 {
 964                   next0 = SNAT_IN2OUT_NEXT_REASS;
 965                   goto trace00;
 966                 }
 967             }
 968
 969           key0.addr = ip0->src_address;
 970           key0.port = udp0->src_port;
 971           key0.protocol = proto0;
 972           key0.fib_index = rx_fib_index0;
 973
 974           kv0.key = key0.as_u64;
 975
 976           if (PREDICT_FALSE
 977               (clib_bihash_search_8_8
 978                (&sm->per_thread_data[thread_index].in2out, &kv0,
 979                 &value0) != 0))
 980             {
 981               if (is_slow_path)
 982                 {
 983                   if (is_output_feature)
 984                     {
 985                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
 986                                                                            ip0,
 987                                                                            proto0,
 988                                                                            udp0->src_port,
 989                                                                            udp0->dst_port,
 990                                                                            thread_index,
 991                                                                            sw_if_index0)))
 992                         goto trace00;
 993                     }
 994                   else
 995                     {
 996                       if (PREDICT_FALSE
 997                           (snat_not_translate
 998                            (sm, node, sw_if_index0, ip0, proto0,
 999                             rx_fib_index0, thread_index)))
1000                         goto trace00;
1001                     }
1002
1003                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1004                                      &s0, node, next0, thread_index, now);
1005                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1006                     goto trace00;
1007
1008                   if (PREDICT_FALSE (!s0))
1009                     goto trace00;
1010                 }
1011               else
1012                 {
1013                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1014                   goto trace00;
1015                 }
1016             }
1017           else
1018             s0 =
1019               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1020                                  value0.value);
1021
1022           b0->flags |= VNET_BUFFER_F_IS_NATED;
1023
1024           old_addr0 = ip0->src_address.as_u32;
1025           ip0->src_address = s0->out2in.addr;
1026           new_addr0 = ip0->src_address.as_u32;
1027           if (!is_output_feature)
1028             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1029
1030           sum0 = ip0->checksum;
1031           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1032                                  ip4_header_t,
1033                                  src_address /* changed member */ );
1034           ip0->checksum = ip_csum_fold (sum0);
1035
1036           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1037             {
1038               old_port0 = tcp0->src_port;
1039               tcp0->src_port = s0->out2in.port;
1040               new_port0 = tcp0->src_port;
1041
1042               sum0 = tcp0->checksum;
1043               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1044                                      ip4_header_t,
1045                                      dst_address /* changed member */ );
1046               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1047                                      ip4_header_t /* cheat */ ,
1048                                      length /* changed member */ );
1049               mss_clamping (sm, tcp0, &sum0);
1050               tcp0->checksum = ip_csum_fold (sum0);
1051             }
1052           else
1053             {
1054               old_port0 = udp0->src_port;
1055               udp0->src_port = s0->out2in.port;
1056               udp0->checksum = 0;
1057             }
1058
1059           /* Accounting */
1060           nat44_session_update_counters (s0, now,
1061                                          vlib_buffer_length_in_chain (vm,
1062                                                                       b0));
1063           /* Per-user LRU list maintenance */
1064           nat44_session_update_lru (sm, s0, thread_index);
1065         trace00:
1066
1067           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1068                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1069             {
1070               snat_in2out_trace_t *t =
1071                 vlib_add_trace (vm, node, b0, sizeof (*t));
1072               t->is_slow_path = is_slow_path;
1073               t->sw_if_index = sw_if_index0;
1074               t->next_index = next0;
1075               t->session_index = ~0;
1076               if (s0)
1077                 t->session_index =
1078                   s0 - sm->per_thread_data[thread_index].sessions;
1079             }
1080
1081           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1082
1083           if (is_output_feature)
1084             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1085
1086           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1087                                   iph_offset1);
1088
1089           udp1 = ip4_next_header (ip1);
1090           tcp1 = (tcp_header_t *) udp1;
1091           icmp1 = (icmp46_header_t *) udp1;
1092
1093           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1094           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1095                                    sw_if_index1);
1096
1097           if (PREDICT_FALSE (ip1->ttl == 1))
1098             {
1099               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1100               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1101                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1102                                            0);
1103               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1104               goto trace01;
1105             }
1106
1107           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1108
1109           /* Next configured feature, probably ip4-lookup */
1110           if (is_slow_path)
1111             {
1112               if (PREDICT_FALSE (proto1 == ~0))
1113                 {
1114                   if (nat_in2out_sm_unknown_proto
1115                       (sm, b1, ip1, rx_fib_index1))
1116                     {
1117                       next1 = SNAT_IN2OUT_NEXT_DROP;
1118                       b1->error =
1119                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1120                     }
1121                   goto trace01;
1122                 }
1123
1124               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1125                 {
1126                   next1 = icmp_in2out_slow_path
1127                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1128                      next1, now, thread_index, &s1);
1129                   goto trace01;
1130                 }
1131             }
1132           else
1133             {
1134               if (PREDICT_FALSE
1135                   (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1136                 {
1137                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1138                   goto trace01;
1139                 }
1140
1141               if (ip4_is_fragment (ip1))
1142                 {
1143                   next1 = SNAT_IN2OUT_NEXT_REASS;
1144                   goto trace01;
1145                 }
1146             }
1147
1148           key1.addr = ip1->src_address;
1149           key1.port = udp1->src_port;
1150           key1.protocol = proto1;
1151           key1.fib_index = rx_fib_index1;
1152
1153           kv1.key = key1.as_u64;
1154
1155           if (PREDICT_FALSE
1156               (clib_bihash_search_8_8
1157                (&sm->per_thread_data[thread_index].in2out, &kv1,
1158                 &value1) != 0))
1159             {
1160               if (is_slow_path)
1161                 {
1162                   if (is_output_feature)
1163                     {
1164                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1165                                                                            ip1,
1166                                                                            proto1,
1167                                                                            udp1->src_port,
1168                                                                            udp1->dst_port,
1169                                                                            thread_index,
1170                                                                            sw_if_index1)))
1171                         goto trace01;
1172                     }
1173                   else
1174                     {
1175                       if (PREDICT_FALSE
1176                           (snat_not_translate
1177                            (sm, node, sw_if_index1, ip1, proto1,
1178                             rx_fib_index1, thread_index)))
1179                         goto trace01;
1180                     }
1181
1182                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1183                                      &s1, node, next1, thread_index, now);
1184                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1185                     goto trace01;
1186
1187                   if (PREDICT_FALSE (!s1))
1188                     goto trace01;
1189                 }
1190               else
1191                 {
1192                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1193                   goto trace01;
1194                 }
1195             }
1196           else
1197             s1 =
1198               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1199                                  value1.value);
1200
1201           b1->flags |= VNET_BUFFER_F_IS_NATED;
1202
1203           old_addr1 = ip1->src_address.as_u32;
1204           ip1->src_address = s1->out2in.addr;
1205           new_addr1 = ip1->src_address.as_u32;
1206           if (!is_output_feature)
1207             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1208
1209           sum1 = ip1->checksum;
1210           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1211                                  ip4_header_t,
1212                                  src_address /* changed member */ );
1213           ip1->checksum = ip_csum_fold (sum1);
1214
1215           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1216             {
1217               old_port1 = tcp1->src_port;
1218               tcp1->src_port = s1->out2in.port;
1219               new_port1 = tcp1->src_port;
1220
1221               sum1 = tcp1->checksum;
1222               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1223                                      ip4_header_t,
1224                                      dst_address /* changed member */ );
1225               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1226                                      ip4_header_t /* cheat */ ,
1227                                      length /* changed member */ );
1228               mss_clamping (sm, tcp1, &sum1);
1229               tcp1->checksum = ip_csum_fold (sum1);
1230             }
1231           else
1232             {
1233               old_port1 = udp1->src_port;
1234               udp1->src_port = s1->out2in.port;
1235               udp1->checksum = 0;
1236             }
1237
1238           /* Accounting */
1239           nat44_session_update_counters (s1, now,
1240                                          vlib_buffer_length_in_chain (vm,
1241                                                                       b1));
1242           /* Per-user LRU list maintenance */
1243           nat44_session_update_lru (sm, s1, thread_index);
1244         trace01:
1245
1246           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1247                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1248             {
1249               snat_in2out_trace_t *t =
1250                 vlib_add_trace (vm, node, b1, sizeof (*t));
1251               t->sw_if_index = sw_if_index1;
1252               t->next_index = next1;
1253               t->session_index = ~0;
1254               if (s1)
1255                 t->session_index =
1256                   s1 - sm->per_thread_data[thread_index].sessions;
1257             }
1258
1259           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1260
1261           /* verify speculative enqueues, maybe switch current next frame */
1262           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1263                                            to_next, n_left_to_next,
1264                                            bi0, bi1, next0, next1);
1265         }
1266
1267       while (n_left_from > 0 && n_left_to_next > 0)
1268         {
1269           u32 bi0;
1270           vlib_buffer_t *b0;
1271           u32 next0;
1272           u32 sw_if_index0;
1273           ip4_header_t *ip0;
1274           ip_csum_t sum0;
1275           u32 new_addr0, old_addr0;
1276           u16 old_port0, new_port0;
1277           udp_header_t *udp0;
1278           tcp_header_t *tcp0;
1279           icmp46_header_t *icmp0;
1280           snat_session_key_t key0;
1281           u32 rx_fib_index0;
1282           u32 proto0;
1283           snat_session_t *s0 = 0;
1284           clib_bihash_kv_8_8_t kv0, value0;
1285           u32 iph_offset0 = 0;
1286
1287           /* speculatively enqueue b0 to the current next frame */
1288           bi0 = from[0];
1289           to_next[0] = bi0;
1290           from += 1;
1291           to_next += 1;
1292           n_left_from -= 1;
1293           n_left_to_next -= 1;
1294
1295           b0 = vlib_get_buffer (vm, bi0);
1296           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1297
1298           if (is_output_feature)
1299             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1300
1301           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1302                                   iph_offset0);
1303
1304           udp0 = ip4_next_header (ip0);
1305           tcp0 = (tcp_header_t *) udp0;
1306           icmp0 = (icmp46_header_t *) udp0;
1307
1308           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1309           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1310                                    sw_if_index0);
1311
1312           if (PREDICT_FALSE (ip0->ttl == 1))
1313             {
1314               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1315               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1316                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1317                                            0);
1318               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1319               goto trace0;
1320             }
1321
1322           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1323
1324           /* Next configured feature, probably ip4-lookup */
1325           if (is_slow_path)
1326             {
1327               if (PREDICT_FALSE (proto0 == ~0))
1328                 {
1329                   if (nat_in2out_sm_unknown_proto
1330                       (sm, b0, ip0, rx_fib_index0))
1331                     {
1332                       next0 = SNAT_IN2OUT_NEXT_DROP;
1333                       b0->error =
1334                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1335                     }
1336                   goto trace0;
1337                 }
1338
1339               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1340                 {
1341                   next0 = icmp_in2out_slow_path
1342                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1343                      next0, now, thread_index, &s0);
1344                   goto trace0;
1345                 }
1346             }
1347           else
1348             {
1349               if (PREDICT_FALSE
1350                   (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1351                 {
1352                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1353                   goto trace0;
1354                 }
1355
1356               if (ip4_is_fragment (ip0))
1357                 {
1358                   next0 = SNAT_IN2OUT_NEXT_REASS;
1359                   goto trace0;
1360                 }
1361             }
1362
1363           key0.addr = ip0->src_address;
1364           key0.port = udp0->src_port;
1365           key0.protocol = proto0;
1366           key0.fib_index = rx_fib_index0;
1367
1368           kv0.key = key0.as_u64;
1369
1370           if (clib_bihash_search_8_8
1371               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1372             {
1373               if (is_slow_path)
1374                 {
1375                   if (is_output_feature)
1376                     {
1377                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1378                                                                            ip0,
1379                                                                            proto0,
1380                                                                            udp0->src_port,
1381                                                                            udp0->dst_port,
1382                                                                            thread_index,
1383                                                                            sw_if_index0)))
1384                         goto trace0;
1385                     }
1386                   else
1387                     {
1388                       if (PREDICT_FALSE
1389                           (snat_not_translate
1390                            (sm, node, sw_if_index0, ip0, proto0,
1391                             rx_fib_index0, thread_index)))
1392                         goto trace0;
1393                     }
1394
1395                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1396                                      &s0, node, next0, thread_index, now);
1397
1398                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1399                     goto trace0;
1400
1401                   if (PREDICT_FALSE (!s0))
1402                     goto trace0;
1403                 }
1404               else
1405                 {
1406                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1407                   goto trace0;
1408                 }
1409             }
1410           else
1411             s0 =
1412               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1413                                  value0.value);
1414
1415           b0->flags |= VNET_BUFFER_F_IS_NATED;
1416
1417           old_addr0 = ip0->src_address.as_u32;
1418           ip0->src_address = s0->out2in.addr;
1419           new_addr0 = ip0->src_address.as_u32;
1420           if (!is_output_feature)
1421             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1422
1423           sum0 = ip0->checksum;
1424           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1425                                  ip4_header_t,
1426                                  src_address /* changed member */ );
1427           ip0->checksum = ip_csum_fold (sum0);
1428
1429           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1430             {
1431               old_port0 = tcp0->src_port;
1432               tcp0->src_port = s0->out2in.port;
1433               new_port0 = tcp0->src_port;
1434
1435               sum0 = tcp0->checksum;
1436               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1437                                      ip4_header_t,
1438                                      dst_address /* changed member */ );
1439               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1440                                      ip4_header_t /* cheat */ ,
1441                                      length /* changed member */ );
1442               mss_clamping (sm, tcp0, &sum0);
1443               tcp0->checksum = ip_csum_fold (sum0);
1444             }
1445           else
1446             {
1447               old_port0 = udp0->src_port;
1448               udp0->src_port = s0->out2in.port;
1449               udp0->checksum = 0;
1450             }
1451
1452           /* Accounting */
1453           nat44_session_update_counters (s0, now,
1454                                          vlib_buffer_length_in_chain (vm,
1455                                                                       b0));
1456           /* Per-user LRU list maintenance */
1457           nat44_session_update_lru (sm, s0, thread_index);
1458
1459         trace0:
1460           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1461                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1462             {
1463               snat_in2out_trace_t *t =
1464                 vlib_add_trace (vm, node, b0, sizeof (*t));
1465               t->is_slow_path = is_slow_path;
1466               t->sw_if_index = sw_if_index0;
1467               t->next_index = next0;
1468               t->session_index = ~0;
1469               if (s0)
1470                 t->session_index =
1471                   s0 - sm->per_thread_data[thread_index].sessions;
1472             }
1473
1474           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1475
1476           /* verify speculative enqueue, maybe switch current next frame */
1477           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1478                                            to_next, n_left_to_next,
1479                                            bi0, next0);
1480         }
1481
1482       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1483     }
1484
1485   vlib_node_increment_counter (vm, stats_node_index,
1486                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1487                                pkts_processed);
1488   return frame->n_vectors;
1489 }
1490
1491 static uword
1492 snat_in2out_fast_path_fn (vlib_main_t * vm,
1493                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1494 {
1495   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1496                                      0);
1497 }
1498
1499 /* *INDENT-OFF* */
1500 VLIB_REGISTER_NODE (snat_in2out_node) = {
1501   .function = snat_in2out_fast_path_fn,
1502   .name = "nat44-in2out",
1503   .vector_size = sizeof (u32),
1504   .format_trace = format_snat_in2out_trace,
1505   .type = VLIB_NODE_TYPE_INTERNAL,
1506
1507   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1508   .error_strings = snat_in2out_error_strings,
1509
1510   .runtime_data_bytes = sizeof (snat_runtime_t),
1511
1512   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1513
1514   /* edit / add dispositions here */
1515   .next_nodes = {
1516     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1517     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1518     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1519     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1520     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1521   },
1522 };
1523 /* *INDENT-ON* */
1524
1525 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1526
1527 static uword
1528 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1529                                  vlib_node_runtime_t * node,
1530                                  vlib_frame_t * frame)
1531 {
1532   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1533                                      1);
1534 }
1535
1536 /* *INDENT-OFF* */
1537 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1538   .function = snat_in2out_output_fast_path_fn,
1539   .name = "nat44-in2out-output",
1540   .vector_size = sizeof (u32),
1541   .format_trace = format_snat_in2out_trace,
1542   .type = VLIB_NODE_TYPE_INTERNAL,
1543
1544   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1545   .error_strings = snat_in2out_error_strings,
1546
1547   .runtime_data_bytes = sizeof (snat_runtime_t),
1548
1549   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1550
1551   /* edit / add dispositions here */
1552   .next_nodes = {
1553     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1554     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1555     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1556     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1557     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1558   },
1559 };
1560 /* *INDENT-ON* */
1561
1562 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1563                               snat_in2out_output_fast_path_fn);
1564
1565 static uword
1566 snat_in2out_slow_path_fn (vlib_main_t * vm,
1567                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1568 {
1569   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1570                                      0);
1571 }
1572
1573 /* *INDENT-OFF* */
1574 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1575   .function = snat_in2out_slow_path_fn,
1576   .name = "nat44-in2out-slowpath",
1577   .vector_size = sizeof (u32),
1578   .format_trace = format_snat_in2out_trace,
1579   .type = VLIB_NODE_TYPE_INTERNAL,
1580
1581   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1582   .error_strings = snat_in2out_error_strings,
1583
1584   .runtime_data_bytes = sizeof (snat_runtime_t),
1585
1586   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1587
1588   /* edit / add dispositions here */
1589   .next_nodes = {
1590     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1591     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1592     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1593     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1594     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1595   },
1596 };
1597 /* *INDENT-ON* */
1598
1599 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1600                               snat_in2out_slow_path_fn);
1601
1602 static uword
1603 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1604                                  vlib_node_runtime_t * node,
1605                                  vlib_frame_t * frame)
1606 {
1607   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1608                                      1);
1609 }
1610
1611 /* *INDENT-OFF* */
1612 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1613   .function = snat_in2out_output_slow_path_fn,
1614   .name = "nat44-in2out-output-slowpath",
1615   .vector_size = sizeof (u32),
1616   .format_trace = format_snat_in2out_trace,
1617   .type = VLIB_NODE_TYPE_INTERNAL,
1618
1619   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1620   .error_strings = snat_in2out_error_strings,
1621
1622   .runtime_data_bytes = sizeof (snat_runtime_t),
1623
1624   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1625
1626   /* edit / add dispositions here */
1627   .next_nodes = {
1628     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1629     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1630     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1631     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1632     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1633   },
1634 };
1635 /* *INDENT-ON* */
1636
1637 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1638                               snat_in2out_output_slow_path_fn);
1639
1640 static uword
1641 nat44_in2out_reass_node_fn (vlib_main_t * vm,
1642                             vlib_node_runtime_t * node, vlib_frame_t * frame)
1643 {
1644   u32 n_left_from, *from, *to_next;
1645   snat_in2out_next_t next_index;
1646   u32 pkts_processed = 0;
1647   snat_main_t *sm = &snat_main;
1648   f64 now = vlib_time_now (vm);
1649   u32 thread_index = vm->thread_index;
1650   snat_main_per_thread_data_t *per_thread_data =
1651     &sm->per_thread_data[thread_index];
1652   u32 *fragments_to_drop = 0;
1653   u32 *fragments_to_loopback = 0;
1654
1655   from = vlib_frame_vector_args (frame);
1656   n_left_from = frame->n_vectors;
1657   next_index = node->cached_next_index;
1658
1659   while (n_left_from > 0)
1660     {
1661       u32 n_left_to_next;
1662
1663       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1664
1665       while (n_left_from > 0 && n_left_to_next > 0)
1666         {
1667           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1668           vlib_buffer_t *b0;
1669           u32 next0;
1670           u8 cached0 = 0;
1671           ip4_header_t *ip0;
1672           nat_reass_ip4_t *reass0;
1673           udp_header_t *udp0;
1674           tcp_header_t *tcp0;
1675           snat_session_key_t key0;
1676           clib_bihash_kv_8_8_t kv0, value0;
1677           snat_session_t *s0 = 0;
1678           u16 old_port0, new_port0;
1679           ip_csum_t sum0;
1680
1681           /* speculatively enqueue b0 to the current next frame */
1682           bi0 = from[0];
1683           to_next[0] = bi0;
1684           from += 1;
1685           to_next += 1;
1686           n_left_from -= 1;
1687           n_left_to_next -= 1;
1688
1689           b0 = vlib_get_buffer (vm, bi0);
1690           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1691
1692           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1693           rx_fib_index0 =
1694             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1695                                                  sw_if_index0);
1696
1697           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1698             {
1699               next0 = SNAT_IN2OUT_NEXT_DROP;
1700               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1701               goto trace0;
1702             }
1703
1704           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1705           udp0 = ip4_next_header (ip0);
1706           tcp0 = (tcp_header_t *) udp0;
1707           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1708
1709           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1710                                                  ip0->dst_address,
1711                                                  ip0->fragment_id,
1712                                                  ip0->protocol,
1713                                                  1, &fragments_to_drop);
1714
1715           if (PREDICT_FALSE (!reass0))
1716             {
1717               next0 = SNAT_IN2OUT_NEXT_DROP;
1718               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1719               nat_log_notice ("maximum reassemblies exceeded");
1720               goto trace0;
1721             }
1722
1723           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1724             {
1725               key0.addr = ip0->src_address;
1726               key0.port = udp0->src_port;
1727               key0.protocol = proto0;
1728               key0.fib_index = rx_fib_index0;
1729               kv0.key = key0.as_u64;
1730
1731               if (clib_bihash_search_8_8
1732                   (&per_thread_data->in2out, &kv0, &value0))
1733                 {
1734                   if (PREDICT_FALSE
1735                       (snat_not_translate
1736                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1737                         thread_index)))
1738                     goto trace0;
1739
1740                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1741                                      &s0, node, next0, thread_index, now);
1742
1743                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1744                     goto trace0;
1745
1746                   if (PREDICT_FALSE (!s0))
1747                     goto trace0;
1748
1749                   reass0->sess_index = s0 - per_thread_data->sessions;
1750                 }
1751               else
1752                 {
1753                   s0 = pool_elt_at_index (per_thread_data->sessions,
1754                                           value0.value);
1755                   reass0->sess_index = value0.value;
1756                 }
1757               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1758             }
1759           else
1760             {
1761               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1762                 {
1763                   if (nat_ip4_reass_add_fragment
1764                       (reass0, bi0, &fragments_to_drop))
1765                     {
1766                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1767                       nat_log_notice
1768                         ("maximum fragments per reassembly exceeded");
1769                       next0 = SNAT_IN2OUT_NEXT_DROP;
1770                       goto trace0;
1771                     }
1772                   cached0 = 1;
1773                   goto trace0;
1774                 }
1775               s0 = pool_elt_at_index (per_thread_data->sessions,
1776                                       reass0->sess_index);
1777             }
1778
1779           old_addr0 = ip0->src_address.as_u32;
1780           ip0->src_address = s0->out2in.addr;
1781           new_addr0 = ip0->src_address.as_u32;
1782           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1783
1784           sum0 = ip0->checksum;
1785           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1786                                  ip4_header_t,
1787                                  src_address /* changed member */ );
1788           ip0->checksum = ip_csum_fold (sum0);
1789
1790           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1791             {
1792               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1793                 {
1794                   old_port0 = tcp0->src_port;
1795                   tcp0->src_port = s0->out2in.port;
1796                   new_port0 = tcp0->src_port;
1797
1798                   sum0 = tcp0->checksum;
1799                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1800                                          ip4_header_t,
1801                                          dst_address /* changed member */ );
1802                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1803                                          ip4_header_t /* cheat */ ,
1804                                          length /* changed member */ );
1805                   tcp0->checksum = ip_csum_fold (sum0);
1806                 }
1807               else
1808                 {
1809                   old_port0 = udp0->src_port;
1810                   udp0->src_port = s0->out2in.port;
1811                   udp0->checksum = 0;
1812                 }
1813             }
1814
1815           /* Hairpinning */
1816           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1817                                    s0->ext_host_port, proto0, 0);
1818
1819           /* Accounting */
1820           nat44_session_update_counters (s0, now,
1821                                          vlib_buffer_length_in_chain (vm,
1822                                                                       b0));
1823           /* Per-user LRU list maintenance */
1824           nat44_session_update_lru (sm, s0, thread_index);
1825
1826         trace0:
1827           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1828                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1829             {
1830               nat44_reass_trace_t *t =
1831                 vlib_add_trace (vm, node, b0, sizeof (*t));
1832               t->cached = cached0;
1833               t->sw_if_index = sw_if_index0;
1834               t->next_index = next0;
1835             }
1836
1837           if (cached0)
1838             {
1839               n_left_to_next++;
1840               to_next--;
1841             }
1842           else
1843             {
1844               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1845
1846               /* verify speculative enqueue, maybe switch current next frame */
1847               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1848                                                to_next, n_left_to_next,
1849                                                bi0, next0);
1850             }
1851
1852           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1853             {
1854               from = vlib_frame_vector_args (frame);
1855               u32 len = vec_len (fragments_to_loopback);
1856               if (len <= VLIB_FRAME_SIZE)
1857                 {
1858                   clib_memcpy (from, fragments_to_loopback,
1859                                sizeof (u32) * len);
1860                   n_left_from = len;
1861                   vec_reset_length (fragments_to_loopback);
1862                 }
1863               else
1864                 {
1865                   clib_memcpy (from,
1866                                fragments_to_loopback + (len -
1867                                                         VLIB_FRAME_SIZE),
1868                                sizeof (u32) * VLIB_FRAME_SIZE);
1869                   n_left_from = VLIB_FRAME_SIZE;
1870                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1871                 }
1872             }
1873         }
1874
1875       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1876     }
1877
1878   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
1879                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1880                                pkts_processed);
1881
1882   nat_send_all_to_node (vm, fragments_to_drop, node,
1883                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1884                         SNAT_IN2OUT_NEXT_DROP);
1885
1886   vec_free (fragments_to_drop);
1887   vec_free (fragments_to_loopback);
1888   return frame->n_vectors;
1889 }
1890
1891 /* *INDENT-OFF* */
1892 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
1893   .function = nat44_in2out_reass_node_fn,
1894   .name = "nat44-in2out-reass",
1895   .vector_size = sizeof (u32),
1896   .format_trace = format_nat44_reass_trace,
1897   .type = VLIB_NODE_TYPE_INTERNAL,
1898
1899   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1900   .error_strings = snat_in2out_error_strings,
1901
1902   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1903   .next_nodes = {
1904     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1905     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1906     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1907     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1908     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1909   },
1910 };
1911 /* *INDENT-ON* */
1912
1913 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
1914                               nat44_in2out_reass_node_fn);
1915
1916 static uword
1917 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1918                                 vlib_node_runtime_t * node,
1919                                 vlib_frame_t * frame)
1920 {
1921   u32 n_left_from, *from, *to_next;
1922   snat_in2out_next_t next_index;
1923   u32 pkts_processed = 0;
1924   snat_main_t *sm = &snat_main;
1925   u32 stats_node_index;
1926
1927   stats_node_index = snat_in2out_fast_node.index;
1928
1929   from = vlib_frame_vector_args (frame);
1930   n_left_from = frame->n_vectors;
1931   next_index = node->cached_next_index;
1932
1933   while (n_left_from > 0)
1934     {
1935       u32 n_left_to_next;
1936
1937       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1938
1939       while (n_left_from > 0 && n_left_to_next > 0)
1940         {
1941           u32 bi0;
1942           vlib_buffer_t *b0;
1943           u32 next0;
1944           u32 sw_if_index0;
1945           ip4_header_t *ip0;
1946           ip_csum_t sum0;
1947           u32 new_addr0, old_addr0;
1948           u16 old_port0, new_port0;
1949           udp_header_t *udp0;
1950           tcp_header_t *tcp0;
1951           icmp46_header_t *icmp0;
1952           snat_session_key_t key0, sm0;
1953           u32 proto0;
1954           u32 rx_fib_index0;
1955
1956           /* speculatively enqueue b0 to the current next frame */
1957           bi0 = from[0];
1958           to_next[0] = bi0;
1959           from += 1;
1960           to_next += 1;
1961           n_left_from -= 1;
1962           n_left_to_next -= 1;
1963
1964           b0 = vlib_get_buffer (vm, bi0);
1965           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1966
1967           ip0 = vlib_buffer_get_current (b0);
1968           udp0 = ip4_next_header (ip0);
1969           tcp0 = (tcp_header_t *) udp0;
1970           icmp0 = (icmp46_header_t *) udp0;
1971
1972           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1973           rx_fib_index0 =
1974             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1975
1976           if (PREDICT_FALSE (ip0->ttl == 1))
1977             {
1978               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1979               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1980                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1981                                            0);
1982               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1983               goto trace0;
1984             }
1985
1986           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1987
1988           if (PREDICT_FALSE (proto0 == ~0))
1989             goto trace0;
1990
1991           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1992             {
1993               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
1994                                    rx_fib_index0, node, next0, ~0, 0, 0);
1995               goto trace0;
1996             }
1997
1998           key0.addr = ip0->src_address;
1999           key0.protocol = proto0;
2000           key0.port = udp0->src_port;
2001           key0.fib_index = rx_fib_index0;
2002
2003           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0, 0))
2004             {
2005               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2006               next0 = SNAT_IN2OUT_NEXT_DROP;
2007               goto trace0;
2008             }
2009
2010           new_addr0 = sm0.addr.as_u32;
2011           new_port0 = sm0.port;
2012           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2013           old_addr0 = ip0->src_address.as_u32;
2014           ip0->src_address.as_u32 = new_addr0;
2015
2016           sum0 = ip0->checksum;
2017           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2018                                  ip4_header_t,
2019                                  src_address /* changed member */ );
2020           ip0->checksum = ip_csum_fold (sum0);
2021
2022           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
2023             {
2024               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2025                 {
2026                   old_port0 = tcp0->src_port;
2027                   tcp0->src_port = new_port0;
2028
2029                   sum0 = tcp0->checksum;
2030                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2031                                          ip4_header_t,
2032                                          dst_address /* changed member */ );
2033                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2034                                          ip4_header_t /* cheat */ ,
2035                                          length /* changed member */ );
2036                   mss_clamping (sm, tcp0, &sum0);
2037                   tcp0->checksum = ip_csum_fold (sum0);
2038                 }
2039               else
2040                 {
2041                   old_port0 = udp0->src_port;
2042                   udp0->src_port = new_port0;
2043                   udp0->checksum = 0;
2044                 }
2045             }
2046           else
2047             {
2048               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2049                 {
2050                   sum0 = tcp0->checksum;
2051                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2052                                          ip4_header_t,
2053                                          dst_address /* changed member */ );
2054                   mss_clamping (sm, tcp0, &sum0);
2055                   tcp0->checksum = ip_csum_fold (sum0);
2056                 }
2057             }
2058
2059           /* Hairpinning */
2060           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2061
2062         trace0:
2063           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2064                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2065             {
2066               snat_in2out_trace_t *t =
2067                 vlib_add_trace (vm, node, b0, sizeof (*t));
2068               t->sw_if_index = sw_if_index0;
2069               t->next_index = next0;
2070             }
2071
2072           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2073
2074           /* verify speculative enqueue, maybe switch current next frame */
2075           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2076                                            to_next, n_left_to_next,
2077                                            bi0, next0);
2078         }
2079
2080       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2081     }
2082
2083   vlib_node_increment_counter (vm, stats_node_index,
2084                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2085                                pkts_processed);
2086   return frame->n_vectors;
2087 }
2088
2089
2090 /* *INDENT-OFF* */
2091 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2092   .function = snat_in2out_fast_static_map_fn,
2093   .name = "nat44-in2out-fast",
2094   .vector_size = sizeof (u32),
2095   .format_trace = format_snat_in2out_fast_trace,
2096   .type = VLIB_NODE_TYPE_INTERNAL,
2097
2098   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2099   .error_strings = snat_in2out_error_strings,
2100
2101   .runtime_data_bytes = sizeof (snat_runtime_t),
2102
2103   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2104
2105   /* edit / add dispositions here */
2106   .next_nodes = {
2107     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2108     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2109     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2110     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2111     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2112   },
2113 };
2114 /* *INDENT-ON* */
2115
2116 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node,
2117                               snat_in2out_fast_static_map_fn);
2118
2119 /*
2120  * fd.io coding-style-patch-verification: ON
2121  *
2122  * Local Variables:
2123  * eval: (c-set-style "gnu")
2124  * End:
2125  */