src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #include <vppinfra/hash.h>
  33 #include <vppinfra/error.h>
  34 #include <vppinfra/elog.h>
  35
  36 typedef struct
  37 {
  38   u32 sw_if_index;
  39   u32 next_index;
  40   u32 session_index;
  41   u32 is_slow_path;
  42 } snat_in2out_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 *
  46 format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  51   char *tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 *
  62 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 vlib_node_registration_t snat_in2out_node;
  75 vlib_node_registration_t snat_in2out_slowpath_node;
  76 vlib_node_registration_t snat_in2out_fast_node;
  77 vlib_node_registration_t snat_in2out_output_node;
  78 vlib_node_registration_t snat_in2out_output_slowpath_node;
  79 vlib_node_registration_t nat44_in2out_reass_node;
  80
  81 #define foreach_snat_in2out_error                       \
  82 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  83 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  84 _(OUT_OF_PORTS, "Out of ports")                         \
  85 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  86 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  87 _(NO_TRANSLATION, "No translation")                     \
  88 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  89 _(DROP_FRAGMENT, "Drop fragment")                       \
  90 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  91 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  92
  93 typedef enum
  94 {
  95 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  96   foreach_snat_in2out_error
  97 #undef _
  98     SNAT_IN2OUT_N_ERROR,
  99 } snat_in2out_error_t;
 100
 101 static char *snat_in2out_error_strings[] = {
 102 #define _(sym,string) string,
 103   foreach_snat_in2out_error
 104 #undef _
 105 };
 106
 107 typedef enum
 108 {
 109   SNAT_IN2OUT_NEXT_LOOKUP,
 110   SNAT_IN2OUT_NEXT_DROP,
 111   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 112   SNAT_IN2OUT_NEXT_SLOW_PATH,
 113   SNAT_IN2OUT_NEXT_REASS,
 114   SNAT_IN2OUT_N_NEXT,
 115 } snat_in2out_next_t;
 116
 117 static inline int
 118 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 119                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 120                     u32 rx_fib_index0, u32 thread_index)
 121 {
 122   udp_header_t *udp0 = ip4_next_header (ip0);
 123   snat_session_key_t key0, sm0;
 124   clib_bihash_kv_8_8_t kv0, value0;
 125
 126   key0.addr = ip0->dst_address;
 127   key0.port = udp0->dst_port;
 128   key0.protocol = proto0;
 129   key0.fib_index = sm->outside_fib_index;
 130   kv0.key = key0.as_u64;
 131
 132   /* NAT packet aimed at external address if */
 133   /* has active sessions */
 134   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 135                               &value0))
 136     {
 137       /* or is static mappings */
 138       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 139         return 0;
 140     }
 141   else
 142     return 0;
 143
 144   if (sm->forwarding_enabled)
 145     return 1;
 146
 147   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 148                                   rx_fib_index0);
 149 }
 150
 151 static inline int
 152 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 153                                   u32 proto0, u16 src_port, u16 dst_port,
 154                                   u32 thread_index, u32 sw_if_index)
 155 {
 156   snat_session_key_t key0;
 157   clib_bihash_kv_8_8_t kv0, value0;
 158   snat_interface_t *i;
 159
 160   /* src NAT check */
 161   key0.addr = ip0->src_address;
 162   key0.port = src_port;
 163   key0.protocol = proto0;
 164   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 165   kv0.key = key0.as_u64;
 166
 167   if (!clib_bihash_search_8_8
 168       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 169     return 1;
 170
 171   /* dst NAT check */
 172   key0.addr = ip0->dst_address;
 173   key0.port = dst_port;
 174   key0.protocol = proto0;
 175   kv0.key = key0.as_u64;
 176   if (!clib_bihash_search_8_8
 177       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 178     {
 179       /* hairpinning */
 180     /* *INDENT-OFF* */
 181     pool_foreach (i, sm->output_feature_interfaces,
 182     ({
 183       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 184         return 0;
 185     }));
 186     /* *INDENT-ON* */
 187       return 1;
 188     }
 189
 190   return 0;
 191 }
 192
 193 int
 194 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 195 {
 196   snat_main_t *sm = &snat_main;
 197   nat44_is_idle_session_ctx_t *ctx = arg;
 198   snat_session_t *s;
 199   u64 sess_timeout_time;
 200   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 201                                                        ctx->thread_index);
 202   clib_bihash_kv_8_8_t s_kv;
 203
 204   s = pool_elt_at_index (tsm->sessions, kv->value);
 205   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 206   if (ctx->now >= sess_timeout_time)
 207     {
 208       s_kv.key = s->out2in.as_u64;
 209       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 210         nat_log_warn ("out2in key del failed");
 211
 212       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 213                                            s->out2in.addr.as_u32,
 214                                            s->in2out.protocol,
 215                                            s->in2out.port,
 216                                            s->out2in.port,
 217                                            s->in2out.fib_index);
 218
 219       if (!snat_is_session_static (s))
 220         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 221                                             &s->out2in);
 222
 223       nat44_delete_session (sm, s, ctx->thread_index);
 224       return 1;
 225     }
 226
 227   return 0;
 228 }
 229
 230 static u32
 231 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 232            ip4_header_t * ip0,
 233            u32 rx_fib_index0,
 234            snat_session_key_t * key0,
 235            snat_session_t ** sessionp,
 236            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 237 {
 238   snat_user_t *u;
 239   snat_session_t *s = 0;
 240   clib_bihash_kv_8_8_t kv0;
 241   snat_session_key_t key1;
 242   udp_header_t *udp0 = ip4_next_header (ip0);
 243   u8 is_sm = 0;
 244   nat_outside_fib_t *outside_fib;
 245   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 246   u8 identity_nat;
 247   fib_prefix_t pfx = {
 248     .fp_proto = FIB_PROTOCOL_IP4,
 249     .fp_len = 32,
 250     .fp_addr = {
 251                 .ip4.as_u32 = ip0->dst_address.as_u32,
 252                 },
 253   };
 254   nat44_is_idle_session_ctx_t ctx0;
 255
 256   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 257     {
 258       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 259       nat_ipfix_logging_max_sessions (sm->max_translations);
 260       nat_log_notice ("maximum sessions exceeded");
 261       return SNAT_IN2OUT_NEXT_DROP;
 262     }
 263
 264   key1.protocol = key0->protocol;
 265
 266   /* First try to match static mapping by local address and port */
 267   if (snat_static_mapping_match
 268       (sm, *key0, &key1, 0, 0, 0, 0, 0, &identity_nat))
 269     {
 270       /* Try to create dynamic translation */
 271       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 272                                                thread_index, &key1,
 273                                                sm->port_per_thread,
 274                                                sm->per_thread_data
 275                                                [thread_index].snat_thread_index))
 276         {
 277           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 278           return SNAT_IN2OUT_NEXT_DROP;
 279         }
 280     }
 281   else
 282     {
 283       if (PREDICT_FALSE (identity_nat))
 284         {
 285           *sessionp = s;
 286           return next0;
 287         }
 288
 289       is_sm = 1;
 290     }
 291
 292   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 293                               thread_index);
 294   if (!u)
 295     {
 296       nat_log_warn ("create NAT user failed");
 297       return SNAT_IN2OUT_NEXT_DROP;
 298     }
 299
 300   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 301   if (!s)
 302     {
 303       nat44_delete_user_with_no_session (sm, u, thread_index);
 304       nat_log_warn ("create NAT session failed");
 305       return SNAT_IN2OUT_NEXT_DROP;
 306     }
 307
 308   if (is_sm)
 309     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 310   user_session_increment (sm, u, is_sm);
 311   s->in2out = *key0;
 312   s->out2in = key1;
 313   s->out2in.protocol = key0->protocol;
 314   s->out2in.fib_index = sm->outside_fib_index;
 315   switch (vec_len (sm->outside_fibs))
 316     {
 317     case 0:
 318       s->out2in.fib_index = sm->outside_fib_index;
 319       break;
 320     case 1:
 321       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 322       break;
 323     default:
 324       /* *INDENT-OFF* */
 325       vec_foreach (outside_fib, sm->outside_fibs)
 326         {
 327           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 328           if (FIB_NODE_INDEX_INVALID != fei)
 329             {
 330               if (fib_entry_get_resolving_interface (fei) != ~0)
 331                 {
 332                   s->out2in.fib_index = outside_fib->fib_index;
 333                   break;
 334                 }
 335             }
 336         }
 337       /* *INDENT-ON* */
 338       break;
 339     }
 340   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 341   s->ext_host_port = udp0->dst_port;
 342   *sessionp = s;
 343
 344   /* Add to translation hashes */
 345   ctx0.now = now;
 346   ctx0.thread_index = thread_index;
 347   kv0.key = s->in2out.as_u64;
 348   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 349   if (clib_bihash_add_or_overwrite_stale_8_8
 350       (&sm->per_thread_data[thread_index].in2out, &kv0,
 351        nat44_i2o_is_idle_session_cb, &ctx0))
 352     nat_log_notice ("in2out key add failed");
 353
 354   kv0.key = s->out2in.as_u64;
 355   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 356
 357   if (clib_bihash_add_or_overwrite_stale_8_8
 358       (&sm->per_thread_data[thread_index].out2in, &kv0,
 359        nat44_o2i_is_idle_session_cb, &ctx0))
 360     nat_log_notice ("out2in key add failed");
 361
 362   /* log NAT event */
 363   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 364                                        s->out2in.addr.as_u32,
 365                                        s->in2out.protocol,
 366                                        s->in2out.port,
 367                                        s->out2in.port, s->in2out.fib_index);
 368   return next0;
 369 }
 370
 371 static_always_inline
 372   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 373                                     snat_session_key_t * p_key0)
 374 {
 375   icmp46_header_t *icmp0;
 376   snat_session_key_t key0;
 377   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 378   ip4_header_t *inner_ip0 = 0;
 379   void *l4_header = 0;
 380   icmp46_header_t *inner_icmp0;
 381
 382   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 383   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 384
 385   if (!icmp_is_error_message (icmp0))
 386     {
 387       key0.protocol = SNAT_PROTOCOL_ICMP;
 388       key0.addr = ip0->src_address;
 389       key0.port = echo0->identifier;
 390     }
 391   else
 392     {
 393       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 394       l4_header = ip4_next_header (inner_ip0);
 395       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 396       key0.addr = inner_ip0->dst_address;
 397       switch (key0.protocol)
 398         {
 399         case SNAT_PROTOCOL_ICMP:
 400           inner_icmp0 = (icmp46_header_t *) l4_header;
 401           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 402           key0.port = inner_echo0->identifier;
 403           break;
 404         case SNAT_PROTOCOL_UDP:
 405         case SNAT_PROTOCOL_TCP:
 406           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 407           break;
 408         default:
 409           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 410         }
 411     }
 412   *p_key0 = key0;
 413   return -1;                    /* success */
 414 }
 415
 416 /**
 417  * Get address and port values to be used for ICMP packet translation
 418  * and create session if needed
 419  *
 420  * @param[in,out] sm             NAT main
 421  * @param[in,out] node           NAT node runtime
 422  * @param[in] thread_index       thread index
 423  * @param[in,out] b0             buffer containing packet to be translated
 424  * @param[out] p_proto           protocol used for matching
 425  * @param[out] p_value           address and port after NAT translation
 426  * @param[out] p_dont_translate  if packet should not be translated
 427  * @param d                      optional parameter
 428  * @param e                      optional parameter
 429  */
 430 u32
 431 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 432                         u32 thread_index, vlib_buffer_t * b0,
 433                         ip4_header_t * ip0, u8 * p_proto,
 434                         snat_session_key_t * p_value,
 435                         u8 * p_dont_translate, void *d, void *e)
 436 {
 437   icmp46_header_t *icmp0;
 438   u32 sw_if_index0;
 439   u32 rx_fib_index0;
 440   snat_session_key_t key0;
 441   snat_session_t *s0 = 0;
 442   u8 dont_translate = 0;
 443   clib_bihash_kv_8_8_t kv0, value0;
 444   u32 next0 = ~0;
 445   int err;
 446
 447   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 448   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 449   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 450
 451   err = icmp_get_key (ip0, &key0);
 452   if (err != -1)
 453     {
 454       b0->error = node->errors[err];
 455       next0 = SNAT_IN2OUT_NEXT_DROP;
 456       goto out;
 457     }
 458   key0.fib_index = rx_fib_index0;
 459
 460   kv0.key = key0.as_u64;
 461
 462   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 463                               &value0))
 464     {
 465       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 466         {
 467           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 468                                                                key0.protocol,
 469                                                                key0.port,
 470                                                                key0.port,
 471                                                                thread_index,
 472                                                                sw_if_index0)))
 473             {
 474               dont_translate = 1;
 475               goto out;
 476             }
 477         }
 478       else
 479         {
 480           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 481                                                  ip0, SNAT_PROTOCOL_ICMP,
 482                                                  rx_fib_index0,
 483                                                  thread_index)))
 484             {
 485               dont_translate = 1;
 486               goto out;
 487             }
 488         }
 489
 490       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 491         {
 492           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 493           next0 = SNAT_IN2OUT_NEXT_DROP;
 494           goto out;
 495         }
 496
 497       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 498                          thread_index, vlib_time_now (sm->vlib_main));
 499
 500       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 501         goto out;
 502
 503       if (!s0)
 504         {
 505           dont_translate = 1;
 506           goto out;
 507         }
 508     }
 509   else
 510     {
 511       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 512                          icmp0->type != ICMP4_echo_reply &&
 513                          !icmp_is_error_message (icmp0)))
 514         {
 515           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 516           next0 = SNAT_IN2OUT_NEXT_DROP;
 517           goto out;
 518         }
 519
 520       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 521                               value0.value);
 522     }
 523
 524 out:
 525   *p_proto = key0.protocol;
 526   if (s0)
 527     *p_value = s0->out2in;
 528   *p_dont_translate = dont_translate;
 529   if (d)
 530     *(snat_session_t **) d = s0;
 531   return next0;
 532 }
 533
 534 /**
 535  * Get address and port values to be used for ICMP packet translation
 536  *
 537  * @param[in] sm                 NAT main
 538  * @param[in,out] node           NAT node runtime
 539  * @param[in] thread_index       thread index
 540  * @param[in,out] b0             buffer containing packet to be translated
 541  * @param[out] p_proto           protocol used for matching
 542  * @param[out] p_value           address and port after NAT translation
 543  * @param[out] p_dont_translate  if packet should not be translated
 544  * @param d                      optional parameter
 545  * @param e                      optional parameter
 546  */
 547 u32
 548 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 549                         u32 thread_index, vlib_buffer_t * b0,
 550                         ip4_header_t * ip0, u8 * p_proto,
 551                         snat_session_key_t * p_value,
 552                         u8 * p_dont_translate, void *d, void *e)
 553 {
 554   icmp46_header_t *icmp0;
 555   u32 sw_if_index0;
 556   u32 rx_fib_index0;
 557   snat_session_key_t key0;
 558   snat_session_key_t sm0;
 559   u8 dont_translate = 0;
 560   u8 is_addr_only;
 561   u32 next0 = ~0;
 562   int err;
 563
 564   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 565   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 566   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 567
 568   err = icmp_get_key (ip0, &key0);
 569   if (err != -1)
 570     {
 571       b0->error = node->errors[err];
 572       next0 = SNAT_IN2OUT_NEXT_DROP;
 573       goto out2;
 574     }
 575   key0.fib_index = rx_fib_index0;
 576
 577   if (snat_static_mapping_match
 578       (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0, 0))
 579     {
 580       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 581                                                   IP_PROTOCOL_ICMP,
 582                                                   rx_fib_index0)))
 583         {
 584           dont_translate = 1;
 585           goto out;
 586         }
 587
 588       if (icmp_is_error_message (icmp0))
 589         {
 590           next0 = SNAT_IN2OUT_NEXT_DROP;
 591           goto out;
 592         }
 593
 594       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 595       next0 = SNAT_IN2OUT_NEXT_DROP;
 596       goto out;
 597     }
 598
 599   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 600                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 601                      !icmp_is_error_message (icmp0)))
 602     {
 603       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 604       next0 = SNAT_IN2OUT_NEXT_DROP;
 605       goto out;
 606     }
 607
 608 out:
 609   *p_value = sm0;
 610 out2:
 611   *p_proto = key0.protocol;
 612   *p_dont_translate = dont_translate;
 613   return next0;
 614 }
 615
 616 u32
 617 icmp_in2out (snat_main_t * sm,
 618              vlib_buffer_t * b0,
 619              ip4_header_t * ip0,
 620              icmp46_header_t * icmp0,
 621              u32 sw_if_index0,
 622              u32 rx_fib_index0,
 623              vlib_node_runtime_t * node,
 624              u32 next0, u32 thread_index, void *d, void *e)
 625 {
 626   snat_session_key_t sm0;
 627   u8 protocol;
 628   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 629   ip4_header_t *inner_ip0;
 630   void *l4_header = 0;
 631   icmp46_header_t *inner_icmp0;
 632   u8 dont_translate;
 633   u32 new_addr0, old_addr0;
 634   u16 old_id0, new_id0;
 635   u16 old_checksum0, new_checksum0;
 636   ip_csum_t sum0;
 637   u16 checksum0;
 638   u32 next0_tmp;
 639
 640   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 641
 642   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 643                                         &protocol, &sm0, &dont_translate, d,
 644                                         e);
 645   if (next0_tmp != ~0)
 646     next0 = next0_tmp;
 647   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 648     goto out;
 649
 650   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 651     {
 652       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 653                                              (u8 *)
 654                                              vlib_buffer_get_current (b0),
 655                                              ntohs (ip0->length) -
 656                                              ip4_header_bytes (ip0), 0);
 657       checksum0 = ~ip_csum_fold (sum0);
 658       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 659         {
 660           next0 = SNAT_IN2OUT_NEXT_DROP;
 661           goto out;
 662         }
 663     }
 664
 665   old_addr0 = ip0->src_address.as_u32;
 666   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 667
 668   sum0 = ip0->checksum;
 669   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 670                          src_address /* changed member */ );
 671   ip0->checksum = ip_csum_fold (sum0);
 672
 673   if (icmp0->checksum == 0)
 674     icmp0->checksum = 0xffff;
 675
 676   if (!icmp_is_error_message (icmp0))
 677     {
 678       new_id0 = sm0.port;
 679       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 680         {
 681           old_id0 = echo0->identifier;
 682           new_id0 = sm0.port;
 683           echo0->identifier = new_id0;
 684
 685           sum0 = icmp0->checksum;
 686           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 687                                  identifier);
 688           icmp0->checksum = ip_csum_fold (sum0);
 689         }
 690     }
 691   else
 692     {
 693       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 694       l4_header = ip4_next_header (inner_ip0);
 695
 696       if (!ip4_header_checksum_is_valid (inner_ip0))
 697         {
 698           next0 = SNAT_IN2OUT_NEXT_DROP;
 699           goto out;
 700         }
 701
 702       /* update inner destination IP address */
 703       old_addr0 = inner_ip0->dst_address.as_u32;
 704       inner_ip0->dst_address = sm0.addr;
 705       new_addr0 = inner_ip0->dst_address.as_u32;
 706       sum0 = icmp0->checksum;
 707       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 708                              dst_address /* changed member */ );
 709       icmp0->checksum = ip_csum_fold (sum0);
 710
 711       /* update inner IP header checksum */
 712       old_checksum0 = inner_ip0->checksum;
 713       sum0 = inner_ip0->checksum;
 714       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 715                              dst_address /* changed member */ );
 716       inner_ip0->checksum = ip_csum_fold (sum0);
 717       new_checksum0 = inner_ip0->checksum;
 718       sum0 = icmp0->checksum;
 719       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 720                              checksum);
 721       icmp0->checksum = ip_csum_fold (sum0);
 722
 723       switch (protocol)
 724         {
 725         case SNAT_PROTOCOL_ICMP:
 726           inner_icmp0 = (icmp46_header_t *) l4_header;
 727           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 728
 729           old_id0 = inner_echo0->identifier;
 730           new_id0 = sm0.port;
 731           inner_echo0->identifier = new_id0;
 732
 733           sum0 = icmp0->checksum;
 734           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 735                                  identifier);
 736           icmp0->checksum = ip_csum_fold (sum0);
 737           break;
 738         case SNAT_PROTOCOL_UDP:
 739         case SNAT_PROTOCOL_TCP:
 740           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 741           new_id0 = sm0.port;
 742           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 743
 744           sum0 = icmp0->checksum;
 745           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 746                                  dst_port);
 747           icmp0->checksum = ip_csum_fold (sum0);
 748           break;
 749         default:
 750           ASSERT (0);
 751         }
 752     }
 753
 754   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 755     {
 756       if (sm->deterministic ||
 757           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 758                                       sm->endpoint_dependent))
 759         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 760     }
 761
 762 out:
 763   return next0;
 764 }
 765
 766 static inline u32
 767 icmp_in2out_slow_path (snat_main_t * sm,
 768                        vlib_buffer_t * b0,
 769                        ip4_header_t * ip0,
 770                        icmp46_header_t * icmp0,
 771                        u32 sw_if_index0,
 772                        u32 rx_fib_index0,
 773                        vlib_node_runtime_t * node,
 774                        u32 next0,
 775                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 776 {
 777   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 778                        next0, thread_index, p_s0, 0);
 779   snat_session_t *s0 = *p_s0;
 780   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 781     {
 782       /* Accounting */
 783       nat44_session_update_counters (s0, now,
 784                                      vlib_buffer_length_in_chain
 785                                      (sm->vlib_main, b0));
 786       /* Per-user LRU list maintenance */
 787       nat44_session_update_lru (sm, s0, thread_index);
 788     }
 789   return next0;
 790 }
 791
 792 static int
 793 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 794                              vlib_buffer_t * b,
 795                              ip4_header_t * ip, u32 rx_fib_index)
 796 {
 797   clib_bihash_kv_8_8_t kv, value;
 798   snat_static_mapping_t *m;
 799   snat_session_key_t m_key;
 800   u32 old_addr, new_addr;
 801   ip_csum_t sum;
 802
 803   m_key.addr = ip->src_address;
 804   m_key.port = 0;
 805   m_key.protocol = 0;
 806   m_key.fib_index = rx_fib_index;
 807   kv.key = m_key.as_u64;
 808   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 809     return 1;
 810
 811   m = pool_elt_at_index (sm->static_mappings, value.value);
 812
 813   old_addr = ip->src_address.as_u32;
 814   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 815   sum = ip->checksum;
 816   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 817   ip->checksum = ip_csum_fold (sum);
 818
 819
 820   /* Hairpinning */
 821   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 822     {
 823       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 824       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 825     }
 826
 827   return 0;
 828 }
 829
 830 static inline uword
 831 snat_in2out_node_fn_inline (vlib_main_t * vm,
 832                             vlib_node_runtime_t * node,
 833                             vlib_frame_t * frame, int is_slow_path,
 834                             int is_output_feature)
 835 {
 836   u32 n_left_from, *from, *to_next;
 837   snat_in2out_next_t next_index;
 838   u32 pkts_processed = 0;
 839   snat_main_t *sm = &snat_main;
 840   f64 now = vlib_time_now (vm);
 841   u32 stats_node_index;
 842   u32 thread_index = vm->thread_index;
 843
 844   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 845     snat_in2out_node.index;
 846
 847   from = vlib_frame_vector_args (frame);
 848   n_left_from = frame->n_vectors;
 849   next_index = node->cached_next_index;
 850
 851   while (n_left_from > 0)
 852     {
 853       u32 n_left_to_next;
 854
 855       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 856
 857       while (n_left_from >= 4 && n_left_to_next >= 2)
 858         {
 859           u32 bi0, bi1;
 860           vlib_buffer_t *b0, *b1;
 861           u32 next0, next1;
 862           u32 sw_if_index0, sw_if_index1;
 863           ip4_header_t *ip0, *ip1;
 864           ip_csum_t sum0, sum1;
 865           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 866           u16 old_port0, new_port0, old_port1, new_port1;
 867           udp_header_t *udp0, *udp1;
 868           tcp_header_t *tcp0, *tcp1;
 869           icmp46_header_t *icmp0, *icmp1;
 870           snat_session_key_t key0, key1;
 871           u32 rx_fib_index0, rx_fib_index1;
 872           u32 proto0, proto1;
 873           snat_session_t *s0 = 0, *s1 = 0;
 874           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 875           u32 iph_offset0 = 0, iph_offset1 = 0;
 876
 877           /* Prefetch next iteration. */
 878           {
 879             vlib_buffer_t *p2, *p3;
 880
 881             p2 = vlib_get_buffer (vm, from[2]);
 882             p3 = vlib_get_buffer (vm, from[3]);
 883
 884             vlib_prefetch_buffer_header (p2, LOAD);
 885             vlib_prefetch_buffer_header (p3, LOAD);
 886
 887             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 888             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 889           }
 890
 891           /* speculatively enqueue b0 and b1 to the current next frame */
 892           to_next[0] = bi0 = from[0];
 893           to_next[1] = bi1 = from[1];
 894           from += 2;
 895           to_next += 2;
 896           n_left_from -= 2;
 897           n_left_to_next -= 2;
 898
 899           b0 = vlib_get_buffer (vm, bi0);
 900           b1 = vlib_get_buffer (vm, bi1);
 901
 902           if (is_output_feature)
 903             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 904
 905           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 906                                   iph_offset0);
 907
 908           udp0 = ip4_next_header (ip0);
 909           tcp0 = (tcp_header_t *) udp0;
 910           icmp0 = (icmp46_header_t *) udp0;
 911
 912           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 913           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 914                                    sw_if_index0);
 915
 916           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 917
 918           if (PREDICT_FALSE (ip0->ttl == 1))
 919             {
 920               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 921               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 922                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 923                                            0);
 924               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 925               goto trace00;
 926             }
 927
 928           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 929
 930           /* Next configured feature, probably ip4-lookup */
 931           if (is_slow_path)
 932             {
 933               if (PREDICT_FALSE (proto0 == ~0))
 934                 {
 935                   if (nat_in2out_sm_unknown_proto
 936                       (sm, b0, ip0, rx_fib_index0))
 937                     {
 938                       next0 = SNAT_IN2OUT_NEXT_DROP;
 939                       b0->error =
 940                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 941                     }
 942                   goto trace00;
 943                 }
 944
 945               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 946                 {
 947                   next0 = icmp_in2out_slow_path
 948                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 949                      node, next0, now, thread_index, &s0);
 950                   goto trace00;
 951                 }
 952             }
 953           else
 954             {
 955               if (PREDICT_FALSE (proto0 == ~0))
 956                 {
 957                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 958                   goto trace00;
 959                 }
 960
 961               if (ip4_is_fragment (ip0))
 962                 {
 963                   next0 = SNAT_IN2OUT_NEXT_REASS;
 964                   goto trace00;
 965                 }
 966
 967               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 968                 {
 969                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 970                   goto trace00;
 971                 }
 972             }
 973
 974           key0.addr = ip0->src_address;
 975           key0.port = udp0->src_port;
 976           key0.protocol = proto0;
 977           key0.fib_index = rx_fib_index0;
 978
 979           kv0.key = key0.as_u64;
 980
 981           if (PREDICT_FALSE
 982               (clib_bihash_search_8_8
 983                (&sm->per_thread_data[thread_index].in2out, &kv0,
 984                 &value0) != 0))
 985             {
 986               if (is_slow_path)
 987                 {
 988                   if (is_output_feature)
 989                     {
 990                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
 991                                                                            ip0,
 992                                                                            proto0,
 993                                                                            udp0->src_port,
 994                                                                            udp0->dst_port,
 995                                                                            thread_index,
 996                                                                            sw_if_index0)))
 997                         goto trace00;
 998                     }
 999                   else
1000                     {
1001                       if (PREDICT_FALSE
1002                           (snat_not_translate
1003                            (sm, node, sw_if_index0, ip0, proto0,
1004                             rx_fib_index0, thread_index)))
1005                         goto trace00;
1006                     }
1007
1008                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1009                                      &s0, node, next0, thread_index, now);
1010                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1011                     goto trace00;
1012
1013                   if (PREDICT_FALSE (!s0))
1014                     goto trace00;
1015                 }
1016               else
1017                 {
1018                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1019                   goto trace00;
1020                 }
1021             }
1022           else
1023             s0 =
1024               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1025                                  value0.value);
1026
1027           b0->flags |= VNET_BUFFER_F_IS_NATED;
1028
1029           old_addr0 = ip0->src_address.as_u32;
1030           ip0->src_address = s0->out2in.addr;
1031           new_addr0 = ip0->src_address.as_u32;
1032           if (!is_output_feature)
1033             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1034
1035           sum0 = ip0->checksum;
1036           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1037                                  ip4_header_t,
1038                                  src_address /* changed member */ );
1039           ip0->checksum = ip_csum_fold (sum0);
1040
1041           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1042             {
1043               old_port0 = tcp0->src_port;
1044               tcp0->src_port = s0->out2in.port;
1045               new_port0 = tcp0->src_port;
1046
1047               sum0 = tcp0->checksum;
1048               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1049                                      ip4_header_t,
1050                                      dst_address /* changed member */ );
1051               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1052                                      ip4_header_t /* cheat */ ,
1053                                      length /* changed member */ );
1054               mss_clamping (sm, tcp0, &sum0);
1055               tcp0->checksum = ip_csum_fold (sum0);
1056             }
1057           else
1058             {
1059               old_port0 = udp0->src_port;
1060               udp0->src_port = s0->out2in.port;
1061               udp0->checksum = 0;
1062             }
1063
1064           /* Accounting */
1065           nat44_session_update_counters (s0, now,
1066                                          vlib_buffer_length_in_chain (vm,
1067                                                                       b0));
1068           /* Per-user LRU list maintenance */
1069           nat44_session_update_lru (sm, s0, thread_index);
1070         trace00:
1071
1072           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1073                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1074             {
1075               snat_in2out_trace_t *t =
1076                 vlib_add_trace (vm, node, b0, sizeof (*t));
1077               t->is_slow_path = is_slow_path;
1078               t->sw_if_index = sw_if_index0;
1079               t->next_index = next0;
1080               t->session_index = ~0;
1081               if (s0)
1082                 t->session_index =
1083                   s0 - sm->per_thread_data[thread_index].sessions;
1084             }
1085
1086           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1087
1088           if (is_output_feature)
1089             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1090
1091           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1092                                   iph_offset1);
1093
1094           udp1 = ip4_next_header (ip1);
1095           tcp1 = (tcp_header_t *) udp1;
1096           icmp1 = (icmp46_header_t *) udp1;
1097
1098           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1099           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1100                                    sw_if_index1);
1101
1102           if (PREDICT_FALSE (ip1->ttl == 1))
1103             {
1104               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1105               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1106                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1107                                            0);
1108               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1109               goto trace01;
1110             }
1111
1112           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1113
1114           /* Next configured feature, probably ip4-lookup */
1115           if (is_slow_path)
1116             {
1117               if (PREDICT_FALSE (proto1 == ~0))
1118                 {
1119                   if (nat_in2out_sm_unknown_proto
1120                       (sm, b1, ip1, rx_fib_index1))
1121                     {
1122                       next1 = SNAT_IN2OUT_NEXT_DROP;
1123                       b1->error =
1124                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1125                     }
1126                   goto trace01;
1127                 }
1128
1129               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1130                 {
1131                   next1 = icmp_in2out_slow_path
1132                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1133                      next1, now, thread_index, &s1);
1134                   goto trace01;
1135                 }
1136             }
1137           else
1138             {
1139               if (PREDICT_FALSE (proto1 == ~0))
1140                 {
1141                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1142                   goto trace01;
1143                 }
1144
1145               if (ip4_is_fragment (ip1))
1146                 {
1147                   next1 = SNAT_IN2OUT_NEXT_REASS;
1148                   goto trace01;
1149                 }
1150
1151               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1152                 {
1153                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1154                   goto trace01;
1155                 }
1156             }
1157
1158           key1.addr = ip1->src_address;
1159           key1.port = udp1->src_port;
1160           key1.protocol = proto1;
1161           key1.fib_index = rx_fib_index1;
1162
1163           kv1.key = key1.as_u64;
1164
1165           if (PREDICT_FALSE
1166               (clib_bihash_search_8_8
1167                (&sm->per_thread_data[thread_index].in2out, &kv1,
1168                 &value1) != 0))
1169             {
1170               if (is_slow_path)
1171                 {
1172                   if (is_output_feature)
1173                     {
1174                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1175                                                                            ip1,
1176                                                                            proto1,
1177                                                                            udp1->src_port,
1178                                                                            udp1->dst_port,
1179                                                                            thread_index,
1180                                                                            sw_if_index1)))
1181                         goto trace01;
1182                     }
1183                   else
1184                     {
1185                       if (PREDICT_FALSE
1186                           (snat_not_translate
1187                            (sm, node, sw_if_index1, ip1, proto1,
1188                             rx_fib_index1, thread_index)))
1189                         goto trace01;
1190                     }
1191
1192                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1193                                      &s1, node, next1, thread_index, now);
1194                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1195                     goto trace01;
1196
1197                   if (PREDICT_FALSE (!s1))
1198                     goto trace01;
1199                 }
1200               else
1201                 {
1202                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1203                   goto trace01;
1204                 }
1205             }
1206           else
1207             s1 =
1208               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1209                                  value1.value);
1210
1211           b1->flags |= VNET_BUFFER_F_IS_NATED;
1212
1213           old_addr1 = ip1->src_address.as_u32;
1214           ip1->src_address = s1->out2in.addr;
1215           new_addr1 = ip1->src_address.as_u32;
1216           if (!is_output_feature)
1217             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1218
1219           sum1 = ip1->checksum;
1220           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1221                                  ip4_header_t,
1222                                  src_address /* changed member */ );
1223           ip1->checksum = ip_csum_fold (sum1);
1224
1225           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1226             {
1227               old_port1 = tcp1->src_port;
1228               tcp1->src_port = s1->out2in.port;
1229               new_port1 = tcp1->src_port;
1230
1231               sum1 = tcp1->checksum;
1232               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1233                                      ip4_header_t,
1234                                      dst_address /* changed member */ );
1235               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1236                                      ip4_header_t /* cheat */ ,
1237                                      length /* changed member */ );
1238               mss_clamping (sm, tcp1, &sum1);
1239               tcp1->checksum = ip_csum_fold (sum1);
1240             }
1241           else
1242             {
1243               old_port1 = udp1->src_port;
1244               udp1->src_port = s1->out2in.port;
1245               udp1->checksum = 0;
1246             }
1247
1248           /* Accounting */
1249           nat44_session_update_counters (s1, now,
1250                                          vlib_buffer_length_in_chain (vm,
1251                                                                       b1));
1252           /* Per-user LRU list maintenance */
1253           nat44_session_update_lru (sm, s1, thread_index);
1254         trace01:
1255
1256           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1257                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1258             {
1259               snat_in2out_trace_t *t =
1260                 vlib_add_trace (vm, node, b1, sizeof (*t));
1261               t->sw_if_index = sw_if_index1;
1262               t->next_index = next1;
1263               t->session_index = ~0;
1264               if (s1)
1265                 t->session_index =
1266                   s1 - sm->per_thread_data[thread_index].sessions;
1267             }
1268
1269           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1270
1271           /* verify speculative enqueues, maybe switch current next frame */
1272           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1273                                            to_next, n_left_to_next,
1274                                            bi0, bi1, next0, next1);
1275         }
1276
1277       while (n_left_from > 0 && n_left_to_next > 0)
1278         {
1279           u32 bi0;
1280           vlib_buffer_t *b0;
1281           u32 next0;
1282           u32 sw_if_index0;
1283           ip4_header_t *ip0;
1284           ip_csum_t sum0;
1285           u32 new_addr0, old_addr0;
1286           u16 old_port0, new_port0;
1287           udp_header_t *udp0;
1288           tcp_header_t *tcp0;
1289           icmp46_header_t *icmp0;
1290           snat_session_key_t key0;
1291           u32 rx_fib_index0;
1292           u32 proto0;
1293           snat_session_t *s0 = 0;
1294           clib_bihash_kv_8_8_t kv0, value0;
1295           u32 iph_offset0 = 0;
1296
1297           /* speculatively enqueue b0 to the current next frame */
1298           bi0 = from[0];
1299           to_next[0] = bi0;
1300           from += 1;
1301           to_next += 1;
1302           n_left_from -= 1;
1303           n_left_to_next -= 1;
1304
1305           b0 = vlib_get_buffer (vm, bi0);
1306           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1307
1308           if (is_output_feature)
1309             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1310
1311           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1312                                   iph_offset0);
1313
1314           udp0 = ip4_next_header (ip0);
1315           tcp0 = (tcp_header_t *) udp0;
1316           icmp0 = (icmp46_header_t *) udp0;
1317
1318           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1319           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1320                                    sw_if_index0);
1321
1322           if (PREDICT_FALSE (ip0->ttl == 1))
1323             {
1324               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1325               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1326                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1327                                            0);
1328               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1329               goto trace0;
1330             }
1331
1332           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1333
1334           /* Next configured feature, probably ip4-lookup */
1335           if (is_slow_path)
1336             {
1337               if (PREDICT_FALSE (proto0 == ~0))
1338                 {
1339                   if (nat_in2out_sm_unknown_proto
1340                       (sm, b0, ip0, rx_fib_index0))
1341                     {
1342                       next0 = SNAT_IN2OUT_NEXT_DROP;
1343                       b0->error =
1344                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1345                     }
1346                   goto trace0;
1347                 }
1348
1349               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1350                 {
1351                   next0 = icmp_in2out_slow_path
1352                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1353                      next0, now, thread_index, &s0);
1354                   goto trace0;
1355                 }
1356             }
1357           else
1358             {
1359               if (PREDICT_FALSE (proto0 == ~0))
1360                 {
1361                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1362                   goto trace0;
1363                 }
1364
1365               if (ip4_is_fragment (ip0))
1366                 {
1367                   next0 = SNAT_IN2OUT_NEXT_REASS;
1368                   goto trace0;
1369                 }
1370
1371               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1372                 {
1373                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1374                   goto trace0;
1375                 }
1376             }
1377
1378           key0.addr = ip0->src_address;
1379           key0.port = udp0->src_port;
1380           key0.protocol = proto0;
1381           key0.fib_index = rx_fib_index0;
1382
1383           kv0.key = key0.as_u64;
1384
1385           if (clib_bihash_search_8_8
1386               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1387             {
1388               if (is_slow_path)
1389                 {
1390                   if (is_output_feature)
1391                     {
1392                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1393                                                                            ip0,
1394                                                                            proto0,
1395                                                                            udp0->src_port,
1396                                                                            udp0->dst_port,
1397                                                                            thread_index,
1398                                                                            sw_if_index0)))
1399                         goto trace0;
1400                     }
1401                   else
1402                     {
1403                       if (PREDICT_FALSE
1404                           (snat_not_translate
1405                            (sm, node, sw_if_index0, ip0, proto0,
1406                             rx_fib_index0, thread_index)))
1407                         goto trace0;
1408                     }
1409
1410                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1411                                      &s0, node, next0, thread_index, now);
1412
1413                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1414                     goto trace0;
1415
1416                   if (PREDICT_FALSE (!s0))
1417                     goto trace0;
1418                 }
1419               else
1420                 {
1421                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1422                   goto trace0;
1423                 }
1424             }
1425           else
1426             s0 =
1427               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1428                                  value0.value);
1429
1430           b0->flags |= VNET_BUFFER_F_IS_NATED;
1431
1432           old_addr0 = ip0->src_address.as_u32;
1433           ip0->src_address = s0->out2in.addr;
1434           new_addr0 = ip0->src_address.as_u32;
1435           if (!is_output_feature)
1436             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1437
1438           sum0 = ip0->checksum;
1439           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1440                                  ip4_header_t,
1441                                  src_address /* changed member */ );
1442           ip0->checksum = ip_csum_fold (sum0);
1443
1444           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1445             {
1446               old_port0 = tcp0->src_port;
1447               tcp0->src_port = s0->out2in.port;
1448               new_port0 = tcp0->src_port;
1449
1450               sum0 = tcp0->checksum;
1451               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1452                                      ip4_header_t,
1453                                      dst_address /* changed member */ );
1454               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1455                                      ip4_header_t /* cheat */ ,
1456                                      length /* changed member */ );
1457               mss_clamping (sm, tcp0, &sum0);
1458               tcp0->checksum = ip_csum_fold (sum0);
1459             }
1460           else
1461             {
1462               old_port0 = udp0->src_port;
1463               udp0->src_port = s0->out2in.port;
1464               udp0->checksum = 0;
1465             }
1466
1467           /* Accounting */
1468           nat44_session_update_counters (s0, now,
1469                                          vlib_buffer_length_in_chain (vm,
1470                                                                       b0));
1471           /* Per-user LRU list maintenance */
1472           nat44_session_update_lru (sm, s0, thread_index);
1473
1474         trace0:
1475           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1476                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1477             {
1478               snat_in2out_trace_t *t =
1479                 vlib_add_trace (vm, node, b0, sizeof (*t));
1480               t->is_slow_path = is_slow_path;
1481               t->sw_if_index = sw_if_index0;
1482               t->next_index = next0;
1483               t->session_index = ~0;
1484               if (s0)
1485                 t->session_index =
1486                   s0 - sm->per_thread_data[thread_index].sessions;
1487             }
1488
1489           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1490
1491           /* verify speculative enqueue, maybe switch current next frame */
1492           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1493                                            to_next, n_left_to_next,
1494                                            bi0, next0);
1495         }
1496
1497       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1498     }
1499
1500   vlib_node_increment_counter (vm, stats_node_index,
1501                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1502                                pkts_processed);
1503   return frame->n_vectors;
1504 }
1505
1506 static uword
1507 snat_in2out_fast_path_fn (vlib_main_t * vm,
1508                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1509 {
1510   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1511                                      0);
1512 }
1513
1514 /* *INDENT-OFF* */
1515 VLIB_REGISTER_NODE (snat_in2out_node) = {
1516   .function = snat_in2out_fast_path_fn,
1517   .name = "nat44-in2out",
1518   .vector_size = sizeof (u32),
1519   .format_trace = format_snat_in2out_trace,
1520   .type = VLIB_NODE_TYPE_INTERNAL,
1521
1522   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1523   .error_strings = snat_in2out_error_strings,
1524
1525   .runtime_data_bytes = sizeof (snat_runtime_t),
1526
1527   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1528
1529   /* edit / add dispositions here */
1530   .next_nodes = {
1531     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1532     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1533     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1534     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1535     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1536   },
1537 };
1538 /* *INDENT-ON* */
1539
1540 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1541
1542 static uword
1543 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1544                                  vlib_node_runtime_t * node,
1545                                  vlib_frame_t * frame)
1546 {
1547   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1548                                      1);
1549 }
1550
1551 /* *INDENT-OFF* */
1552 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1553   .function = snat_in2out_output_fast_path_fn,
1554   .name = "nat44-in2out-output",
1555   .vector_size = sizeof (u32),
1556   .format_trace = format_snat_in2out_trace,
1557   .type = VLIB_NODE_TYPE_INTERNAL,
1558
1559   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1560   .error_strings = snat_in2out_error_strings,
1561
1562   .runtime_data_bytes = sizeof (snat_runtime_t),
1563
1564   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1565
1566   /* edit / add dispositions here */
1567   .next_nodes = {
1568     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1569     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1570     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1571     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1572     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1573   },
1574 };
1575 /* *INDENT-ON* */
1576
1577 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1578                               snat_in2out_output_fast_path_fn);
1579
1580 static uword
1581 snat_in2out_slow_path_fn (vlib_main_t * vm,
1582                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1583 {
1584   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1585                                      0);
1586 }
1587
1588 /* *INDENT-OFF* */
1589 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1590   .function = snat_in2out_slow_path_fn,
1591   .name = "nat44-in2out-slowpath",
1592   .vector_size = sizeof (u32),
1593   .format_trace = format_snat_in2out_trace,
1594   .type = VLIB_NODE_TYPE_INTERNAL,
1595
1596   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1597   .error_strings = snat_in2out_error_strings,
1598
1599   .runtime_data_bytes = sizeof (snat_runtime_t),
1600
1601   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1602
1603   /* edit / add dispositions here */
1604   .next_nodes = {
1605     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1606     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1607     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1608     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1609     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1610   },
1611 };
1612 /* *INDENT-ON* */
1613
1614 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1615                               snat_in2out_slow_path_fn);
1616
1617 static uword
1618 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1619                                  vlib_node_runtime_t * node,
1620                                  vlib_frame_t * frame)
1621 {
1622   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1623                                      1);
1624 }
1625
1626 /* *INDENT-OFF* */
1627 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1628   .function = snat_in2out_output_slow_path_fn,
1629   .name = "nat44-in2out-output-slowpath",
1630   .vector_size = sizeof (u32),
1631   .format_trace = format_snat_in2out_trace,
1632   .type = VLIB_NODE_TYPE_INTERNAL,
1633
1634   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1635   .error_strings = snat_in2out_error_strings,
1636
1637   .runtime_data_bytes = sizeof (snat_runtime_t),
1638
1639   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1640
1641   /* edit / add dispositions here */
1642   .next_nodes = {
1643     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1644     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1645     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1646     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1647     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1648   },
1649 };
1650 /* *INDENT-ON* */
1651
1652 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1653                               snat_in2out_output_slow_path_fn);
1654
1655 static uword
1656 nat44_in2out_reass_node_fn (vlib_main_t * vm,
1657                             vlib_node_runtime_t * node, vlib_frame_t * frame)
1658 {
1659   u32 n_left_from, *from, *to_next;
1660   snat_in2out_next_t next_index;
1661   u32 pkts_processed = 0;
1662   snat_main_t *sm = &snat_main;
1663   f64 now = vlib_time_now (vm);
1664   u32 thread_index = vm->thread_index;
1665   snat_main_per_thread_data_t *per_thread_data =
1666     &sm->per_thread_data[thread_index];
1667   u32 *fragments_to_drop = 0;
1668   u32 *fragments_to_loopback = 0;
1669
1670   from = vlib_frame_vector_args (frame);
1671   n_left_from = frame->n_vectors;
1672   next_index = node->cached_next_index;
1673
1674   while (n_left_from > 0)
1675     {
1676       u32 n_left_to_next;
1677
1678       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1679
1680       while (n_left_from > 0 && n_left_to_next > 0)
1681         {
1682           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1683           vlib_buffer_t *b0;
1684           u32 next0;
1685           u8 cached0 = 0;
1686           ip4_header_t *ip0;
1687           nat_reass_ip4_t *reass0;
1688           udp_header_t *udp0;
1689           tcp_header_t *tcp0;
1690           icmp46_header_t *icmp0;
1691           snat_session_key_t key0;
1692           clib_bihash_kv_8_8_t kv0, value0;
1693           snat_session_t *s0 = 0;
1694           u16 old_port0, new_port0;
1695           ip_csum_t sum0;
1696
1697           /* speculatively enqueue b0 to the current next frame */
1698           bi0 = from[0];
1699           to_next[0] = bi0;
1700           from += 1;
1701           to_next += 1;
1702           n_left_from -= 1;
1703           n_left_to_next -= 1;
1704
1705           b0 = vlib_get_buffer (vm, bi0);
1706           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1707
1708           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1709           rx_fib_index0 =
1710             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1711                                                  sw_if_index0);
1712
1713           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1714             {
1715               next0 = SNAT_IN2OUT_NEXT_DROP;
1716               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1717               goto trace0;
1718             }
1719
1720           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1721           udp0 = ip4_next_header (ip0);
1722           tcp0 = (tcp_header_t *) udp0;
1723           icmp0 = (icmp46_header_t *) udp0;
1724           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1725
1726           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1727                                                  ip0->dst_address,
1728                                                  ip0->fragment_id,
1729                                                  ip0->protocol,
1730                                                  1, &fragments_to_drop);
1731
1732           if (PREDICT_FALSE (!reass0))
1733             {
1734               next0 = SNAT_IN2OUT_NEXT_DROP;
1735               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1736               nat_log_notice ("maximum reassemblies exceeded");
1737               goto trace0;
1738             }
1739
1740           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1741             {
1742               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1743                 {
1744                   next0 = icmp_in2out_slow_path
1745                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1746                      next0, now, thread_index, &s0);
1747
1748                   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP))
1749                     {
1750                       if (s0)
1751                         reass0->sess_index = s0 - per_thread_data->sessions;
1752                       else
1753                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1754                       nat_ip4_reass_get_frags (reass0,
1755                                                &fragments_to_loopback);
1756                     }
1757
1758                   goto trace0;
1759                 }
1760
1761               key0.addr = ip0->src_address;
1762               key0.port = udp0->src_port;
1763               key0.protocol = proto0;
1764               key0.fib_index = rx_fib_index0;
1765               kv0.key = key0.as_u64;
1766
1767               if (clib_bihash_search_8_8
1768                   (&per_thread_data->in2out, &kv0, &value0))
1769                 {
1770                   if (PREDICT_FALSE
1771                       (snat_not_translate
1772                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1773                         thread_index)))
1774                     goto trace0;
1775
1776                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1777                                      &s0, node, next0, thread_index, now);
1778
1779                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1780                     goto trace0;
1781
1782                   if (PREDICT_FALSE (!s0))
1783                     goto trace0;
1784
1785                   reass0->sess_index = s0 - per_thread_data->sessions;
1786                 }
1787               else
1788                 {
1789                   s0 = pool_elt_at_index (per_thread_data->sessions,
1790                                           value0.value);
1791                   reass0->sess_index = value0.value;
1792                 }
1793               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1794             }
1795           else
1796             {
1797               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1798                 {
1799                   if (nat_ip4_reass_add_fragment
1800                       (reass0, bi0, &fragments_to_drop))
1801                     {
1802                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1803                       nat_log_notice
1804                         ("maximum fragments per reassembly exceeded");
1805                       next0 = SNAT_IN2OUT_NEXT_DROP;
1806                       goto trace0;
1807                     }
1808                   cached0 = 1;
1809                   goto trace0;
1810                 }
1811               s0 = pool_elt_at_index (per_thread_data->sessions,
1812                                       reass0->sess_index);
1813             }
1814
1815           old_addr0 = ip0->src_address.as_u32;
1816           ip0->src_address = s0->out2in.addr;
1817           new_addr0 = ip0->src_address.as_u32;
1818           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1819
1820           sum0 = ip0->checksum;
1821           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1822                                  ip4_header_t,
1823                                  src_address /* changed member */ );
1824           ip0->checksum = ip_csum_fold (sum0);
1825
1826           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1827             {
1828               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1829                 {
1830                   old_port0 = tcp0->src_port;
1831                   tcp0->src_port = s0->out2in.port;
1832                   new_port0 = tcp0->src_port;
1833
1834                   sum0 = tcp0->checksum;
1835                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1836                                          ip4_header_t,
1837                                          dst_address /* changed member */ );
1838                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1839                                          ip4_header_t /* cheat */ ,
1840                                          length /* changed member */ );
1841                   tcp0->checksum = ip_csum_fold (sum0);
1842                 }
1843               else
1844                 {
1845                   old_port0 = udp0->src_port;
1846                   udp0->src_port = s0->out2in.port;
1847                   udp0->checksum = 0;
1848                 }
1849             }
1850
1851           /* Hairpinning */
1852           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1853                                    s0->ext_host_port, proto0, 0);
1854
1855           /* Accounting */
1856           nat44_session_update_counters (s0, now,
1857                                          vlib_buffer_length_in_chain (vm,
1858                                                                       b0));
1859           /* Per-user LRU list maintenance */
1860           nat44_session_update_lru (sm, s0, thread_index);
1861
1862         trace0:
1863           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1864                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1865             {
1866               nat44_reass_trace_t *t =
1867                 vlib_add_trace (vm, node, b0, sizeof (*t));
1868               t->cached = cached0;
1869               t->sw_if_index = sw_if_index0;
1870               t->next_index = next0;
1871             }
1872
1873           if (cached0)
1874             {
1875               n_left_to_next++;
1876               to_next--;
1877             }
1878           else
1879             {
1880               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1881
1882               /* verify speculative enqueue, maybe switch current next frame */
1883               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1884                                                to_next, n_left_to_next,
1885                                                bi0, next0);
1886             }
1887
1888           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1889             {
1890               from = vlib_frame_vector_args (frame);
1891               u32 len = vec_len (fragments_to_loopback);
1892               if (len <= VLIB_FRAME_SIZE)
1893                 {
1894                   clib_memcpy_fast (from, fragments_to_loopback,
1895                                     sizeof (u32) * len);
1896                   n_left_from = len;
1897                   vec_reset_length (fragments_to_loopback);
1898                 }
1899               else
1900                 {
1901                   clib_memcpy_fast (from, fragments_to_loopback +
1902                                     (len - VLIB_FRAME_SIZE),
1903                                     sizeof (u32) * VLIB_FRAME_SIZE);
1904                   n_left_from = VLIB_FRAME_SIZE;
1905                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1906                 }
1907             }
1908         }
1909
1910       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1911     }
1912
1913   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
1914                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1915                                pkts_processed);
1916
1917   nat_send_all_to_node (vm, fragments_to_drop, node,
1918                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1919                         SNAT_IN2OUT_NEXT_DROP);
1920
1921   vec_free (fragments_to_drop);
1922   vec_free (fragments_to_loopback);
1923   return frame->n_vectors;
1924 }
1925
1926 /* *INDENT-OFF* */
1927 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
1928   .function = nat44_in2out_reass_node_fn,
1929   .name = "nat44-in2out-reass",
1930   .vector_size = sizeof (u32),
1931   .format_trace = format_nat44_reass_trace,
1932   .type = VLIB_NODE_TYPE_INTERNAL,
1933
1934   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1935   .error_strings = snat_in2out_error_strings,
1936
1937   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1938   .next_nodes = {
1939     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1940     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1941     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1942     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1943     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1944   },
1945 };
1946 /* *INDENT-ON* */
1947
1948 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
1949                               nat44_in2out_reass_node_fn);
1950
1951 static uword
1952 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1953                                 vlib_node_runtime_t * node,
1954                                 vlib_frame_t * frame)
1955 {
1956   u32 n_left_from, *from, *to_next;
1957   snat_in2out_next_t next_index;
1958   u32 pkts_processed = 0;
1959   snat_main_t *sm = &snat_main;
1960   u32 stats_node_index;
1961
1962   stats_node_index = snat_in2out_fast_node.index;
1963
1964   from = vlib_frame_vector_args (frame);
1965   n_left_from = frame->n_vectors;
1966   next_index = node->cached_next_index;
1967
1968   while (n_left_from > 0)
1969     {
1970       u32 n_left_to_next;
1971
1972       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1973
1974       while (n_left_from > 0 && n_left_to_next > 0)
1975         {
1976           u32 bi0;
1977           vlib_buffer_t *b0;
1978           u32 next0;
1979           u32 sw_if_index0;
1980           ip4_header_t *ip0;
1981           ip_csum_t sum0;
1982           u32 new_addr0, old_addr0;
1983           u16 old_port0, new_port0;
1984           udp_header_t *udp0;
1985           tcp_header_t *tcp0;
1986           icmp46_header_t *icmp0;
1987           snat_session_key_t key0, sm0;
1988           u32 proto0;
1989           u32 rx_fib_index0;
1990
1991           /* speculatively enqueue b0 to the current next frame */
1992           bi0 = from[0];
1993           to_next[0] = bi0;
1994           from += 1;
1995           to_next += 1;
1996           n_left_from -= 1;
1997           n_left_to_next -= 1;
1998
1999           b0 = vlib_get_buffer (vm, bi0);
2000           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2001
2002           ip0 = vlib_buffer_get_current (b0);
2003           udp0 = ip4_next_header (ip0);
2004           tcp0 = (tcp_header_t *) udp0;
2005           icmp0 = (icmp46_header_t *) udp0;
2006
2007           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2008           rx_fib_index0 =
2009             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2010
2011           if (PREDICT_FALSE (ip0->ttl == 1))
2012             {
2013               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2014               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2015                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2016                                            0);
2017               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2018               goto trace0;
2019             }
2020
2021           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2022
2023           if (PREDICT_FALSE (proto0 == ~0))
2024             goto trace0;
2025
2026           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2027             {
2028               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
2029                                    rx_fib_index0, node, next0, ~0, 0, 0);
2030               goto trace0;
2031             }
2032
2033           key0.addr = ip0->src_address;
2034           key0.protocol = proto0;
2035           key0.port = udp0->src_port;
2036           key0.fib_index = rx_fib_index0;
2037
2038           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0, 0))
2039             {
2040               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2041               next0 = SNAT_IN2OUT_NEXT_DROP;
2042               goto trace0;
2043             }
2044
2045           new_addr0 = sm0.addr.as_u32;
2046           new_port0 = sm0.port;
2047           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2048           old_addr0 = ip0->src_address.as_u32;
2049           ip0->src_address.as_u32 = new_addr0;
2050
2051           sum0 = ip0->checksum;
2052           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2053                                  ip4_header_t,
2054                                  src_address /* changed member */ );
2055           ip0->checksum = ip_csum_fold (sum0);
2056
2057           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
2058             {
2059               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2060                 {
2061                   old_port0 = tcp0->src_port;
2062                   tcp0->src_port = new_port0;
2063
2064                   sum0 = tcp0->checksum;
2065                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2066                                          ip4_header_t,
2067                                          dst_address /* changed member */ );
2068                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2069                                          ip4_header_t /* cheat */ ,
2070                                          length /* changed member */ );
2071                   mss_clamping (sm, tcp0, &sum0);
2072                   tcp0->checksum = ip_csum_fold (sum0);
2073                 }
2074               else
2075                 {
2076                   old_port0 = udp0->src_port;
2077                   udp0->src_port = new_port0;
2078                   udp0->checksum = 0;
2079                 }
2080             }
2081           else
2082             {
2083               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2084                 {
2085                   sum0 = tcp0->checksum;
2086                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2087                                          ip4_header_t,
2088                                          dst_address /* changed member */ );
2089                   mss_clamping (sm, tcp0, &sum0);
2090                   tcp0->checksum = ip_csum_fold (sum0);
2091                 }
2092             }
2093
2094           /* Hairpinning */
2095           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2096
2097         trace0:
2098           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2099                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2100             {
2101               snat_in2out_trace_t *t =
2102                 vlib_add_trace (vm, node, b0, sizeof (*t));
2103               t->sw_if_index = sw_if_index0;
2104               t->next_index = next0;
2105             }
2106
2107           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2108
2109           /* verify speculative enqueue, maybe switch current next frame */
2110           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2111                                            to_next, n_left_to_next,
2112                                            bi0, next0);
2113         }
2114
2115       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2116     }
2117
2118   vlib_node_increment_counter (vm, stats_node_index,
2119                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2120                                pkts_processed);
2121   return frame->n_vectors;
2122 }
2123
2124
2125 /* *INDENT-OFF* */
2126 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2127   .function = snat_in2out_fast_static_map_fn,
2128   .name = "nat44-in2out-fast",
2129   .vector_size = sizeof (u32),
2130   .format_trace = format_snat_in2out_fast_trace,
2131   .type = VLIB_NODE_TYPE_INTERNAL,
2132
2133   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2134   .error_strings = snat_in2out_error_strings,
2135
2136   .runtime_data_bytes = sizeof (snat_runtime_t),
2137
2138   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2139
2140   /* edit / add dispositions here */
2141   .next_nodes = {
2142     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2143     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2144     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2145     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2146     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2147   },
2148 };
2149 /* *INDENT-ON* */
2150
2151 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node,
2152                               snat_in2out_fast_static_map_fn);
2153
2154 /*
2155  * fd.io coding-style-patch-verification: ON
2156  *
2157  * Local Variables:
2158  * eval: (c-set-style "gnu")
2159  * End:
2160  */