src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #include <vppinfra/hash.h>
  33 #include <vppinfra/error.h>
  34 #include <vppinfra/elog.h>
  35
  36 typedef struct
  37 {
  38   u32 sw_if_index;
  39   u32 next_index;
  40   u32 session_index;
  41   u32 is_slow_path;
  42 } snat_in2out_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 *
  46 format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  51   char *tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 *
  62 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 vlib_node_registration_t snat_in2out_node;
  75 vlib_node_registration_t snat_in2out_slowpath_node;
  76 vlib_node_registration_t snat_in2out_fast_node;
  77 vlib_node_registration_t snat_in2out_output_node;
  78 vlib_node_registration_t snat_in2out_output_slowpath_node;
  79 vlib_node_registration_t nat44_in2out_reass_node;
  80
  81 #define foreach_snat_in2out_error                       \
  82 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  83 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  84 _(OUT_OF_PORTS, "Out of ports")                         \
  85 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  86 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  87 _(NO_TRANSLATION, "No translation")                     \
  88 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  89 _(DROP_FRAGMENT, "Drop fragment")                       \
  90 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  91 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  92
  93 typedef enum
  94 {
  95 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  96   foreach_snat_in2out_error
  97 #undef _
  98     SNAT_IN2OUT_N_ERROR,
  99 } snat_in2out_error_t;
 100
 101 static char *snat_in2out_error_strings[] = {
 102 #define _(sym,string) string,
 103   foreach_snat_in2out_error
 104 #undef _
 105 };
 106
 107 typedef enum
 108 {
 109   SNAT_IN2OUT_NEXT_LOOKUP,
 110   SNAT_IN2OUT_NEXT_DROP,
 111   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 112   SNAT_IN2OUT_NEXT_SLOW_PATH,
 113   SNAT_IN2OUT_NEXT_REASS,
 114   SNAT_IN2OUT_N_NEXT,
 115 } snat_in2out_next_t;
 116
 117 static inline int
 118 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 119                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 120                     u32 rx_fib_index0, u32 thread_index)
 121 {
 122   udp_header_t *udp0 = ip4_next_header (ip0);
 123   snat_session_key_t key0, sm0;
 124   clib_bihash_kv_8_8_t kv0, value0;
 125
 126   key0.addr = ip0->dst_address;
 127   key0.port = udp0->dst_port;
 128   key0.protocol = proto0;
 129   key0.fib_index = sm->outside_fib_index;
 130   kv0.key = key0.as_u64;
 131
 132   /* NAT packet aimed at external address if */
 133   /* has active sessions */
 134   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 135                               &value0))
 136     {
 137       /* or is static mappings */
 138       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
 139         return 0;
 140     }
 141   else
 142     return 0;
 143
 144   if (sm->forwarding_enabled)
 145     return 1;
 146
 147   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 148                                   rx_fib_index0);
 149 }
 150
 151 static inline int
 152 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 153                                   u32 proto0, u16 src_port, u16 dst_port,
 154                                   u32 thread_index, u32 sw_if_index)
 155 {
 156   snat_session_key_t key0;
 157   clib_bihash_kv_8_8_t kv0, value0;
 158   snat_interface_t *i;
 159
 160   /* src NAT check */
 161   key0.addr = ip0->src_address;
 162   key0.port = src_port;
 163   key0.protocol = proto0;
 164   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 165   kv0.key = key0.as_u64;
 166
 167   if (!clib_bihash_search_8_8
 168       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 169     return 1;
 170
 171   /* dst NAT check */
 172   key0.addr = ip0->dst_address;
 173   key0.port = dst_port;
 174   key0.protocol = proto0;
 175   kv0.key = key0.as_u64;
 176   if (!clib_bihash_search_8_8
 177       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 178     {
 179       /* hairpinning */
 180     /* *INDENT-OFF* */
 181     pool_foreach (i, sm->output_feature_interfaces,
 182     ({
 183       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 184         return 0;
 185     }));
 186     /* *INDENT-ON* */
 187       return 1;
 188     }
 189
 190   return 0;
 191 }
 192
 193 int
 194 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 195 {
 196   snat_main_t *sm = &snat_main;
 197   nat44_is_idle_session_ctx_t *ctx = arg;
 198   snat_session_t *s;
 199   u64 sess_timeout_time;
 200   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 201                                                        ctx->thread_index);
 202   clib_bihash_kv_8_8_t s_kv;
 203
 204   s = pool_elt_at_index (tsm->sessions, kv->value);
 205   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 206   if (ctx->now >= sess_timeout_time)
 207     {
 208       s_kv.key = s->out2in.as_u64;
 209       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 210         nat_log_warn ("out2in key del failed");
 211
 212       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 213                                            s->out2in.addr.as_u32,
 214                                            s->in2out.protocol,
 215                                            s->in2out.port,
 216                                            s->out2in.port,
 217                                            s->in2out.fib_index);
 218
 219       if (!snat_is_session_static (s))
 220         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 221                                             &s->out2in);
 222
 223       nat44_delete_session (sm, s, ctx->thread_index);
 224       return 1;
 225     }
 226
 227   return 0;
 228 }
 229
 230 static u32
 231 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 232            ip4_header_t * ip0,
 233            u32 rx_fib_index0,
 234            snat_session_key_t * key0,
 235            snat_session_t ** sessionp,
 236            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 237 {
 238   snat_user_t *u;
 239   snat_session_t *s;
 240   clib_bihash_kv_8_8_t kv0;
 241   snat_session_key_t key1;
 242   udp_header_t *udp0 = ip4_next_header (ip0);
 243   u8 is_sm = 0;
 244   nat_outside_fib_t *outside_fib;
 245   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 246   fib_prefix_t pfx = {
 247     .fp_proto = FIB_PROTOCOL_IP4,
 248     .fp_len = 32,
 249     .fp_addr = {
 250                 .ip4.as_u32 = ip0->dst_address.as_u32,
 251                 },
 252   };
 253   nat44_is_idle_session_ctx_t ctx0;
 254
 255   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 256     {
 257       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 258       nat_ipfix_logging_max_sessions (sm->max_translations);
 259       nat_log_notice ("maximum sessions exceeded");
 260       return SNAT_IN2OUT_NEXT_DROP;
 261     }
 262
 263   key1.protocol = key0->protocol;
 264
 265   /* First try to match static mapping by local address and port */
 266   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0, 0))
 267     {
 268       /* Try to create dynamic translation */
 269       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 270                                                thread_index, &key1,
 271                                                sm->port_per_thread,
 272                                                sm->per_thread_data
 273                                                [thread_index].snat_thread_index))
 274         {
 275           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 276           return SNAT_IN2OUT_NEXT_DROP;
 277         }
 278     }
 279   else
 280     is_sm = 1;
 281
 282   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 283                               thread_index);
 284   if (!u)
 285     {
 286       nat_log_warn ("create NAT user failed");
 287       return SNAT_IN2OUT_NEXT_DROP;
 288     }
 289
 290   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 291   if (!s)
 292     {
 293       nat44_delete_user_with_no_session (sm, u, thread_index);
 294       nat_log_warn ("create NAT session failed");
 295       return SNAT_IN2OUT_NEXT_DROP;
 296     }
 297
 298   if (is_sm)
 299     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 300   user_session_increment (sm, u, is_sm);
 301   s->in2out = *key0;
 302   s->out2in = key1;
 303   s->out2in.protocol = key0->protocol;
 304   s->out2in.fib_index = sm->outside_fib_index;
 305   switch (vec_len (sm->outside_fibs))
 306     {
 307     case 0:
 308       s->out2in.fib_index = sm->outside_fib_index;
 309       break;
 310     case 1:
 311       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 312       break;
 313     default:
 314       /* *INDENT-OFF* */
 315       vec_foreach (outside_fib, sm->outside_fibs)
 316         {
 317           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 318           if (FIB_NODE_INDEX_INVALID != fei)
 319             {
 320               if (fib_entry_get_resolving_interface (fei) != ~0)
 321                 {
 322                   s->out2in.fib_index = outside_fib->fib_index;
 323                   break;
 324                 }
 325             }
 326         }
 327       /* *INDENT-ON* */
 328       break;
 329     }
 330   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 331   s->ext_host_port = udp0->dst_port;
 332   *sessionp = s;
 333
 334   /* Add to translation hashes */
 335   ctx0.now = now;
 336   ctx0.thread_index = thread_index;
 337   kv0.key = s->in2out.as_u64;
 338   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 339   if (clib_bihash_add_or_overwrite_stale_8_8
 340       (&sm->per_thread_data[thread_index].in2out, &kv0,
 341        nat44_i2o_is_idle_session_cb, &ctx0))
 342     nat_log_notice ("in2out key add failed");
 343
 344   kv0.key = s->out2in.as_u64;
 345   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 346
 347   if (clib_bihash_add_or_overwrite_stale_8_8
 348       (&sm->per_thread_data[thread_index].out2in, &kv0,
 349        nat44_o2i_is_idle_session_cb, &ctx0))
 350     nat_log_notice ("out2in key add failed");
 351
 352   /* log NAT event */
 353   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 354                                        s->out2in.addr.as_u32,
 355                                        s->in2out.protocol,
 356                                        s->in2out.port,
 357                                        s->out2in.port, s->in2out.fib_index);
 358   return next0;
 359 }
 360
 361 static_always_inline
 362   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 363                                     snat_session_key_t * p_key0)
 364 {
 365   icmp46_header_t *icmp0;
 366   snat_session_key_t key0;
 367   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 368   ip4_header_t *inner_ip0 = 0;
 369   void *l4_header = 0;
 370   icmp46_header_t *inner_icmp0;
 371
 372   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 373   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 374
 375   if (!icmp_is_error_message (icmp0))
 376     {
 377       key0.protocol = SNAT_PROTOCOL_ICMP;
 378       key0.addr = ip0->src_address;
 379       key0.port = echo0->identifier;
 380     }
 381   else
 382     {
 383       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 384       l4_header = ip4_next_header (inner_ip0);
 385       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 386       key0.addr = inner_ip0->dst_address;
 387       switch (key0.protocol)
 388         {
 389         case SNAT_PROTOCOL_ICMP:
 390           inner_icmp0 = (icmp46_header_t *) l4_header;
 391           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 392           key0.port = inner_echo0->identifier;
 393           break;
 394         case SNAT_PROTOCOL_UDP:
 395         case SNAT_PROTOCOL_TCP:
 396           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 397           break;
 398         default:
 399           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 400         }
 401     }
 402   *p_key0 = key0;
 403   return -1;                    /* success */
 404 }
 405
 406 /**
 407  * Get address and port values to be used for ICMP packet translation
 408  * and create session if needed
 409  *
 410  * @param[in,out] sm             NAT main
 411  * @param[in,out] node           NAT node runtime
 412  * @param[in] thread_index       thread index
 413  * @param[in,out] b0             buffer containing packet to be translated
 414  * @param[out] p_proto           protocol used for matching
 415  * @param[out] p_value           address and port after NAT translation
 416  * @param[out] p_dont_translate  if packet should not be translated
 417  * @param d                      optional parameter
 418  * @param e                      optional parameter
 419  */
 420 u32
 421 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 422                         u32 thread_index, vlib_buffer_t * b0,
 423                         ip4_header_t * ip0, u8 * p_proto,
 424                         snat_session_key_t * p_value,
 425                         u8 * p_dont_translate, void *d, void *e)
 426 {
 427   icmp46_header_t *icmp0;
 428   u32 sw_if_index0;
 429   u32 rx_fib_index0;
 430   snat_session_key_t key0;
 431   snat_session_t *s0 = 0;
 432   u8 dont_translate = 0;
 433   clib_bihash_kv_8_8_t kv0, value0;
 434   u32 next0 = ~0;
 435   int err;
 436
 437   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 438   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 439   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 440
 441   err = icmp_get_key (ip0, &key0);
 442   if (err != -1)
 443     {
 444       b0->error = node->errors[err];
 445       next0 = SNAT_IN2OUT_NEXT_DROP;
 446       goto out;
 447     }
 448   key0.fib_index = rx_fib_index0;
 449
 450   kv0.key = key0.as_u64;
 451
 452   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 453                               &value0))
 454     {
 455       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 456         {
 457           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 458                                                                key0.protocol,
 459                                                                key0.port,
 460                                                                key0.port,
 461                                                                thread_index,
 462                                                                sw_if_index0)))
 463             {
 464               dont_translate = 1;
 465               goto out;
 466             }
 467         }
 468       else
 469         {
 470           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 471                                                  ip0, SNAT_PROTOCOL_ICMP,
 472                                                  rx_fib_index0,
 473                                                  thread_index)))
 474             {
 475               dont_translate = 1;
 476               goto out;
 477             }
 478         }
 479
 480       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 481         {
 482           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 483           next0 = SNAT_IN2OUT_NEXT_DROP;
 484           goto out;
 485         }
 486
 487       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 488                          thread_index, vlib_time_now (sm->vlib_main));
 489
 490       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 491         goto out;
 492     }
 493   else
 494     {
 495       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 496                          icmp0->type != ICMP4_echo_reply &&
 497                          !icmp_is_error_message (icmp0)))
 498         {
 499           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 500           next0 = SNAT_IN2OUT_NEXT_DROP;
 501           goto out;
 502         }
 503
 504       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 505                               value0.value);
 506     }
 507
 508 out:
 509   *p_proto = key0.protocol;
 510   if (s0)
 511     *p_value = s0->out2in;
 512   *p_dont_translate = dont_translate;
 513   if (d)
 514     *(snat_session_t **) d = s0;
 515   return next0;
 516 }
 517
 518 /**
 519  * Get address and port values to be used for ICMP packet translation
 520  *
 521  * @param[in] sm                 NAT main
 522  * @param[in,out] node           NAT node runtime
 523  * @param[in] thread_index       thread index
 524  * @param[in,out] b0             buffer containing packet to be translated
 525  * @param[out] p_proto           protocol used for matching
 526  * @param[out] p_value           address and port after NAT translation
 527  * @param[out] p_dont_translate  if packet should not be translated
 528  * @param d                      optional parameter
 529  * @param e                      optional parameter
 530  */
 531 u32
 532 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 533                         u32 thread_index, vlib_buffer_t * b0,
 534                         ip4_header_t * ip0, u8 * p_proto,
 535                         snat_session_key_t * p_value,
 536                         u8 * p_dont_translate, void *d, void *e)
 537 {
 538   icmp46_header_t *icmp0;
 539   u32 sw_if_index0;
 540   u32 rx_fib_index0;
 541   snat_session_key_t key0;
 542   snat_session_key_t sm0;
 543   u8 dont_translate = 0;
 544   u8 is_addr_only;
 545   u32 next0 = ~0;
 546   int err;
 547
 548   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 549   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 550   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 551
 552   err = icmp_get_key (ip0, &key0);
 553   if (err != -1)
 554     {
 555       b0->error = node->errors[err];
 556       next0 = SNAT_IN2OUT_NEXT_DROP;
 557       goto out2;
 558     }
 559   key0.fib_index = rx_fib_index0;
 560
 561   if (snat_static_mapping_match (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0))
 562     {
 563       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 564                                                   IP_PROTOCOL_ICMP,
 565                                                   rx_fib_index0)))
 566         {
 567           dont_translate = 1;
 568           goto out;
 569         }
 570
 571       if (icmp_is_error_message (icmp0))
 572         {
 573           next0 = SNAT_IN2OUT_NEXT_DROP;
 574           goto out;
 575         }
 576
 577       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 578       next0 = SNAT_IN2OUT_NEXT_DROP;
 579       goto out;
 580     }
 581
 582   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 583                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 584                      !icmp_is_error_message (icmp0)))
 585     {
 586       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 587       next0 = SNAT_IN2OUT_NEXT_DROP;
 588       goto out;
 589     }
 590
 591 out:
 592   *p_value = sm0;
 593 out2:
 594   *p_proto = key0.protocol;
 595   *p_dont_translate = dont_translate;
 596   return next0;
 597 }
 598
 599 u32
 600 icmp_in2out (snat_main_t * sm,
 601              vlib_buffer_t * b0,
 602              ip4_header_t * ip0,
 603              icmp46_header_t * icmp0,
 604              u32 sw_if_index0,
 605              u32 rx_fib_index0,
 606              vlib_node_runtime_t * node,
 607              u32 next0, u32 thread_index, void *d, void *e)
 608 {
 609   snat_session_key_t sm0;
 610   u8 protocol;
 611   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 612   ip4_header_t *inner_ip0;
 613   void *l4_header = 0;
 614   icmp46_header_t *inner_icmp0;
 615   u8 dont_translate;
 616   u32 new_addr0, old_addr0;
 617   u16 old_id0, new_id0;
 618   u16 old_checksum0, new_checksum0;
 619   ip_csum_t sum0;
 620   u16 checksum0;
 621   u32 next0_tmp;
 622
 623   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 624
 625   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 626                                         &protocol, &sm0, &dont_translate, d,
 627                                         e);
 628   if (next0_tmp != ~0)
 629     next0 = next0_tmp;
 630   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 631     goto out;
 632
 633   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 634     {
 635       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 636                                              (u8 *)
 637                                              vlib_buffer_get_current (b0),
 638                                              ntohs (ip0->length) -
 639                                              ip4_header_bytes (ip0), 0);
 640       checksum0 = ~ip_csum_fold (sum0);
 641       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 642         {
 643           next0 = SNAT_IN2OUT_NEXT_DROP;
 644           goto out;
 645         }
 646     }
 647
 648   old_addr0 = ip0->src_address.as_u32;
 649   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 650
 651   sum0 = ip0->checksum;
 652   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 653                          src_address /* changed member */ );
 654   ip0->checksum = ip_csum_fold (sum0);
 655
 656   if (icmp0->checksum == 0)
 657     icmp0->checksum = 0xffff;
 658
 659   if (!icmp_is_error_message (icmp0))
 660     {
 661       new_id0 = sm0.port;
 662       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 663         {
 664           old_id0 = echo0->identifier;
 665           new_id0 = sm0.port;
 666           echo0->identifier = new_id0;
 667
 668           sum0 = icmp0->checksum;
 669           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 670                                  identifier);
 671           icmp0->checksum = ip_csum_fold (sum0);
 672         }
 673     }
 674   else
 675     {
 676       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 677       l4_header = ip4_next_header (inner_ip0);
 678
 679       if (!ip4_header_checksum_is_valid (inner_ip0))
 680         {
 681           next0 = SNAT_IN2OUT_NEXT_DROP;
 682           goto out;
 683         }
 684
 685       /* update inner destination IP address */
 686       old_addr0 = inner_ip0->dst_address.as_u32;
 687       inner_ip0->dst_address = sm0.addr;
 688       new_addr0 = inner_ip0->dst_address.as_u32;
 689       sum0 = icmp0->checksum;
 690       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 691                              dst_address /* changed member */ );
 692       icmp0->checksum = ip_csum_fold (sum0);
 693
 694       /* update inner IP header checksum */
 695       old_checksum0 = inner_ip0->checksum;
 696       sum0 = inner_ip0->checksum;
 697       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 698                              dst_address /* changed member */ );
 699       inner_ip0->checksum = ip_csum_fold (sum0);
 700       new_checksum0 = inner_ip0->checksum;
 701       sum0 = icmp0->checksum;
 702       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 703                              checksum);
 704       icmp0->checksum = ip_csum_fold (sum0);
 705
 706       switch (protocol)
 707         {
 708         case SNAT_PROTOCOL_ICMP:
 709           inner_icmp0 = (icmp46_header_t *) l4_header;
 710           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 711
 712           old_id0 = inner_echo0->identifier;
 713           new_id0 = sm0.port;
 714           inner_echo0->identifier = new_id0;
 715
 716           sum0 = icmp0->checksum;
 717           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 718                                  identifier);
 719           icmp0->checksum = ip_csum_fold (sum0);
 720           break;
 721         case SNAT_PROTOCOL_UDP:
 722         case SNAT_PROTOCOL_TCP:
 723           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 724           new_id0 = sm0.port;
 725           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 726
 727           sum0 = icmp0->checksum;
 728           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 729                                  dst_port);
 730           icmp0->checksum = ip_csum_fold (sum0);
 731           break;
 732         default:
 733           ASSERT (0);
 734         }
 735     }
 736
 737   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 738     {
 739       if (sm->deterministic ||
 740           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 741                                       sm->endpoint_dependent))
 742         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 743     }
 744
 745 out:
 746   return next0;
 747 }
 748
 749 static inline u32
 750 icmp_in2out_slow_path (snat_main_t * sm,
 751                        vlib_buffer_t * b0,
 752                        ip4_header_t * ip0,
 753                        icmp46_header_t * icmp0,
 754                        u32 sw_if_index0,
 755                        u32 rx_fib_index0,
 756                        vlib_node_runtime_t * node,
 757                        u32 next0,
 758                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 759 {
 760   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 761                        next0, thread_index, p_s0, 0);
 762   snat_session_t *s0 = *p_s0;
 763   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 764     {
 765       /* Accounting */
 766       nat44_session_update_counters (s0, now,
 767                                      vlib_buffer_length_in_chain
 768                                      (sm->vlib_main, b0));
 769       /* Per-user LRU list maintenance */
 770       nat44_session_update_lru (sm, s0, thread_index);
 771     }
 772   return next0;
 773 }
 774
 775 static int
 776 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 777                              vlib_buffer_t * b,
 778                              ip4_header_t * ip, u32 rx_fib_index)
 779 {
 780   clib_bihash_kv_8_8_t kv, value;
 781   snat_static_mapping_t *m;
 782   snat_session_key_t m_key;
 783   u32 old_addr, new_addr;
 784   ip_csum_t sum;
 785
 786   m_key.addr = ip->src_address;
 787   m_key.port = 0;
 788   m_key.protocol = 0;
 789   m_key.fib_index = rx_fib_index;
 790   kv.key = m_key.as_u64;
 791   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 792     return 1;
 793
 794   m = pool_elt_at_index (sm->static_mappings, value.value);
 795
 796   old_addr = ip->src_address.as_u32;
 797   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 798   sum = ip->checksum;
 799   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 800   ip->checksum = ip_csum_fold (sum);
 801
 802
 803   /* Hairpinning */
 804   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 805     {
 806       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 807       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 808     }
 809
 810   return 0;
 811 }
 812
 813 static inline uword
 814 snat_in2out_node_fn_inline (vlib_main_t * vm,
 815                             vlib_node_runtime_t * node,
 816                             vlib_frame_t * frame, int is_slow_path,
 817                             int is_output_feature)
 818 {
 819   u32 n_left_from, *from, *to_next;
 820   snat_in2out_next_t next_index;
 821   u32 pkts_processed = 0;
 822   snat_main_t *sm = &snat_main;
 823   f64 now = vlib_time_now (vm);
 824   u32 stats_node_index;
 825   u32 thread_index = vm->thread_index;
 826
 827   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 828     snat_in2out_node.index;
 829
 830   from = vlib_frame_vector_args (frame);
 831   n_left_from = frame->n_vectors;
 832   next_index = node->cached_next_index;
 833
 834   while (n_left_from > 0)
 835     {
 836       u32 n_left_to_next;
 837
 838       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 839
 840       while (n_left_from >= 4 && n_left_to_next >= 2)
 841         {
 842           u32 bi0, bi1;
 843           vlib_buffer_t *b0, *b1;
 844           u32 next0, next1;
 845           u32 sw_if_index0, sw_if_index1;
 846           ip4_header_t *ip0, *ip1;
 847           ip_csum_t sum0, sum1;
 848           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 849           u16 old_port0, new_port0, old_port1, new_port1;
 850           udp_header_t *udp0, *udp1;
 851           tcp_header_t *tcp0, *tcp1;
 852           icmp46_header_t *icmp0, *icmp1;
 853           snat_session_key_t key0, key1;
 854           u32 rx_fib_index0, rx_fib_index1;
 855           u32 proto0, proto1;
 856           snat_session_t *s0 = 0, *s1 = 0;
 857           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 858           u32 iph_offset0 = 0, iph_offset1 = 0;
 859
 860           /* Prefetch next iteration. */
 861           {
 862             vlib_buffer_t *p2, *p3;
 863
 864             p2 = vlib_get_buffer (vm, from[2]);
 865             p3 = vlib_get_buffer (vm, from[3]);
 866
 867             vlib_prefetch_buffer_header (p2, LOAD);
 868             vlib_prefetch_buffer_header (p3, LOAD);
 869
 870             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 871             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 872           }
 873
 874           /* speculatively enqueue b0 and b1 to the current next frame */
 875           to_next[0] = bi0 = from[0];
 876           to_next[1] = bi1 = from[1];
 877           from += 2;
 878           to_next += 2;
 879           n_left_from -= 2;
 880           n_left_to_next -= 2;
 881
 882           b0 = vlib_get_buffer (vm, bi0);
 883           b1 = vlib_get_buffer (vm, bi1);
 884
 885           if (is_output_feature)
 886             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 887
 888           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 889                                   iph_offset0);
 890
 891           udp0 = ip4_next_header (ip0);
 892           tcp0 = (tcp_header_t *) udp0;
 893           icmp0 = (icmp46_header_t *) udp0;
 894
 895           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 896           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 897                                    sw_if_index0);
 898
 899           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 900
 901           if (PREDICT_FALSE (ip0->ttl == 1))
 902             {
 903               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 904               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 905                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 906                                            0);
 907               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 908               goto trace00;
 909             }
 910
 911           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 912
 913           /* Next configured feature, probably ip4-lookup */
 914           if (is_slow_path)
 915             {
 916               if (PREDICT_FALSE (proto0 == ~0))
 917                 {
 918                   if (nat_in2out_sm_unknown_proto
 919                       (sm, b0, ip0, rx_fib_index0))
 920                     {
 921                       next0 = SNAT_IN2OUT_NEXT_DROP;
 922                       b0->error =
 923                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 924                     }
 925                   goto trace00;
 926                 }
 927
 928               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 929                 {
 930                   next0 = icmp_in2out_slow_path
 931                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 932                      node, next0, now, thread_index, &s0);
 933                   goto trace00;
 934                 }
 935             }
 936           else
 937             {
 938               if (PREDICT_FALSE
 939                   (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 940                 {
 941                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 942                   goto trace00;
 943                 }
 944
 945               if (ip4_is_fragment (ip0))
 946                 {
 947                   next0 = SNAT_IN2OUT_NEXT_REASS;
 948                   goto trace00;
 949                 }
 950             }
 951
 952           key0.addr = ip0->src_address;
 953           key0.port = udp0->src_port;
 954           key0.protocol = proto0;
 955           key0.fib_index = rx_fib_index0;
 956
 957           kv0.key = key0.as_u64;
 958
 959           if (PREDICT_FALSE
 960               (clib_bihash_search_8_8
 961                (&sm->per_thread_data[thread_index].in2out, &kv0,
 962                 &value0) != 0))
 963             {
 964               if (is_slow_path)
 965                 {
 966                   if (is_output_feature)
 967                     {
 968                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
 969                                                                            ip0,
 970                                                                            proto0,
 971                                                                            udp0->src_port,
 972                                                                            udp0->dst_port,
 973                                                                            thread_index,
 974                                                                            sw_if_index0)))
 975                         goto trace00;
 976                     }
 977                   else
 978                     {
 979                       if (PREDICT_FALSE
 980                           (snat_not_translate
 981                            (sm, node, sw_if_index0, ip0, proto0,
 982                             rx_fib_index0, thread_index)))
 983                         goto trace00;
 984                     }
 985
 986                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 987                                      &s0, node, next0, thread_index, now);
 988                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 989                     goto trace00;
 990                 }
 991               else
 992                 {
 993                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 994                   goto trace00;
 995                 }
 996             }
 997           else
 998             s0 =
 999               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1000                                  value0.value);
1001
1002           b0->flags |= VNET_BUFFER_F_IS_NATED;
1003
1004           old_addr0 = ip0->src_address.as_u32;
1005           ip0->src_address = s0->out2in.addr;
1006           new_addr0 = ip0->src_address.as_u32;
1007           if (!is_output_feature)
1008             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1009
1010           sum0 = ip0->checksum;
1011           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1012                                  ip4_header_t,
1013                                  src_address /* changed member */ );
1014           ip0->checksum = ip_csum_fold (sum0);
1015
1016           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1017             {
1018               old_port0 = tcp0->src_port;
1019               tcp0->src_port = s0->out2in.port;
1020               new_port0 = tcp0->src_port;
1021
1022               sum0 = tcp0->checksum;
1023               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1024                                      ip4_header_t,
1025                                      dst_address /* changed member */ );
1026               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1027                                      ip4_header_t /* cheat */ ,
1028                                      length /* changed member */ );
1029               mss_clamping (sm, tcp0, &sum0);
1030               tcp0->checksum = ip_csum_fold (sum0);
1031             }
1032           else
1033             {
1034               old_port0 = udp0->src_port;
1035               udp0->src_port = s0->out2in.port;
1036               udp0->checksum = 0;
1037             }
1038
1039           /* Accounting */
1040           nat44_session_update_counters (s0, now,
1041                                          vlib_buffer_length_in_chain (vm,
1042                                                                       b0));
1043           /* Per-user LRU list maintenance */
1044           nat44_session_update_lru (sm, s0, thread_index);
1045         trace00:
1046
1047           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1048                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1049             {
1050               snat_in2out_trace_t *t =
1051                 vlib_add_trace (vm, node, b0, sizeof (*t));
1052               t->is_slow_path = is_slow_path;
1053               t->sw_if_index = sw_if_index0;
1054               t->next_index = next0;
1055               t->session_index = ~0;
1056               if (s0)
1057                 t->session_index =
1058                   s0 - sm->per_thread_data[thread_index].sessions;
1059             }
1060
1061           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1062
1063           if (is_output_feature)
1064             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1065
1066           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1067                                   iph_offset1);
1068
1069           udp1 = ip4_next_header (ip1);
1070           tcp1 = (tcp_header_t *) udp1;
1071           icmp1 = (icmp46_header_t *) udp1;
1072
1073           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1074           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1075                                    sw_if_index1);
1076
1077           if (PREDICT_FALSE (ip1->ttl == 1))
1078             {
1079               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1080               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1081                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1082                                            0);
1083               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1084               goto trace01;
1085             }
1086
1087           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1088
1089           /* Next configured feature, probably ip4-lookup */
1090           if (is_slow_path)
1091             {
1092               if (PREDICT_FALSE (proto1 == ~0))
1093                 {
1094                   if (nat_in2out_sm_unknown_proto
1095                       (sm, b1, ip1, rx_fib_index1))
1096                     {
1097                       next1 = SNAT_IN2OUT_NEXT_DROP;
1098                       b1->error =
1099                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1100                     }
1101                   goto trace01;
1102                 }
1103
1104               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1105                 {
1106                   next1 = icmp_in2out_slow_path
1107                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1108                      next1, now, thread_index, &s1);
1109                   goto trace01;
1110                 }
1111             }
1112           else
1113             {
1114               if (PREDICT_FALSE
1115                   (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1116                 {
1117                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1118                   goto trace01;
1119                 }
1120
1121               if (ip4_is_fragment (ip1))
1122                 {
1123                   next1 = SNAT_IN2OUT_NEXT_REASS;
1124                   goto trace01;
1125                 }
1126             }
1127
1128           key1.addr = ip1->src_address;
1129           key1.port = udp1->src_port;
1130           key1.protocol = proto1;
1131           key1.fib_index = rx_fib_index1;
1132
1133           kv1.key = key1.as_u64;
1134
1135           if (PREDICT_FALSE
1136               (clib_bihash_search_8_8
1137                (&sm->per_thread_data[thread_index].in2out, &kv1,
1138                 &value1) != 0))
1139             {
1140               if (is_slow_path)
1141                 {
1142                   if (is_output_feature)
1143                     {
1144                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1145                                                                            ip1,
1146                                                                            proto1,
1147                                                                            udp1->src_port,
1148                                                                            udp1->dst_port,
1149                                                                            thread_index,
1150                                                                            sw_if_index1)))
1151                         goto trace01;
1152                     }
1153                   else
1154                     {
1155                       if (PREDICT_FALSE
1156                           (snat_not_translate
1157                            (sm, node, sw_if_index1, ip1, proto1,
1158                             rx_fib_index1, thread_index)))
1159                         goto trace01;
1160                     }
1161
1162                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1163                                      &s1, node, next1, thread_index, now);
1164                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1165                     goto trace01;
1166                 }
1167               else
1168                 {
1169                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1170                   goto trace01;
1171                 }
1172             }
1173           else
1174             s1 =
1175               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1176                                  value1.value);
1177
1178           b1->flags |= VNET_BUFFER_F_IS_NATED;
1179
1180           old_addr1 = ip1->src_address.as_u32;
1181           ip1->src_address = s1->out2in.addr;
1182           new_addr1 = ip1->src_address.as_u32;
1183           if (!is_output_feature)
1184             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1185
1186           sum1 = ip1->checksum;
1187           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1188                                  ip4_header_t,
1189                                  src_address /* changed member */ );
1190           ip1->checksum = ip_csum_fold (sum1);
1191
1192           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1193             {
1194               old_port1 = tcp1->src_port;
1195               tcp1->src_port = s1->out2in.port;
1196               new_port1 = tcp1->src_port;
1197
1198               sum1 = tcp1->checksum;
1199               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1200                                      ip4_header_t,
1201                                      dst_address /* changed member */ );
1202               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1203                                      ip4_header_t /* cheat */ ,
1204                                      length /* changed member */ );
1205               mss_clamping (sm, tcp1, &sum1);
1206               tcp1->checksum = ip_csum_fold (sum1);
1207             }
1208           else
1209             {
1210               old_port1 = udp1->src_port;
1211               udp1->src_port = s1->out2in.port;
1212               udp1->checksum = 0;
1213             }
1214
1215           /* Accounting */
1216           nat44_session_update_counters (s1, now,
1217                                          vlib_buffer_length_in_chain (vm,
1218                                                                       b1));
1219           /* Per-user LRU list maintenance */
1220           nat44_session_update_lru (sm, s1, thread_index);
1221         trace01:
1222
1223           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1224                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1225             {
1226               snat_in2out_trace_t *t =
1227                 vlib_add_trace (vm, node, b1, sizeof (*t));
1228               t->sw_if_index = sw_if_index1;
1229               t->next_index = next1;
1230               t->session_index = ~0;
1231               if (s1)
1232                 t->session_index =
1233                   s1 - sm->per_thread_data[thread_index].sessions;
1234             }
1235
1236           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1237
1238           /* verify speculative enqueues, maybe switch current next frame */
1239           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1240                                            to_next, n_left_to_next,
1241                                            bi0, bi1, next0, next1);
1242         }
1243
1244       while (n_left_from > 0 && n_left_to_next > 0)
1245         {
1246           u32 bi0;
1247           vlib_buffer_t *b0;
1248           u32 next0;
1249           u32 sw_if_index0;
1250           ip4_header_t *ip0;
1251           ip_csum_t sum0;
1252           u32 new_addr0, old_addr0;
1253           u16 old_port0, new_port0;
1254           udp_header_t *udp0;
1255           tcp_header_t *tcp0;
1256           icmp46_header_t *icmp0;
1257           snat_session_key_t key0;
1258           u32 rx_fib_index0;
1259           u32 proto0;
1260           snat_session_t *s0 = 0;
1261           clib_bihash_kv_8_8_t kv0, value0;
1262           u32 iph_offset0 = 0;
1263
1264           /* speculatively enqueue b0 to the current next frame */
1265           bi0 = from[0];
1266           to_next[0] = bi0;
1267           from += 1;
1268           to_next += 1;
1269           n_left_from -= 1;
1270           n_left_to_next -= 1;
1271
1272           b0 = vlib_get_buffer (vm, bi0);
1273           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1274
1275           if (is_output_feature)
1276             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1277
1278           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1279                                   iph_offset0);
1280
1281           udp0 = ip4_next_header (ip0);
1282           tcp0 = (tcp_header_t *) udp0;
1283           icmp0 = (icmp46_header_t *) udp0;
1284
1285           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1286           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1287                                    sw_if_index0);
1288
1289           if (PREDICT_FALSE (ip0->ttl == 1))
1290             {
1291               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1292               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1293                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1294                                            0);
1295               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1296               goto trace0;
1297             }
1298
1299           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1300
1301           /* Next configured feature, probably ip4-lookup */
1302           if (is_slow_path)
1303             {
1304               if (PREDICT_FALSE (proto0 == ~0))
1305                 {
1306                   if (nat_in2out_sm_unknown_proto
1307                       (sm, b0, ip0, rx_fib_index0))
1308                     {
1309                       next0 = SNAT_IN2OUT_NEXT_DROP;
1310                       b0->error =
1311                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1312                     }
1313                   goto trace0;
1314                 }
1315
1316               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1317                 {
1318                   next0 = icmp_in2out_slow_path
1319                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1320                      next0, now, thread_index, &s0);
1321                   goto trace0;
1322                 }
1323             }
1324           else
1325             {
1326               if (PREDICT_FALSE
1327                   (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1328                 {
1329                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1330                   goto trace0;
1331                 }
1332
1333               if (ip4_is_fragment (ip0))
1334                 {
1335                   next0 = SNAT_IN2OUT_NEXT_REASS;
1336                   goto trace0;
1337                 }
1338             }
1339
1340           key0.addr = ip0->src_address;
1341           key0.port = udp0->src_port;
1342           key0.protocol = proto0;
1343           key0.fib_index = rx_fib_index0;
1344
1345           kv0.key = key0.as_u64;
1346
1347           if (clib_bihash_search_8_8
1348               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1349             {
1350               if (is_slow_path)
1351                 {
1352                   if (is_output_feature)
1353                     {
1354                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1355                                                                            ip0,
1356                                                                            proto0,
1357                                                                            udp0->src_port,
1358                                                                            udp0->dst_port,
1359                                                                            thread_index,
1360                                                                            sw_if_index0)))
1361                         goto trace0;
1362                     }
1363                   else
1364                     {
1365                       if (PREDICT_FALSE
1366                           (snat_not_translate
1367                            (sm, node, sw_if_index0, ip0, proto0,
1368                             rx_fib_index0, thread_index)))
1369                         goto trace0;
1370                     }
1371
1372                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1373                                      &s0, node, next0, thread_index, now);
1374
1375                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1376                     goto trace0;
1377                 }
1378               else
1379                 {
1380                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1381                   goto trace0;
1382                 }
1383             }
1384           else
1385             s0 =
1386               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1387                                  value0.value);
1388
1389           b0->flags |= VNET_BUFFER_F_IS_NATED;
1390
1391           old_addr0 = ip0->src_address.as_u32;
1392           ip0->src_address = s0->out2in.addr;
1393           new_addr0 = ip0->src_address.as_u32;
1394           if (!is_output_feature)
1395             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1396
1397           sum0 = ip0->checksum;
1398           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1399                                  ip4_header_t,
1400                                  src_address /* changed member */ );
1401           ip0->checksum = ip_csum_fold (sum0);
1402
1403           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1404             {
1405               old_port0 = tcp0->src_port;
1406               tcp0->src_port = s0->out2in.port;
1407               new_port0 = tcp0->src_port;
1408
1409               sum0 = tcp0->checksum;
1410               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1411                                      ip4_header_t,
1412                                      dst_address /* changed member */ );
1413               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1414                                      ip4_header_t /* cheat */ ,
1415                                      length /* changed member */ );
1416               mss_clamping (sm, tcp0, &sum0);
1417               tcp0->checksum = ip_csum_fold (sum0);
1418             }
1419           else
1420             {
1421               old_port0 = udp0->src_port;
1422               udp0->src_port = s0->out2in.port;
1423               udp0->checksum = 0;
1424             }
1425
1426           /* Accounting */
1427           nat44_session_update_counters (s0, now,
1428                                          vlib_buffer_length_in_chain (vm,
1429                                                                       b0));
1430           /* Per-user LRU list maintenance */
1431           nat44_session_update_lru (sm, s0, thread_index);
1432
1433         trace0:
1434           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1435                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1436             {
1437               snat_in2out_trace_t *t =
1438                 vlib_add_trace (vm, node, b0, sizeof (*t));
1439               t->is_slow_path = is_slow_path;
1440               t->sw_if_index = sw_if_index0;
1441               t->next_index = next0;
1442               t->session_index = ~0;
1443               if (s0)
1444                 t->session_index =
1445                   s0 - sm->per_thread_data[thread_index].sessions;
1446             }
1447
1448           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1449
1450           /* verify speculative enqueue, maybe switch current next frame */
1451           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1452                                            to_next, n_left_to_next,
1453                                            bi0, next0);
1454         }
1455
1456       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1457     }
1458
1459   vlib_node_increment_counter (vm, stats_node_index,
1460                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1461                                pkts_processed);
1462   return frame->n_vectors;
1463 }
1464
1465 static uword
1466 snat_in2out_fast_path_fn (vlib_main_t * vm,
1467                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1468 {
1469   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1470                                      0);
1471 }
1472
1473 /* *INDENT-OFF* */
1474 VLIB_REGISTER_NODE (snat_in2out_node) = {
1475   .function = snat_in2out_fast_path_fn,
1476   .name = "nat44-in2out",
1477   .vector_size = sizeof (u32),
1478   .format_trace = format_snat_in2out_trace,
1479   .type = VLIB_NODE_TYPE_INTERNAL,
1480
1481   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1482   .error_strings = snat_in2out_error_strings,
1483
1484   .runtime_data_bytes = sizeof (snat_runtime_t),
1485
1486   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1487
1488   /* edit / add dispositions here */
1489   .next_nodes = {
1490     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1491     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1492     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1493     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1494     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1495   },
1496 };
1497 /* *INDENT-ON* */
1498
1499 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1500
1501 static uword
1502 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1503                                  vlib_node_runtime_t * node,
1504                                  vlib_frame_t * frame)
1505 {
1506   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1507                                      1);
1508 }
1509
1510 /* *INDENT-OFF* */
1511 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1512   .function = snat_in2out_output_fast_path_fn,
1513   .name = "nat44-in2out-output",
1514   .vector_size = sizeof (u32),
1515   .format_trace = format_snat_in2out_trace,
1516   .type = VLIB_NODE_TYPE_INTERNAL,
1517
1518   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1519   .error_strings = snat_in2out_error_strings,
1520
1521   .runtime_data_bytes = sizeof (snat_runtime_t),
1522
1523   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1524
1525   /* edit / add dispositions here */
1526   .next_nodes = {
1527     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1528     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1529     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1530     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1531     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1532   },
1533 };
1534 /* *INDENT-ON* */
1535
1536 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1537                               snat_in2out_output_fast_path_fn);
1538
1539 static uword
1540 snat_in2out_slow_path_fn (vlib_main_t * vm,
1541                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1542 {
1543   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1544                                      0);
1545 }
1546
1547 /* *INDENT-OFF* */
1548 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1549   .function = snat_in2out_slow_path_fn,
1550   .name = "nat44-in2out-slowpath",
1551   .vector_size = sizeof (u32),
1552   .format_trace = format_snat_in2out_trace,
1553   .type = VLIB_NODE_TYPE_INTERNAL,
1554
1555   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1556   .error_strings = snat_in2out_error_strings,
1557
1558   .runtime_data_bytes = sizeof (snat_runtime_t),
1559
1560   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1561
1562   /* edit / add dispositions here */
1563   .next_nodes = {
1564     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1565     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1566     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1567     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1568     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1569   },
1570 };
1571 /* *INDENT-ON* */
1572
1573 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1574                               snat_in2out_slow_path_fn);
1575
1576 static uword
1577 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1578                                  vlib_node_runtime_t * node,
1579                                  vlib_frame_t * frame)
1580 {
1581   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1582                                      1);
1583 }
1584
1585 /* *INDENT-OFF* */
1586 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1587   .function = snat_in2out_output_slow_path_fn,
1588   .name = "nat44-in2out-output-slowpath",
1589   .vector_size = sizeof (u32),
1590   .format_trace = format_snat_in2out_trace,
1591   .type = VLIB_NODE_TYPE_INTERNAL,
1592
1593   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1594   .error_strings = snat_in2out_error_strings,
1595
1596   .runtime_data_bytes = sizeof (snat_runtime_t),
1597
1598   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1599
1600   /* edit / add dispositions here */
1601   .next_nodes = {
1602     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1603     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1604     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1605     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1606     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1607   },
1608 };
1609 /* *INDENT-ON* */
1610
1611 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1612                               snat_in2out_output_slow_path_fn);
1613
1614 static uword
1615 nat44_in2out_reass_node_fn (vlib_main_t * vm,
1616                             vlib_node_runtime_t * node, vlib_frame_t * frame)
1617 {
1618   u32 n_left_from, *from, *to_next;
1619   snat_in2out_next_t next_index;
1620   u32 pkts_processed = 0;
1621   snat_main_t *sm = &snat_main;
1622   f64 now = vlib_time_now (vm);
1623   u32 thread_index = vm->thread_index;
1624   snat_main_per_thread_data_t *per_thread_data =
1625     &sm->per_thread_data[thread_index];
1626   u32 *fragments_to_drop = 0;
1627   u32 *fragments_to_loopback = 0;
1628
1629   from = vlib_frame_vector_args (frame);
1630   n_left_from = frame->n_vectors;
1631   next_index = node->cached_next_index;
1632
1633   while (n_left_from > 0)
1634     {
1635       u32 n_left_to_next;
1636
1637       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1638
1639       while (n_left_from > 0 && n_left_to_next > 0)
1640         {
1641           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1642           vlib_buffer_t *b0;
1643           u32 next0;
1644           u8 cached0 = 0;
1645           ip4_header_t *ip0;
1646           nat_reass_ip4_t *reass0;
1647           udp_header_t *udp0;
1648           tcp_header_t *tcp0;
1649           snat_session_key_t key0;
1650           clib_bihash_kv_8_8_t kv0, value0;
1651           snat_session_t *s0 = 0;
1652           u16 old_port0, new_port0;
1653           ip_csum_t sum0;
1654
1655           /* speculatively enqueue b0 to the current next frame */
1656           bi0 = from[0];
1657           to_next[0] = bi0;
1658           from += 1;
1659           to_next += 1;
1660           n_left_from -= 1;
1661           n_left_to_next -= 1;
1662
1663           b0 = vlib_get_buffer (vm, bi0);
1664           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1665
1666           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1667           rx_fib_index0 =
1668             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1669                                                  sw_if_index0);
1670
1671           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1672             {
1673               next0 = SNAT_IN2OUT_NEXT_DROP;
1674               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1675               goto trace0;
1676             }
1677
1678           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1679           udp0 = ip4_next_header (ip0);
1680           tcp0 = (tcp_header_t *) udp0;
1681           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1682
1683           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1684                                                  ip0->dst_address,
1685                                                  ip0->fragment_id,
1686                                                  ip0->protocol,
1687                                                  1, &fragments_to_drop);
1688
1689           if (PREDICT_FALSE (!reass0))
1690             {
1691               next0 = SNAT_IN2OUT_NEXT_DROP;
1692               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1693               nat_log_notice ("maximum reassemblies exceeded");
1694               goto trace0;
1695             }
1696
1697           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1698             {
1699               key0.addr = ip0->src_address;
1700               key0.port = udp0->src_port;
1701               key0.protocol = proto0;
1702               key0.fib_index = rx_fib_index0;
1703               kv0.key = key0.as_u64;
1704
1705               if (clib_bihash_search_8_8
1706                   (&per_thread_data->in2out, &kv0, &value0))
1707                 {
1708                   if (PREDICT_FALSE
1709                       (snat_not_translate
1710                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1711                         thread_index)))
1712                     goto trace0;
1713
1714                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1715                                      &s0, node, next0, thread_index, now);
1716
1717                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1718                     goto trace0;
1719
1720                   reass0->sess_index = s0 - per_thread_data->sessions;
1721                 }
1722               else
1723                 {
1724                   s0 = pool_elt_at_index (per_thread_data->sessions,
1725                                           value0.value);
1726                   reass0->sess_index = value0.value;
1727                 }
1728               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1729             }
1730           else
1731             {
1732               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1733                 {
1734                   if (nat_ip4_reass_add_fragment
1735                       (reass0, bi0, &fragments_to_drop))
1736                     {
1737                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1738                       nat_log_notice
1739                         ("maximum fragments per reassembly exceeded");
1740                       next0 = SNAT_IN2OUT_NEXT_DROP;
1741                       goto trace0;
1742                     }
1743                   cached0 = 1;
1744                   goto trace0;
1745                 }
1746               s0 = pool_elt_at_index (per_thread_data->sessions,
1747                                       reass0->sess_index);
1748             }
1749
1750           old_addr0 = ip0->src_address.as_u32;
1751           ip0->src_address = s0->out2in.addr;
1752           new_addr0 = ip0->src_address.as_u32;
1753           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1754
1755           sum0 = ip0->checksum;
1756           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1757                                  ip4_header_t,
1758                                  src_address /* changed member */ );
1759           ip0->checksum = ip_csum_fold (sum0);
1760
1761           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1762             {
1763               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1764                 {
1765                   old_port0 = tcp0->src_port;
1766                   tcp0->src_port = s0->out2in.port;
1767                   new_port0 = tcp0->src_port;
1768
1769                   sum0 = tcp0->checksum;
1770                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1771                                          ip4_header_t,
1772                                          dst_address /* changed member */ );
1773                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1774                                          ip4_header_t /* cheat */ ,
1775                                          length /* changed member */ );
1776                   tcp0->checksum = ip_csum_fold (sum0);
1777                 }
1778               else
1779                 {
1780                   old_port0 = udp0->src_port;
1781                   udp0->src_port = s0->out2in.port;
1782                   udp0->checksum = 0;
1783                 }
1784             }
1785
1786           /* Hairpinning */
1787           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1788                                    s0->ext_host_port, proto0, 0);
1789
1790           /* Accounting */
1791           nat44_session_update_counters (s0, now,
1792                                          vlib_buffer_length_in_chain (vm,
1793                                                                       b0));
1794           /* Per-user LRU list maintenance */
1795           nat44_session_update_lru (sm, s0, thread_index);
1796
1797         trace0:
1798           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1799                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1800             {
1801               nat44_reass_trace_t *t =
1802                 vlib_add_trace (vm, node, b0, sizeof (*t));
1803               t->cached = cached0;
1804               t->sw_if_index = sw_if_index0;
1805               t->next_index = next0;
1806             }
1807
1808           if (cached0)
1809             {
1810               n_left_to_next++;
1811               to_next--;
1812             }
1813           else
1814             {
1815               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1816
1817               /* verify speculative enqueue, maybe switch current next frame */
1818               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1819                                                to_next, n_left_to_next,
1820                                                bi0, next0);
1821             }
1822
1823           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1824             {
1825               from = vlib_frame_vector_args (frame);
1826               u32 len = vec_len (fragments_to_loopback);
1827               if (len <= VLIB_FRAME_SIZE)
1828                 {
1829                   clib_memcpy (from, fragments_to_loopback,
1830                                sizeof (u32) * len);
1831                   n_left_from = len;
1832                   vec_reset_length (fragments_to_loopback);
1833                 }
1834               else
1835                 {
1836                   clib_memcpy (from,
1837                                fragments_to_loopback + (len -
1838                                                         VLIB_FRAME_SIZE),
1839                                sizeof (u32) * VLIB_FRAME_SIZE);
1840                   n_left_from = VLIB_FRAME_SIZE;
1841                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1842                 }
1843             }
1844         }
1845
1846       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1847     }
1848
1849   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
1850                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1851                                pkts_processed);
1852
1853   nat_send_all_to_node (vm, fragments_to_drop, node,
1854                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1855                         SNAT_IN2OUT_NEXT_DROP);
1856
1857   vec_free (fragments_to_drop);
1858   vec_free (fragments_to_loopback);
1859   return frame->n_vectors;
1860 }
1861
1862 /* *INDENT-OFF* */
1863 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
1864   .function = nat44_in2out_reass_node_fn,
1865   .name = "nat44-in2out-reass",
1866   .vector_size = sizeof (u32),
1867   .format_trace = format_nat44_reass_trace,
1868   .type = VLIB_NODE_TYPE_INTERNAL,
1869
1870   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1871   .error_strings = snat_in2out_error_strings,
1872
1873   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1874   .next_nodes = {
1875     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1876     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1877     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1878     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1879     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1880   },
1881 };
1882 /* *INDENT-ON* */
1883
1884 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
1885                               nat44_in2out_reass_node_fn);
1886
1887 static uword
1888 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1889                                 vlib_node_runtime_t * node,
1890                                 vlib_frame_t * frame)
1891 {
1892   u32 n_left_from, *from, *to_next;
1893   snat_in2out_next_t next_index;
1894   u32 pkts_processed = 0;
1895   snat_main_t *sm = &snat_main;
1896   u32 stats_node_index;
1897
1898   stats_node_index = snat_in2out_fast_node.index;
1899
1900   from = vlib_frame_vector_args (frame);
1901   n_left_from = frame->n_vectors;
1902   next_index = node->cached_next_index;
1903
1904   while (n_left_from > 0)
1905     {
1906       u32 n_left_to_next;
1907
1908       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1909
1910       while (n_left_from > 0 && n_left_to_next > 0)
1911         {
1912           u32 bi0;
1913           vlib_buffer_t *b0;
1914           u32 next0;
1915           u32 sw_if_index0;
1916           ip4_header_t *ip0;
1917           ip_csum_t sum0;
1918           u32 new_addr0, old_addr0;
1919           u16 old_port0, new_port0;
1920           udp_header_t *udp0;
1921           tcp_header_t *tcp0;
1922           icmp46_header_t *icmp0;
1923           snat_session_key_t key0, sm0;
1924           u32 proto0;
1925           u32 rx_fib_index0;
1926
1927           /* speculatively enqueue b0 to the current next frame */
1928           bi0 = from[0];
1929           to_next[0] = bi0;
1930           from += 1;
1931           to_next += 1;
1932           n_left_from -= 1;
1933           n_left_to_next -= 1;
1934
1935           b0 = vlib_get_buffer (vm, bi0);
1936           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1937
1938           ip0 = vlib_buffer_get_current (b0);
1939           udp0 = ip4_next_header (ip0);
1940           tcp0 = (tcp_header_t *) udp0;
1941           icmp0 = (icmp46_header_t *) udp0;
1942
1943           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1944           rx_fib_index0 =
1945             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1946
1947           if (PREDICT_FALSE (ip0->ttl == 1))
1948             {
1949               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1950               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1951                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1952                                            0);
1953               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1954               goto trace0;
1955             }
1956
1957           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1958
1959           if (PREDICT_FALSE (proto0 == ~0))
1960             goto trace0;
1961
1962           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1963             {
1964               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
1965                                    rx_fib_index0, node, next0, ~0, 0, 0);
1966               goto trace0;
1967             }
1968
1969           key0.addr = ip0->src_address;
1970           key0.protocol = proto0;
1971           key0.port = udp0->src_port;
1972           key0.fib_index = rx_fib_index0;
1973
1974           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0))
1975             {
1976               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1977               next0 = SNAT_IN2OUT_NEXT_DROP;
1978               goto trace0;
1979             }
1980
1981           new_addr0 = sm0.addr.as_u32;
1982           new_port0 = sm0.port;
1983           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1984           old_addr0 = ip0->src_address.as_u32;
1985           ip0->src_address.as_u32 = new_addr0;
1986
1987           sum0 = ip0->checksum;
1988           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1989                                  ip4_header_t,
1990                                  src_address /* changed member */ );
1991           ip0->checksum = ip_csum_fold (sum0);
1992
1993           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1994             {
1995               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1996                 {
1997                   old_port0 = tcp0->src_port;
1998                   tcp0->src_port = new_port0;
1999
2000                   sum0 = tcp0->checksum;
2001                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2002                                          ip4_header_t,
2003                                          dst_address /* changed member */ );
2004                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2005                                          ip4_header_t /* cheat */ ,
2006                                          length /* changed member */ );
2007                   mss_clamping (sm, tcp0, &sum0);
2008                   tcp0->checksum = ip_csum_fold (sum0);
2009                 }
2010               else
2011                 {
2012                   old_port0 = udp0->src_port;
2013                   udp0->src_port = new_port0;
2014                   udp0->checksum = 0;
2015                 }
2016             }
2017           else
2018             {
2019               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2020                 {
2021                   sum0 = tcp0->checksum;
2022                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2023                                          ip4_header_t,
2024                                          dst_address /* changed member */ );
2025                   mss_clamping (sm, tcp0, &sum0);
2026                   tcp0->checksum = ip_csum_fold (sum0);
2027                 }
2028             }
2029
2030           /* Hairpinning */
2031           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2032
2033         trace0:
2034           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2035                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2036             {
2037               snat_in2out_trace_t *t =
2038                 vlib_add_trace (vm, node, b0, sizeof (*t));
2039               t->sw_if_index = sw_if_index0;
2040               t->next_index = next0;
2041             }
2042
2043           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2044
2045           /* verify speculative enqueue, maybe switch current next frame */
2046           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2047                                            to_next, n_left_to_next,
2048                                            bi0, next0);
2049         }
2050
2051       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2052     }
2053
2054   vlib_node_increment_counter (vm, stats_node_index,
2055                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2056                                pkts_processed);
2057   return frame->n_vectors;
2058 }
2059
2060
2061 /* *INDENT-OFF* */
2062 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2063   .function = snat_in2out_fast_static_map_fn,
2064   .name = "nat44-in2out-fast",
2065   .vector_size = sizeof (u32),
2066   .format_trace = format_snat_in2out_fast_trace,
2067   .type = VLIB_NODE_TYPE_INTERNAL,
2068
2069   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2070   .error_strings = snat_in2out_error_strings,
2071
2072   .runtime_data_bytes = sizeof (snat_runtime_t),
2073
2074   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2075
2076   /* edit / add dispositions here */
2077   .next_nodes = {
2078     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2079     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2080     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2081     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2082     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2083   },
2084 };
2085 /* *INDENT-ON* */
2086
2087 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node,
2088                               snat_in2out_fast_static_map_fn);
2089
2090 /*
2091  * fd.io coding-style-patch-verification: ON
2092  *
2093  * Local Variables:
2094  * eval: (c-set-style "gnu")
2095  * End:
2096  */