src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31
  32 #include <vppinfra/hash.h>
  33 #include <vppinfra/error.h>
  34 #include <vppinfra/elog.h>
  35
  36 typedef struct
  37 {
  38   u32 sw_if_index;
  39   u32 next_index;
  40   u32 session_index;
  41   u32 is_slow_path;
  42 } snat_in2out_trace_t;
  43
  44 /* packet trace format function */
  45 static u8 *
  46 format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  51   char *tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 *
  62 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 vlib_node_registration_t snat_in2out_node;
  75 vlib_node_registration_t snat_in2out_slowpath_node;
  76 vlib_node_registration_t snat_in2out_fast_node;
  77 vlib_node_registration_t snat_in2out_output_node;
  78 vlib_node_registration_t snat_in2out_output_slowpath_node;
  79 vlib_node_registration_t nat44_in2out_reass_node;
  80
  81 #define foreach_snat_in2out_error                       \
  82 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
  83 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
  84 _(OUT_OF_PORTS, "Out of ports")                         \
  85 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
  86 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  87 _(NO_TRANSLATION, "No translation")                     \
  88 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
  89 _(DROP_FRAGMENT, "Drop fragment")                       \
  90 _(MAX_REASS, "Maximum reassemblies exceeded")           \
  91 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
  92
  93 typedef enum
  94 {
  95 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  96   foreach_snat_in2out_error
  97 #undef _
  98     SNAT_IN2OUT_N_ERROR,
  99 } snat_in2out_error_t;
 100
 101 static char *snat_in2out_error_strings[] = {
 102 #define _(sym,string) string,
 103   foreach_snat_in2out_error
 104 #undef _
 105 };
 106
 107 typedef enum
 108 {
 109   SNAT_IN2OUT_NEXT_LOOKUP,
 110   SNAT_IN2OUT_NEXT_DROP,
 111   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 112   SNAT_IN2OUT_NEXT_SLOW_PATH,
 113   SNAT_IN2OUT_NEXT_REASS,
 114   SNAT_IN2OUT_N_NEXT,
 115 } snat_in2out_next_t;
 116
 117 static inline int
 118 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 119                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 120                     u32 rx_fib_index0, u32 thread_index)
 121 {
 122   udp_header_t *udp0 = ip4_next_header (ip0);
 123   snat_session_key_t key0, sm0;
 124   clib_bihash_kv_8_8_t kv0, value0;
 125
 126   key0.addr = ip0->dst_address;
 127   key0.port = udp0->dst_port;
 128   key0.protocol = proto0;
 129   key0.fib_index = sm->outside_fib_index;
 130   kv0.key = key0.as_u64;
 131
 132   /* NAT packet aimed at external address if */
 133   /* has active sessions */
 134   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 135                               &value0))
 136     {
 137       /* or is static mappings */
 138       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
 139         return 0;
 140     }
 141   else
 142     return 0;
 143
 144   if (sm->forwarding_enabled)
 145     return 1;
 146
 147   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 148                                   rx_fib_index0);
 149 }
 150
 151 static inline int
 152 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 153                                   u32 proto0, u16 src_port, u16 dst_port,
 154                                   u32 thread_index, u32 sw_if_index)
 155 {
 156   snat_session_key_t key0;
 157   clib_bihash_kv_8_8_t kv0, value0;
 158   snat_interface_t *i;
 159
 160   /* src NAT check */
 161   key0.addr = ip0->src_address;
 162   key0.port = src_port;
 163   key0.protocol = proto0;
 164   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 165   kv0.key = key0.as_u64;
 166
 167   if (!clib_bihash_search_8_8
 168       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 169     return 1;
 170
 171   /* dst NAT check */
 172   key0.addr = ip0->dst_address;
 173   key0.port = dst_port;
 174   key0.protocol = proto0;
 175   kv0.key = key0.as_u64;
 176   if (!clib_bihash_search_8_8
 177       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 178     {
 179       /* hairpinning */
 180     /* *INDENT-OFF* */
 181     pool_foreach (i, sm->output_feature_interfaces,
 182     ({
 183       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 184         return 0;
 185     }));
 186     /* *INDENT-ON* */
 187       return 1;
 188     }
 189
 190   return 0;
 191 }
 192
 193 int
 194 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 195 {
 196   snat_main_t *sm = &snat_main;
 197   nat44_is_idle_session_ctx_t *ctx = arg;
 198   snat_session_t *s;
 199   u64 sess_timeout_time;
 200   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 201                                                        ctx->thread_index);
 202   clib_bihash_kv_8_8_t s_kv;
 203
 204   s = pool_elt_at_index (tsm->sessions, kv->value);
 205   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 206   if (ctx->now >= sess_timeout_time)
 207     {
 208       s_kv.key = s->out2in.as_u64;
 209       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 210         nat_log_warn ("out2in key del failed");
 211
 212       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
 213                                            s->out2in.addr.as_u32,
 214                                            s->in2out.protocol,
 215                                            s->in2out.port,
 216                                            s->out2in.port,
 217                                            s->in2out.fib_index);
 218
 219       if (!snat_is_session_static (s))
 220         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 221                                             &s->out2in);
 222
 223       nat44_delete_session (sm, s, ctx->thread_index);
 224       return 1;
 225     }
 226
 227   return 0;
 228 }
 229
 230 static u32
 231 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 232            ip4_header_t * ip0,
 233            u32 rx_fib_index0,
 234            snat_session_key_t * key0,
 235            snat_session_t ** sessionp,
 236            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 237 {
 238   snat_user_t *u;
 239   snat_session_t *s;
 240   clib_bihash_kv_8_8_t kv0;
 241   snat_session_key_t key1;
 242   udp_header_t *udp0 = ip4_next_header (ip0);
 243   u8 is_sm = 0;
 244   nat_outside_fib_t *outside_fib;
 245   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 246   fib_prefix_t pfx = {
 247     .fp_proto = FIB_PROTOCOL_IP4,
 248     .fp_len = 32,
 249     .fp_addr = {
 250                 .ip4.as_u32 = ip0->dst_address.as_u32,
 251                 },
 252   };
 253   nat44_is_idle_session_ctx_t ctx0;
 254
 255   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 256     {
 257       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 258       nat_ipfix_logging_max_sessions (sm->max_translations);
 259       nat_log_notice ("maximum sessions exceeded");
 260       return SNAT_IN2OUT_NEXT_DROP;
 261     }
 262
 263   key1.protocol = key0->protocol;
 264
 265   /* First try to match static mapping by local address and port */
 266   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0, 0))
 267     {
 268       /* Try to create dynamic translation */
 269       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 270                                                thread_index, &key1,
 271                                                sm->port_per_thread,
 272                                                sm->per_thread_data
 273                                                [thread_index].snat_thread_index))
 274         {
 275           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 276           return SNAT_IN2OUT_NEXT_DROP;
 277         }
 278     }
 279   else
 280     is_sm = 1;
 281
 282   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 283                               thread_index);
 284   if (!u)
 285     {
 286       nat_log_warn ("create NAT user failed");
 287       return SNAT_IN2OUT_NEXT_DROP;
 288     }
 289
 290   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 291   if (!s)
 292     {
 293       nat44_delete_user_with_no_session (sm, u, thread_index);
 294       nat_log_warn ("create NAT session failed");
 295       return SNAT_IN2OUT_NEXT_DROP;
 296     }
 297
 298   if (is_sm)
 299     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 300   user_session_increment (sm, u, is_sm);
 301   s->in2out = *key0;
 302   s->out2in = key1;
 303   s->out2in.protocol = key0->protocol;
 304   s->out2in.fib_index = sm->outside_fib_index;
 305   switch (vec_len (sm->outside_fibs))
 306     {
 307     case 0:
 308       s->out2in.fib_index = sm->outside_fib_index;
 309       break;
 310     case 1:
 311       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 312       break;
 313     default:
 314       /* *INDENT-OFF* */
 315       vec_foreach (outside_fib, sm->outside_fibs)
 316         {
 317           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 318           if (FIB_NODE_INDEX_INVALID != fei)
 319             {
 320               if (fib_entry_get_resolving_interface (fei) != ~0)
 321                 {
 322                   s->out2in.fib_index = outside_fib->fib_index;
 323                   break;
 324                 }
 325             }
 326         }
 327       /* *INDENT-ON* */
 328       break;
 329     }
 330   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 331   s->ext_host_port = udp0->dst_port;
 332   *sessionp = s;
 333
 334   /* Add to translation hashes */
 335   ctx0.now = now;
 336   ctx0.thread_index = thread_index;
 337   kv0.key = s->in2out.as_u64;
 338   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 339   if (clib_bihash_add_or_overwrite_stale_8_8
 340       (&sm->per_thread_data[thread_index].in2out, &kv0,
 341        nat44_i2o_is_idle_session_cb, &ctx0))
 342     nat_log_notice ("in2out key add failed");
 343
 344   kv0.key = s->out2in.as_u64;
 345   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 346
 347   if (clib_bihash_add_or_overwrite_stale_8_8
 348       (&sm->per_thread_data[thread_index].out2in, &kv0,
 349        nat44_o2i_is_idle_session_cb, &ctx0))
 350     nat_log_notice ("out2in key add failed");
 351
 352   /* log NAT event */
 353   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
 354                                        s->out2in.addr.as_u32,
 355                                        s->in2out.protocol,
 356                                        s->in2out.port,
 357                                        s->out2in.port, s->in2out.fib_index);
 358   return next0;
 359 }
 360
 361 static_always_inline
 362   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 363                                     snat_session_key_t * p_key0)
 364 {
 365   icmp46_header_t *icmp0;
 366   snat_session_key_t key0;
 367   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 368   ip4_header_t *inner_ip0 = 0;
 369   void *l4_header = 0;
 370   icmp46_header_t *inner_icmp0;
 371
 372   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 373   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 374
 375   if (!icmp_is_error_message (icmp0))
 376     {
 377       key0.protocol = SNAT_PROTOCOL_ICMP;
 378       key0.addr = ip0->src_address;
 379       key0.port = echo0->identifier;
 380     }
 381   else
 382     {
 383       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 384       l4_header = ip4_next_header (inner_ip0);
 385       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 386       key0.addr = inner_ip0->dst_address;
 387       switch (key0.protocol)
 388         {
 389         case SNAT_PROTOCOL_ICMP:
 390           inner_icmp0 = (icmp46_header_t *) l4_header;
 391           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 392           key0.port = inner_echo0->identifier;
 393           break;
 394         case SNAT_PROTOCOL_UDP:
 395         case SNAT_PROTOCOL_TCP:
 396           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 397           break;
 398         default:
 399           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 400         }
 401     }
 402   *p_key0 = key0;
 403   return -1;                    /* success */
 404 }
 405
 406 /**
 407  * Get address and port values to be used for ICMP packet translation
 408  * and create session if needed
 409  *
 410  * @param[in,out] sm             NAT main
 411  * @param[in,out] node           NAT node runtime
 412  * @param[in] thread_index       thread index
 413  * @param[in,out] b0             buffer containing packet to be translated
 414  * @param[out] p_proto           protocol used for matching
 415  * @param[out] p_value           address and port after NAT translation
 416  * @param[out] p_dont_translate  if packet should not be translated
 417  * @param d                      optional parameter
 418  * @param e                      optional parameter
 419  */
 420 u32
 421 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 422                         u32 thread_index, vlib_buffer_t * b0,
 423                         ip4_header_t * ip0, u8 * p_proto,
 424                         snat_session_key_t * p_value,
 425                         u8 * p_dont_translate, void *d, void *e)
 426 {
 427   icmp46_header_t *icmp0;
 428   u32 sw_if_index0;
 429   u32 rx_fib_index0;
 430   snat_session_key_t key0;
 431   snat_session_t *s0 = 0;
 432   u8 dont_translate = 0;
 433   clib_bihash_kv_8_8_t kv0, value0;
 434   u32 next0 = ~0;
 435   int err;
 436
 437   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 438   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 439   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 440
 441   err = icmp_get_key (ip0, &key0);
 442   if (err != -1)
 443     {
 444       b0->error = node->errors[err];
 445       next0 = SNAT_IN2OUT_NEXT_DROP;
 446       goto out;
 447     }
 448   key0.fib_index = rx_fib_index0;
 449
 450   kv0.key = key0.as_u64;
 451
 452   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 453                               &value0))
 454     {
 455       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 456         {
 457           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 458                                                                key0.protocol,
 459                                                                key0.port,
 460                                                                key0.port,
 461                                                                thread_index,
 462                                                                sw_if_index0)))
 463             {
 464               dont_translate = 1;
 465               goto out;
 466             }
 467         }
 468       else
 469         {
 470           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 471                                                  ip0, SNAT_PROTOCOL_ICMP,
 472                                                  rx_fib_index0,
 473                                                  thread_index)))
 474             {
 475               dont_translate = 1;
 476               goto out;
 477             }
 478         }
 479
 480       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 481         {
 482           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 483           next0 = SNAT_IN2OUT_NEXT_DROP;
 484           goto out;
 485         }
 486
 487       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 488                          thread_index, vlib_time_now (sm->vlib_main));
 489
 490       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 491         goto out;
 492     }
 493   else
 494     {
 495       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 496                          icmp0->type != ICMP4_echo_reply &&
 497                          !icmp_is_error_message (icmp0)))
 498         {
 499           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 500           next0 = SNAT_IN2OUT_NEXT_DROP;
 501           goto out;
 502         }
 503
 504       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 505                               value0.value);
 506     }
 507
 508 out:
 509   *p_proto = key0.protocol;
 510   if (s0)
 511     *p_value = s0->out2in;
 512   *p_dont_translate = dont_translate;
 513   if (d)
 514     *(snat_session_t **) d = s0;
 515   return next0;
 516 }
 517
 518 /**
 519  * Get address and port values to be used for ICMP packet translation
 520  *
 521  * @param[in] sm                 NAT main
 522  * @param[in,out] node           NAT node runtime
 523  * @param[in] thread_index       thread index
 524  * @param[in,out] b0             buffer containing packet to be translated
 525  * @param[out] p_proto           protocol used for matching
 526  * @param[out] p_value           address and port after NAT translation
 527  * @param[out] p_dont_translate  if packet should not be translated
 528  * @param d                      optional parameter
 529  * @param e                      optional parameter
 530  */
 531 u32
 532 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 533                         u32 thread_index, vlib_buffer_t * b0,
 534                         ip4_header_t * ip0, u8 * p_proto,
 535                         snat_session_key_t * p_value,
 536                         u8 * p_dont_translate, void *d, void *e)
 537 {
 538   icmp46_header_t *icmp0;
 539   u32 sw_if_index0;
 540   u32 rx_fib_index0;
 541   snat_session_key_t key0;
 542   snat_session_key_t sm0;
 543   u8 dont_translate = 0;
 544   u8 is_addr_only;
 545   u32 next0 = ~0;
 546   int err;
 547
 548   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 549   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 550   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 551
 552   err = icmp_get_key (ip0, &key0);
 553   if (err != -1)
 554     {
 555       b0->error = node->errors[err];
 556       next0 = SNAT_IN2OUT_NEXT_DROP;
 557       goto out2;
 558     }
 559   key0.fib_index = rx_fib_index0;
 560
 561   if (snat_static_mapping_match (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0))
 562     {
 563       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 564                                                   IP_PROTOCOL_ICMP,
 565                                                   rx_fib_index0)))
 566         {
 567           dont_translate = 1;
 568           goto out;
 569         }
 570
 571       if (icmp_is_error_message (icmp0))
 572         {
 573           next0 = SNAT_IN2OUT_NEXT_DROP;
 574           goto out;
 575         }
 576
 577       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 578       next0 = SNAT_IN2OUT_NEXT_DROP;
 579       goto out;
 580     }
 581
 582   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 583                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 584                      !icmp_is_error_message (icmp0)))
 585     {
 586       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 587       next0 = SNAT_IN2OUT_NEXT_DROP;
 588       goto out;
 589     }
 590
 591 out:
 592   *p_value = sm0;
 593 out2:
 594   *p_proto = key0.protocol;
 595   *p_dont_translate = dont_translate;
 596   return next0;
 597 }
 598
 599 u32
 600 icmp_in2out (snat_main_t * sm,
 601              vlib_buffer_t * b0,
 602              ip4_header_t * ip0,
 603              icmp46_header_t * icmp0,
 604              u32 sw_if_index0,
 605              u32 rx_fib_index0,
 606              vlib_node_runtime_t * node,
 607              u32 next0, u32 thread_index, void *d, void *e)
 608 {
 609   snat_session_key_t sm0;
 610   u8 protocol;
 611   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 612   ip4_header_t *inner_ip0;
 613   void *l4_header = 0;
 614   icmp46_header_t *inner_icmp0;
 615   u8 dont_translate;
 616   u32 new_addr0, old_addr0;
 617   u16 old_id0, new_id0;
 618   u16 old_checksum0, new_checksum0;
 619   ip_csum_t sum0;
 620   u16 checksum0;
 621   u32 next0_tmp;
 622
 623   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 624
 625   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 626                                         &protocol, &sm0, &dont_translate, d,
 627                                         e);
 628   if (next0_tmp != ~0)
 629     next0 = next0_tmp;
 630   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 631     goto out;
 632
 633   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 634     {
 635       sum0 = ip_incremental_checksum (0, icmp0,
 636                                       ntohs (ip0->length) -
 637                                       ip4_header_bytes (ip0));
 638       checksum0 = ~ip_csum_fold (sum0);
 639       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 640         {
 641           next0 = SNAT_IN2OUT_NEXT_DROP;
 642           goto out;
 643         }
 644     }
 645
 646   old_addr0 = ip0->src_address.as_u32;
 647   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 648
 649   sum0 = ip0->checksum;
 650   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 651                          src_address /* changed member */ );
 652   ip0->checksum = ip_csum_fold (sum0);
 653
 654   if (icmp0->checksum == 0)
 655     icmp0->checksum = 0xffff;
 656
 657   if (!icmp_is_error_message (icmp0))
 658     {
 659       new_id0 = sm0.port;
 660       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 661         {
 662           old_id0 = echo0->identifier;
 663           new_id0 = sm0.port;
 664           echo0->identifier = new_id0;
 665
 666           sum0 = icmp0->checksum;
 667           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 668                                  identifier);
 669           icmp0->checksum = ip_csum_fold (sum0);
 670         }
 671     }
 672   else
 673     {
 674       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 675       l4_header = ip4_next_header (inner_ip0);
 676
 677       if (!ip4_header_checksum_is_valid (inner_ip0))
 678         {
 679           next0 = SNAT_IN2OUT_NEXT_DROP;
 680           goto out;
 681         }
 682
 683       /* update inner destination IP address */
 684       old_addr0 = inner_ip0->dst_address.as_u32;
 685       inner_ip0->dst_address = sm0.addr;
 686       new_addr0 = inner_ip0->dst_address.as_u32;
 687       sum0 = icmp0->checksum;
 688       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 689                              dst_address /* changed member */ );
 690       icmp0->checksum = ip_csum_fold (sum0);
 691
 692       /* update inner IP header checksum */
 693       old_checksum0 = inner_ip0->checksum;
 694       sum0 = inner_ip0->checksum;
 695       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 696                              dst_address /* changed member */ );
 697       inner_ip0->checksum = ip_csum_fold (sum0);
 698       new_checksum0 = inner_ip0->checksum;
 699       sum0 = icmp0->checksum;
 700       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 701                              checksum);
 702       icmp0->checksum = ip_csum_fold (sum0);
 703
 704       switch (protocol)
 705         {
 706         case SNAT_PROTOCOL_ICMP:
 707           inner_icmp0 = (icmp46_header_t *) l4_header;
 708           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 709
 710           old_id0 = inner_echo0->identifier;
 711           new_id0 = sm0.port;
 712           inner_echo0->identifier = new_id0;
 713
 714           sum0 = icmp0->checksum;
 715           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 716                                  identifier);
 717           icmp0->checksum = ip_csum_fold (sum0);
 718           break;
 719         case SNAT_PROTOCOL_UDP:
 720         case SNAT_PROTOCOL_TCP:
 721           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 722           new_id0 = sm0.port;
 723           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 724
 725           sum0 = icmp0->checksum;
 726           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 727                                  dst_port);
 728           icmp0->checksum = ip_csum_fold (sum0);
 729           break;
 730         default:
 731           ASSERT (0);
 732         }
 733     }
 734
 735   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 736     {
 737       if (sm->deterministic ||
 738           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 739                                       sm->endpoint_dependent))
 740         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 741     }
 742
 743 out:
 744   return next0;
 745 }
 746
 747 static inline u32
 748 icmp_in2out_slow_path (snat_main_t * sm,
 749                        vlib_buffer_t * b0,
 750                        ip4_header_t * ip0,
 751                        icmp46_header_t * icmp0,
 752                        u32 sw_if_index0,
 753                        u32 rx_fib_index0,
 754                        vlib_node_runtime_t * node,
 755                        u32 next0,
 756                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 757 {
 758   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 759                        next0, thread_index, p_s0, 0);
 760   snat_session_t *s0 = *p_s0;
 761   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 762     {
 763       /* Accounting */
 764       nat44_session_update_counters (s0, now,
 765                                      vlib_buffer_length_in_chain
 766                                      (sm->vlib_main, b0));
 767       /* Per-user LRU list maintenance */
 768       nat44_session_update_lru (sm, s0, thread_index);
 769     }
 770   return next0;
 771 }
 772
 773 static int
 774 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 775                              vlib_buffer_t * b,
 776                              ip4_header_t * ip, u32 rx_fib_index)
 777 {
 778   clib_bihash_kv_8_8_t kv, value;
 779   snat_static_mapping_t *m;
 780   snat_session_key_t m_key;
 781   u32 old_addr, new_addr;
 782   ip_csum_t sum;
 783
 784   m_key.addr = ip->src_address;
 785   m_key.port = 0;
 786   m_key.protocol = 0;
 787   m_key.fib_index = rx_fib_index;
 788   kv.key = m_key.as_u64;
 789   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 790     return 1;
 791
 792   m = pool_elt_at_index (sm->static_mappings, value.value);
 793
 794   old_addr = ip->src_address.as_u32;
 795   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 796   sum = ip->checksum;
 797   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 798   ip->checksum = ip_csum_fold (sum);
 799
 800
 801   /* Hairpinning */
 802   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 803     {
 804       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 805       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 806     }
 807
 808   return 0;
 809 }
 810
 811 static inline uword
 812 snat_in2out_node_fn_inline (vlib_main_t * vm,
 813                             vlib_node_runtime_t * node,
 814                             vlib_frame_t * frame, int is_slow_path,
 815                             int is_output_feature)
 816 {
 817   u32 n_left_from, *from, *to_next;
 818   snat_in2out_next_t next_index;
 819   u32 pkts_processed = 0;
 820   snat_main_t *sm = &snat_main;
 821   f64 now = vlib_time_now (vm);
 822   u32 stats_node_index;
 823   u32 thread_index = vm->thread_index;
 824
 825   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
 826     snat_in2out_node.index;
 827
 828   from = vlib_frame_vector_args (frame);
 829   n_left_from = frame->n_vectors;
 830   next_index = node->cached_next_index;
 831
 832   while (n_left_from > 0)
 833     {
 834       u32 n_left_to_next;
 835
 836       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 837
 838       while (n_left_from >= 4 && n_left_to_next >= 2)
 839         {
 840           u32 bi0, bi1;
 841           vlib_buffer_t *b0, *b1;
 842           u32 next0, next1;
 843           u32 sw_if_index0, sw_if_index1;
 844           ip4_header_t *ip0, *ip1;
 845           ip_csum_t sum0, sum1;
 846           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 847           u16 old_port0, new_port0, old_port1, new_port1;
 848           udp_header_t *udp0, *udp1;
 849           tcp_header_t *tcp0, *tcp1;
 850           icmp46_header_t *icmp0, *icmp1;
 851           snat_session_key_t key0, key1;
 852           u32 rx_fib_index0, rx_fib_index1;
 853           u32 proto0, proto1;
 854           snat_session_t *s0 = 0, *s1 = 0;
 855           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 856           u32 iph_offset0 = 0, iph_offset1 = 0;
 857
 858           /* Prefetch next iteration. */
 859           {
 860             vlib_buffer_t *p2, *p3;
 861
 862             p2 = vlib_get_buffer (vm, from[2]);
 863             p3 = vlib_get_buffer (vm, from[3]);
 864
 865             vlib_prefetch_buffer_header (p2, LOAD);
 866             vlib_prefetch_buffer_header (p3, LOAD);
 867
 868             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 869             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 870           }
 871
 872           /* speculatively enqueue b0 and b1 to the current next frame */
 873           to_next[0] = bi0 = from[0];
 874           to_next[1] = bi1 = from[1];
 875           from += 2;
 876           to_next += 2;
 877           n_left_from -= 2;
 878           n_left_to_next -= 2;
 879
 880           b0 = vlib_get_buffer (vm, bi0);
 881           b1 = vlib_get_buffer (vm, bi1);
 882
 883           if (is_output_feature)
 884             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 885
 886           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 887                                   iph_offset0);
 888
 889           udp0 = ip4_next_header (ip0);
 890           tcp0 = (tcp_header_t *) udp0;
 891           icmp0 = (icmp46_header_t *) udp0;
 892
 893           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 894           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 895                                    sw_if_index0);
 896
 897           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 898
 899           if (PREDICT_FALSE (ip0->ttl == 1))
 900             {
 901               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 902               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 903                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 904                                            0);
 905               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 906               goto trace00;
 907             }
 908
 909           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 910
 911           /* Next configured feature, probably ip4-lookup */
 912           if (is_slow_path)
 913             {
 914               if (PREDICT_FALSE (proto0 == ~0))
 915                 {
 916                   if (nat_in2out_sm_unknown_proto
 917                       (sm, b0, ip0, rx_fib_index0))
 918                     {
 919                       next0 = SNAT_IN2OUT_NEXT_DROP;
 920                       b0->error =
 921                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 922                     }
 923                   goto trace00;
 924                 }
 925
 926               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 927                 {
 928                   next0 = icmp_in2out_slow_path
 929                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 930                      node, next0, now, thread_index, &s0);
 931                   goto trace00;
 932                 }
 933             }
 934           else
 935             {
 936               if (PREDICT_FALSE
 937                   (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
 938                 {
 939                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 940                   goto trace00;
 941                 }
 942
 943               if (ip4_is_fragment (ip0))
 944                 {
 945                   next0 = SNAT_IN2OUT_NEXT_REASS;
 946                   goto trace00;
 947                 }
 948             }
 949
 950           key0.addr = ip0->src_address;
 951           key0.port = udp0->src_port;
 952           key0.protocol = proto0;
 953           key0.fib_index = rx_fib_index0;
 954
 955           kv0.key = key0.as_u64;
 956
 957           if (PREDICT_FALSE
 958               (clib_bihash_search_8_8
 959                (&sm->per_thread_data[thread_index].in2out, &kv0,
 960                 &value0) != 0))
 961             {
 962               if (is_slow_path)
 963                 {
 964                   if (is_output_feature)
 965                     {
 966                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
 967                                                                            ip0,
 968                                                                            proto0,
 969                                                                            udp0->src_port,
 970                                                                            udp0->dst_port,
 971                                                                            thread_index,
 972                                                                            sw_if_index0)))
 973                         goto trace00;
 974                     }
 975                   else
 976                     {
 977                       if (PREDICT_FALSE
 978                           (snat_not_translate
 979                            (sm, node, sw_if_index0, ip0, proto0,
 980                             rx_fib_index0, thread_index)))
 981                         goto trace00;
 982                     }
 983
 984                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 985                                      &s0, node, next0, thread_index, now);
 986                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 987                     goto trace00;
 988                 }
 989               else
 990                 {
 991                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 992                   goto trace00;
 993                 }
 994             }
 995           else
 996             s0 =
 997               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 998                                  value0.value);
 999
1000           b0->flags |= VNET_BUFFER_F_IS_NATED;
1001
1002           old_addr0 = ip0->src_address.as_u32;
1003           ip0->src_address = s0->out2in.addr;
1004           new_addr0 = ip0->src_address.as_u32;
1005           if (!is_output_feature)
1006             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1007
1008           sum0 = ip0->checksum;
1009           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1010                                  ip4_header_t,
1011                                  src_address /* changed member */ );
1012           ip0->checksum = ip_csum_fold (sum0);
1013
1014           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1015             {
1016               old_port0 = tcp0->src_port;
1017               tcp0->src_port = s0->out2in.port;
1018               new_port0 = tcp0->src_port;
1019
1020               sum0 = tcp0->checksum;
1021               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1022                                      ip4_header_t,
1023                                      dst_address /* changed member */ );
1024               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1025                                      ip4_header_t /* cheat */ ,
1026                                      length /* changed member */ );
1027               mss_clamping (sm, tcp0, &sum0);
1028               tcp0->checksum = ip_csum_fold (sum0);
1029             }
1030           else
1031             {
1032               old_port0 = udp0->src_port;
1033               udp0->src_port = s0->out2in.port;
1034               udp0->checksum = 0;
1035             }
1036
1037           /* Accounting */
1038           nat44_session_update_counters (s0, now,
1039                                          vlib_buffer_length_in_chain (vm,
1040                                                                       b0));
1041           /* Per-user LRU list maintenance */
1042           nat44_session_update_lru (sm, s0, thread_index);
1043         trace00:
1044
1045           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1046                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1047             {
1048               snat_in2out_trace_t *t =
1049                 vlib_add_trace (vm, node, b0, sizeof (*t));
1050               t->is_slow_path = is_slow_path;
1051               t->sw_if_index = sw_if_index0;
1052               t->next_index = next0;
1053               t->session_index = ~0;
1054               if (s0)
1055                 t->session_index =
1056                   s0 - sm->per_thread_data[thread_index].sessions;
1057             }
1058
1059           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1060
1061           if (is_output_feature)
1062             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1063
1064           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1065                                   iph_offset1);
1066
1067           udp1 = ip4_next_header (ip1);
1068           tcp1 = (tcp_header_t *) udp1;
1069           icmp1 = (icmp46_header_t *) udp1;
1070
1071           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1072           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1073                                    sw_if_index1);
1074
1075           if (PREDICT_FALSE (ip1->ttl == 1))
1076             {
1077               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1078               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1079                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1080                                            0);
1081               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1082               goto trace01;
1083             }
1084
1085           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1086
1087           /* Next configured feature, probably ip4-lookup */
1088           if (is_slow_path)
1089             {
1090               if (PREDICT_FALSE (proto1 == ~0))
1091                 {
1092                   if (nat_in2out_sm_unknown_proto
1093                       (sm, b1, ip1, rx_fib_index1))
1094                     {
1095                       next1 = SNAT_IN2OUT_NEXT_DROP;
1096                       b1->error =
1097                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1098                     }
1099                   goto trace01;
1100                 }
1101
1102               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1103                 {
1104                   next1 = icmp_in2out_slow_path
1105                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1106                      next1, now, thread_index, &s1);
1107                   goto trace01;
1108                 }
1109             }
1110           else
1111             {
1112               if (PREDICT_FALSE
1113                   (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1114                 {
1115                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1116                   goto trace01;
1117                 }
1118
1119               if (ip4_is_fragment (ip1))
1120                 {
1121                   next1 = SNAT_IN2OUT_NEXT_REASS;
1122                   goto trace01;
1123                 }
1124             }
1125
1126           key1.addr = ip1->src_address;
1127           key1.port = udp1->src_port;
1128           key1.protocol = proto1;
1129           key1.fib_index = rx_fib_index1;
1130
1131           kv1.key = key1.as_u64;
1132
1133           if (PREDICT_FALSE
1134               (clib_bihash_search_8_8
1135                (&sm->per_thread_data[thread_index].in2out, &kv1,
1136                 &value1) != 0))
1137             {
1138               if (is_slow_path)
1139                 {
1140                   if (is_output_feature)
1141                     {
1142                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1143                                                                            ip1,
1144                                                                            proto1,
1145                                                                            udp1->src_port,
1146                                                                            udp1->dst_port,
1147                                                                            thread_index,
1148                                                                            sw_if_index1)))
1149                         goto trace01;
1150                     }
1151                   else
1152                     {
1153                       if (PREDICT_FALSE
1154                           (snat_not_translate
1155                            (sm, node, sw_if_index1, ip1, proto1,
1156                             rx_fib_index1, thread_index)))
1157                         goto trace01;
1158                     }
1159
1160                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1161                                      &s1, node, next1, thread_index, now);
1162                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1163                     goto trace01;
1164                 }
1165               else
1166                 {
1167                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1168                   goto trace01;
1169                 }
1170             }
1171           else
1172             s1 =
1173               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1174                                  value1.value);
1175
1176           b1->flags |= VNET_BUFFER_F_IS_NATED;
1177
1178           old_addr1 = ip1->src_address.as_u32;
1179           ip1->src_address = s1->out2in.addr;
1180           new_addr1 = ip1->src_address.as_u32;
1181           if (!is_output_feature)
1182             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1183
1184           sum1 = ip1->checksum;
1185           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1186                                  ip4_header_t,
1187                                  src_address /* changed member */ );
1188           ip1->checksum = ip_csum_fold (sum1);
1189
1190           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1191             {
1192               old_port1 = tcp1->src_port;
1193               tcp1->src_port = s1->out2in.port;
1194               new_port1 = tcp1->src_port;
1195
1196               sum1 = tcp1->checksum;
1197               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1198                                      ip4_header_t,
1199                                      dst_address /* changed member */ );
1200               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1201                                      ip4_header_t /* cheat */ ,
1202                                      length /* changed member */ );
1203               mss_clamping (sm, tcp1, &sum1);
1204               tcp1->checksum = ip_csum_fold (sum1);
1205             }
1206           else
1207             {
1208               old_port1 = udp1->src_port;
1209               udp1->src_port = s1->out2in.port;
1210               udp1->checksum = 0;
1211             }
1212
1213           /* Accounting */
1214           nat44_session_update_counters (s1, now,
1215                                          vlib_buffer_length_in_chain (vm,
1216                                                                       b1));
1217           /* Per-user LRU list maintenance */
1218           nat44_session_update_lru (sm, s1, thread_index);
1219         trace01:
1220
1221           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1222                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1223             {
1224               snat_in2out_trace_t *t =
1225                 vlib_add_trace (vm, node, b1, sizeof (*t));
1226               t->sw_if_index = sw_if_index1;
1227               t->next_index = next1;
1228               t->session_index = ~0;
1229               if (s1)
1230                 t->session_index =
1231                   s1 - sm->per_thread_data[thread_index].sessions;
1232             }
1233
1234           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1235
1236           /* verify speculative enqueues, maybe switch current next frame */
1237           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1238                                            to_next, n_left_to_next,
1239                                            bi0, bi1, next0, next1);
1240         }
1241
1242       while (n_left_from > 0 && n_left_to_next > 0)
1243         {
1244           u32 bi0;
1245           vlib_buffer_t *b0;
1246           u32 next0;
1247           u32 sw_if_index0;
1248           ip4_header_t *ip0;
1249           ip_csum_t sum0;
1250           u32 new_addr0, old_addr0;
1251           u16 old_port0, new_port0;
1252           udp_header_t *udp0;
1253           tcp_header_t *tcp0;
1254           icmp46_header_t *icmp0;
1255           snat_session_key_t key0;
1256           u32 rx_fib_index0;
1257           u32 proto0;
1258           snat_session_t *s0 = 0;
1259           clib_bihash_kv_8_8_t kv0, value0;
1260           u32 iph_offset0 = 0;
1261
1262           /* speculatively enqueue b0 to the current next frame */
1263           bi0 = from[0];
1264           to_next[0] = bi0;
1265           from += 1;
1266           to_next += 1;
1267           n_left_from -= 1;
1268           n_left_to_next -= 1;
1269
1270           b0 = vlib_get_buffer (vm, bi0);
1271           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1272
1273           if (is_output_feature)
1274             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1275
1276           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1277                                   iph_offset0);
1278
1279           udp0 = ip4_next_header (ip0);
1280           tcp0 = (tcp_header_t *) udp0;
1281           icmp0 = (icmp46_header_t *) udp0;
1282
1283           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1284           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1285                                    sw_if_index0);
1286
1287           if (PREDICT_FALSE (ip0->ttl == 1))
1288             {
1289               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1290               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1291                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1292                                            0);
1293               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1294               goto trace0;
1295             }
1296
1297           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1298
1299           /* Next configured feature, probably ip4-lookup */
1300           if (is_slow_path)
1301             {
1302               if (PREDICT_FALSE (proto0 == ~0))
1303                 {
1304                   if (nat_in2out_sm_unknown_proto
1305                       (sm, b0, ip0, rx_fib_index0))
1306                     {
1307                       next0 = SNAT_IN2OUT_NEXT_DROP;
1308                       b0->error =
1309                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1310                     }
1311                   goto trace0;
1312                 }
1313
1314               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1315                 {
1316                   next0 = icmp_in2out_slow_path
1317                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1318                      next0, now, thread_index, &s0);
1319                   goto trace0;
1320                 }
1321             }
1322           else
1323             {
1324               if (PREDICT_FALSE
1325                   (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1326                 {
1327                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1328                   goto trace0;
1329                 }
1330
1331               if (ip4_is_fragment (ip0))
1332                 {
1333                   next0 = SNAT_IN2OUT_NEXT_REASS;
1334                   goto trace0;
1335                 }
1336             }
1337
1338           key0.addr = ip0->src_address;
1339           key0.port = udp0->src_port;
1340           key0.protocol = proto0;
1341           key0.fib_index = rx_fib_index0;
1342
1343           kv0.key = key0.as_u64;
1344
1345           if (clib_bihash_search_8_8
1346               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1347             {
1348               if (is_slow_path)
1349                 {
1350                   if (is_output_feature)
1351                     {
1352                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1353                                                                            ip0,
1354                                                                            proto0,
1355                                                                            udp0->src_port,
1356                                                                            udp0->dst_port,
1357                                                                            thread_index,
1358                                                                            sw_if_index0)))
1359                         goto trace0;
1360                     }
1361                   else
1362                     {
1363                       if (PREDICT_FALSE
1364                           (snat_not_translate
1365                            (sm, node, sw_if_index0, ip0, proto0,
1366                             rx_fib_index0, thread_index)))
1367                         goto trace0;
1368                     }
1369
1370                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1371                                      &s0, node, next0, thread_index, now);
1372
1373                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1374                     goto trace0;
1375                 }
1376               else
1377                 {
1378                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1379                   goto trace0;
1380                 }
1381             }
1382           else
1383             s0 =
1384               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1385                                  value0.value);
1386
1387           b0->flags |= VNET_BUFFER_F_IS_NATED;
1388
1389           old_addr0 = ip0->src_address.as_u32;
1390           ip0->src_address = s0->out2in.addr;
1391           new_addr0 = ip0->src_address.as_u32;
1392           if (!is_output_feature)
1393             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1394
1395           sum0 = ip0->checksum;
1396           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1397                                  ip4_header_t,
1398                                  src_address /* changed member */ );
1399           ip0->checksum = ip_csum_fold (sum0);
1400
1401           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1402             {
1403               old_port0 = tcp0->src_port;
1404               tcp0->src_port = s0->out2in.port;
1405               new_port0 = tcp0->src_port;
1406
1407               sum0 = tcp0->checksum;
1408               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1409                                      ip4_header_t,
1410                                      dst_address /* changed member */ );
1411               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1412                                      ip4_header_t /* cheat */ ,
1413                                      length /* changed member */ );
1414               mss_clamping (sm, tcp0, &sum0);
1415               tcp0->checksum = ip_csum_fold (sum0);
1416             }
1417           else
1418             {
1419               old_port0 = udp0->src_port;
1420               udp0->src_port = s0->out2in.port;
1421               udp0->checksum = 0;
1422             }
1423
1424           /* Accounting */
1425           nat44_session_update_counters (s0, now,
1426                                          vlib_buffer_length_in_chain (vm,
1427                                                                       b0));
1428           /* Per-user LRU list maintenance */
1429           nat44_session_update_lru (sm, s0, thread_index);
1430
1431         trace0:
1432           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1433                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1434             {
1435               snat_in2out_trace_t *t =
1436                 vlib_add_trace (vm, node, b0, sizeof (*t));
1437               t->is_slow_path = is_slow_path;
1438               t->sw_if_index = sw_if_index0;
1439               t->next_index = next0;
1440               t->session_index = ~0;
1441               if (s0)
1442                 t->session_index =
1443                   s0 - sm->per_thread_data[thread_index].sessions;
1444             }
1445
1446           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1447
1448           /* verify speculative enqueue, maybe switch current next frame */
1449           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1450                                            to_next, n_left_to_next,
1451                                            bi0, next0);
1452         }
1453
1454       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1455     }
1456
1457   vlib_node_increment_counter (vm, stats_node_index,
1458                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1459                                pkts_processed);
1460   return frame->n_vectors;
1461 }
1462
1463 static uword
1464 snat_in2out_fast_path_fn (vlib_main_t * vm,
1465                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1466 {
1467   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1468                                      0);
1469 }
1470
1471 /* *INDENT-OFF* */
1472 VLIB_REGISTER_NODE (snat_in2out_node) = {
1473   .function = snat_in2out_fast_path_fn,
1474   .name = "nat44-in2out",
1475   .vector_size = sizeof (u32),
1476   .format_trace = format_snat_in2out_trace,
1477   .type = VLIB_NODE_TYPE_INTERNAL,
1478
1479   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1480   .error_strings = snat_in2out_error_strings,
1481
1482   .runtime_data_bytes = sizeof (snat_runtime_t),
1483
1484   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1485
1486   /* edit / add dispositions here */
1487   .next_nodes = {
1488     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1489     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1490     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1491     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1492     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1493   },
1494 };
1495 /* *INDENT-ON* */
1496
1497 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1498
1499 static uword
1500 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1501                                  vlib_node_runtime_t * node,
1502                                  vlib_frame_t * frame)
1503 {
1504   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1505                                      1);
1506 }
1507
1508 /* *INDENT-OFF* */
1509 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1510   .function = snat_in2out_output_fast_path_fn,
1511   .name = "nat44-in2out-output",
1512   .vector_size = sizeof (u32),
1513   .format_trace = format_snat_in2out_trace,
1514   .type = VLIB_NODE_TYPE_INTERNAL,
1515
1516   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1517   .error_strings = snat_in2out_error_strings,
1518
1519   .runtime_data_bytes = sizeof (snat_runtime_t),
1520
1521   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1522
1523   /* edit / add dispositions here */
1524   .next_nodes = {
1525     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1526     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1527     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1528     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1529     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1530   },
1531 };
1532 /* *INDENT-ON* */
1533
1534 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1535                               snat_in2out_output_fast_path_fn);
1536
1537 static uword
1538 snat_in2out_slow_path_fn (vlib_main_t * vm,
1539                           vlib_node_runtime_t * node, vlib_frame_t * frame)
1540 {
1541   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1542                                      0);
1543 }
1544
1545 /* *INDENT-OFF* */
1546 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1547   .function = snat_in2out_slow_path_fn,
1548   .name = "nat44-in2out-slowpath",
1549   .vector_size = sizeof (u32),
1550   .format_trace = format_snat_in2out_trace,
1551   .type = VLIB_NODE_TYPE_INTERNAL,
1552
1553   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1554   .error_strings = snat_in2out_error_strings,
1555
1556   .runtime_data_bytes = sizeof (snat_runtime_t),
1557
1558   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1559
1560   /* edit / add dispositions here */
1561   .next_nodes = {
1562     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1563     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1564     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1565     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1566     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1567   },
1568 };
1569 /* *INDENT-ON* */
1570
1571 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1572                               snat_in2out_slow_path_fn);
1573
1574 static uword
1575 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1576                                  vlib_node_runtime_t * node,
1577                                  vlib_frame_t * frame)
1578 {
1579   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1580                                      1);
1581 }
1582
1583 /* *INDENT-OFF* */
1584 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1585   .function = snat_in2out_output_slow_path_fn,
1586   .name = "nat44-in2out-output-slowpath",
1587   .vector_size = sizeof (u32),
1588   .format_trace = format_snat_in2out_trace,
1589   .type = VLIB_NODE_TYPE_INTERNAL,
1590
1591   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1592   .error_strings = snat_in2out_error_strings,
1593
1594   .runtime_data_bytes = sizeof (snat_runtime_t),
1595
1596   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1597
1598   /* edit / add dispositions here */
1599   .next_nodes = {
1600     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1601     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1602     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1603     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1604     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1605   },
1606 };
1607 /* *INDENT-ON* */
1608
1609 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1610                               snat_in2out_output_slow_path_fn);
1611
1612 static uword
1613 nat44_in2out_reass_node_fn (vlib_main_t * vm,
1614                             vlib_node_runtime_t * node, vlib_frame_t * frame)
1615 {
1616   u32 n_left_from, *from, *to_next;
1617   snat_in2out_next_t next_index;
1618   u32 pkts_processed = 0;
1619   snat_main_t *sm = &snat_main;
1620   f64 now = vlib_time_now (vm);
1621   u32 thread_index = vm->thread_index;
1622   snat_main_per_thread_data_t *per_thread_data =
1623     &sm->per_thread_data[thread_index];
1624   u32 *fragments_to_drop = 0;
1625   u32 *fragments_to_loopback = 0;
1626
1627   from = vlib_frame_vector_args (frame);
1628   n_left_from = frame->n_vectors;
1629   next_index = node->cached_next_index;
1630
1631   while (n_left_from > 0)
1632     {
1633       u32 n_left_to_next;
1634
1635       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1636
1637       while (n_left_from > 0 && n_left_to_next > 0)
1638         {
1639           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1640           vlib_buffer_t *b0;
1641           u32 next0;
1642           u8 cached0 = 0;
1643           ip4_header_t *ip0;
1644           nat_reass_ip4_t *reass0;
1645           udp_header_t *udp0;
1646           tcp_header_t *tcp0;
1647           snat_session_key_t key0;
1648           clib_bihash_kv_8_8_t kv0, value0;
1649           snat_session_t *s0 = 0;
1650           u16 old_port0, new_port0;
1651           ip_csum_t sum0;
1652
1653           /* speculatively enqueue b0 to the current next frame */
1654           bi0 = from[0];
1655           to_next[0] = bi0;
1656           from += 1;
1657           to_next += 1;
1658           n_left_from -= 1;
1659           n_left_to_next -= 1;
1660
1661           b0 = vlib_get_buffer (vm, bi0);
1662           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1663
1664           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1665           rx_fib_index0 =
1666             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1667                                                  sw_if_index0);
1668
1669           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1670             {
1671               next0 = SNAT_IN2OUT_NEXT_DROP;
1672               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1673               goto trace0;
1674             }
1675
1676           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1677           udp0 = ip4_next_header (ip0);
1678           tcp0 = (tcp_header_t *) udp0;
1679           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1680
1681           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1682                                                  ip0->dst_address,
1683                                                  ip0->fragment_id,
1684                                                  ip0->protocol,
1685                                                  1, &fragments_to_drop);
1686
1687           if (PREDICT_FALSE (!reass0))
1688             {
1689               next0 = SNAT_IN2OUT_NEXT_DROP;
1690               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1691               nat_log_notice ("maximum reassemblies exceeded");
1692               goto trace0;
1693             }
1694
1695           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1696             {
1697               key0.addr = ip0->src_address;
1698               key0.port = udp0->src_port;
1699               key0.protocol = proto0;
1700               key0.fib_index = rx_fib_index0;
1701               kv0.key = key0.as_u64;
1702
1703               if (clib_bihash_search_8_8
1704                   (&per_thread_data->in2out, &kv0, &value0))
1705                 {
1706                   if (PREDICT_FALSE
1707                       (snat_not_translate
1708                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1709                         thread_index)))
1710                     goto trace0;
1711
1712                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1713                                      &s0, node, next0, thread_index, now);
1714
1715                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1716                     goto trace0;
1717
1718                   reass0->sess_index = s0 - per_thread_data->sessions;
1719                 }
1720               else
1721                 {
1722                   s0 = pool_elt_at_index (per_thread_data->sessions,
1723                                           value0.value);
1724                   reass0->sess_index = value0.value;
1725                 }
1726               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1727             }
1728           else
1729             {
1730               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1731                 {
1732                   if (nat_ip4_reass_add_fragment
1733                       (reass0, bi0, &fragments_to_drop))
1734                     {
1735                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1736                       nat_log_notice
1737                         ("maximum fragments per reassembly exceeded");
1738                       next0 = SNAT_IN2OUT_NEXT_DROP;
1739                       goto trace0;
1740                     }
1741                   cached0 = 1;
1742                   goto trace0;
1743                 }
1744               s0 = pool_elt_at_index (per_thread_data->sessions,
1745                                       reass0->sess_index);
1746             }
1747
1748           old_addr0 = ip0->src_address.as_u32;
1749           ip0->src_address = s0->out2in.addr;
1750           new_addr0 = ip0->src_address.as_u32;
1751           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1752
1753           sum0 = ip0->checksum;
1754           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1755                                  ip4_header_t,
1756                                  src_address /* changed member */ );
1757           ip0->checksum = ip_csum_fold (sum0);
1758
1759           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1760             {
1761               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1762                 {
1763                   old_port0 = tcp0->src_port;
1764                   tcp0->src_port = s0->out2in.port;
1765                   new_port0 = tcp0->src_port;
1766
1767                   sum0 = tcp0->checksum;
1768                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1769                                          ip4_header_t,
1770                                          dst_address /* changed member */ );
1771                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1772                                          ip4_header_t /* cheat */ ,
1773                                          length /* changed member */ );
1774                   tcp0->checksum = ip_csum_fold (sum0);
1775                 }
1776               else
1777                 {
1778                   old_port0 = udp0->src_port;
1779                   udp0->src_port = s0->out2in.port;
1780                   udp0->checksum = 0;
1781                 }
1782             }
1783
1784           /* Hairpinning */
1785           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1786                                    s0->ext_host_port, proto0, 0);
1787
1788           /* Accounting */
1789           nat44_session_update_counters (s0, now,
1790                                          vlib_buffer_length_in_chain (vm,
1791                                                                       b0));
1792           /* Per-user LRU list maintenance */
1793           nat44_session_update_lru (sm, s0, thread_index);
1794
1795         trace0:
1796           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1797                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1798             {
1799               nat44_reass_trace_t *t =
1800                 vlib_add_trace (vm, node, b0, sizeof (*t));
1801               t->cached = cached0;
1802               t->sw_if_index = sw_if_index0;
1803               t->next_index = next0;
1804             }
1805
1806           if (cached0)
1807             {
1808               n_left_to_next++;
1809               to_next--;
1810             }
1811           else
1812             {
1813               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1814
1815               /* verify speculative enqueue, maybe switch current next frame */
1816               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1817                                                to_next, n_left_to_next,
1818                                                bi0, next0);
1819             }
1820
1821           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1822             {
1823               from = vlib_frame_vector_args (frame);
1824               u32 len = vec_len (fragments_to_loopback);
1825               if (len <= VLIB_FRAME_SIZE)
1826                 {
1827                   clib_memcpy (from, fragments_to_loopback,
1828                                sizeof (u32) * len);
1829                   n_left_from = len;
1830                   vec_reset_length (fragments_to_loopback);
1831                 }
1832               else
1833                 {
1834                   clib_memcpy (from,
1835                                fragments_to_loopback + (len -
1836                                                         VLIB_FRAME_SIZE),
1837                                sizeof (u32) * VLIB_FRAME_SIZE);
1838                   n_left_from = VLIB_FRAME_SIZE;
1839                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1840                 }
1841             }
1842         }
1843
1844       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1845     }
1846
1847   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
1848                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1849                                pkts_processed);
1850
1851   nat_send_all_to_node (vm, fragments_to_drop, node,
1852                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1853                         SNAT_IN2OUT_NEXT_DROP);
1854
1855   vec_free (fragments_to_drop);
1856   vec_free (fragments_to_loopback);
1857   return frame->n_vectors;
1858 }
1859
1860 /* *INDENT-OFF* */
1861 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
1862   .function = nat44_in2out_reass_node_fn,
1863   .name = "nat44-in2out-reass",
1864   .vector_size = sizeof (u32),
1865   .format_trace = format_nat44_reass_trace,
1866   .type = VLIB_NODE_TYPE_INTERNAL,
1867
1868   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1869   .error_strings = snat_in2out_error_strings,
1870
1871   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1872   .next_nodes = {
1873     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1874     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1875     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1876     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1877     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1878   },
1879 };
1880 /* *INDENT-ON* */
1881
1882 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
1883                               nat44_in2out_reass_node_fn);
1884
1885 static uword
1886 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
1887                                 vlib_node_runtime_t * node,
1888                                 vlib_frame_t * frame)
1889 {
1890   u32 n_left_from, *from, *to_next;
1891   snat_in2out_next_t next_index;
1892   u32 pkts_processed = 0;
1893   snat_main_t *sm = &snat_main;
1894   u32 stats_node_index;
1895
1896   stats_node_index = snat_in2out_fast_node.index;
1897
1898   from = vlib_frame_vector_args (frame);
1899   n_left_from = frame->n_vectors;
1900   next_index = node->cached_next_index;
1901
1902   while (n_left_from > 0)
1903     {
1904       u32 n_left_to_next;
1905
1906       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1907
1908       while (n_left_from > 0 && n_left_to_next > 0)
1909         {
1910           u32 bi0;
1911           vlib_buffer_t *b0;
1912           u32 next0;
1913           u32 sw_if_index0;
1914           ip4_header_t *ip0;
1915           ip_csum_t sum0;
1916           u32 new_addr0, old_addr0;
1917           u16 old_port0, new_port0;
1918           udp_header_t *udp0;
1919           tcp_header_t *tcp0;
1920           icmp46_header_t *icmp0;
1921           snat_session_key_t key0, sm0;
1922           u32 proto0;
1923           u32 rx_fib_index0;
1924
1925           /* speculatively enqueue b0 to the current next frame */
1926           bi0 = from[0];
1927           to_next[0] = bi0;
1928           from += 1;
1929           to_next += 1;
1930           n_left_from -= 1;
1931           n_left_to_next -= 1;
1932
1933           b0 = vlib_get_buffer (vm, bi0);
1934           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1935
1936           ip0 = vlib_buffer_get_current (b0);
1937           udp0 = ip4_next_header (ip0);
1938           tcp0 = (tcp_header_t *) udp0;
1939           icmp0 = (icmp46_header_t *) udp0;
1940
1941           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1942           rx_fib_index0 =
1943             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1944
1945           if (PREDICT_FALSE (ip0->ttl == 1))
1946             {
1947               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1948               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1949                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1950                                            0);
1951               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1952               goto trace0;
1953             }
1954
1955           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1956
1957           if (PREDICT_FALSE (proto0 == ~0))
1958             goto trace0;
1959
1960           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1961             {
1962               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
1963                                    rx_fib_index0, node, next0, ~0, 0, 0);
1964               goto trace0;
1965             }
1966
1967           key0.addr = ip0->src_address;
1968           key0.protocol = proto0;
1969           key0.port = udp0->src_port;
1970           key0.fib_index = rx_fib_index0;
1971
1972           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0))
1973             {
1974               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1975               next0 = SNAT_IN2OUT_NEXT_DROP;
1976               goto trace0;
1977             }
1978
1979           new_addr0 = sm0.addr.as_u32;
1980           new_port0 = sm0.port;
1981           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1982           old_addr0 = ip0->src_address.as_u32;
1983           ip0->src_address.as_u32 = new_addr0;
1984
1985           sum0 = ip0->checksum;
1986           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1987                                  ip4_header_t,
1988                                  src_address /* changed member */ );
1989           ip0->checksum = ip_csum_fold (sum0);
1990
1991           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1992             {
1993               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1994                 {
1995                   old_port0 = tcp0->src_port;
1996                   tcp0->src_port = new_port0;
1997
1998                   sum0 = tcp0->checksum;
1999                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2000                                          ip4_header_t,
2001                                          dst_address /* changed member */ );
2002                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2003                                          ip4_header_t /* cheat */ ,
2004                                          length /* changed member */ );
2005                   mss_clamping (sm, tcp0, &sum0);
2006                   tcp0->checksum = ip_csum_fold (sum0);
2007                 }
2008               else
2009                 {
2010                   old_port0 = udp0->src_port;
2011                   udp0->src_port = new_port0;
2012                   udp0->checksum = 0;
2013                 }
2014             }
2015           else
2016             {
2017               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2018                 {
2019                   sum0 = tcp0->checksum;
2020                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2021                                          ip4_header_t,
2022                                          dst_address /* changed member */ );
2023                   mss_clamping (sm, tcp0, &sum0);
2024                   tcp0->checksum = ip_csum_fold (sum0);
2025                 }
2026             }
2027
2028           /* Hairpinning */
2029           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2030
2031         trace0:
2032           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2033                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2034             {
2035               snat_in2out_trace_t *t =
2036                 vlib_add_trace (vm, node, b0, sizeof (*t));
2037               t->sw_if_index = sw_if_index0;
2038               t->next_index = next0;
2039             }
2040
2041           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2042
2043           /* verify speculative enqueue, maybe switch current next frame */
2044           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2045                                            to_next, n_left_to_next,
2046                                            bi0, next0);
2047         }
2048
2049       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2050     }
2051
2052   vlib_node_increment_counter (vm, stats_node_index,
2053                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2054                                pkts_processed);
2055   return frame->n_vectors;
2056 }
2057
2058
2059 /* *INDENT-OFF* */
2060 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2061   .function = snat_in2out_fast_static_map_fn,
2062   .name = "nat44-in2out-fast",
2063   .vector_size = sizeof (u32),
2064   .format_trace = format_snat_in2out_fast_trace,
2065   .type = VLIB_NODE_TYPE_INTERNAL,
2066
2067   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2068   .error_strings = snat_in2out_error_strings,
2069
2070   .runtime_data_bytes = sizeof (snat_runtime_t),
2071
2072   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2073
2074   /* edit / add dispositions here */
2075   .next_nodes = {
2076     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2077     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2078     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2079     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2080     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2081   },
2082 };
2083 /* *INDENT-ON* */
2084
2085 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node,
2086                               snat_in2out_fast_static_map_fn);
2087
2088 /*
2089  * fd.io coding-style-patch-verification: ON
2090  *
2091  * Local Variables:
2092  * eval: (c-set-style "gnu")
2093  * End:
2094  */