src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <vnet/udp/udp.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 #include <vppinfra/hash.h>
  36 #include <vppinfra/error.h>
  37 #include <vppinfra/elog.h>
  38 #include <nat/lib/nat_inlines.h>
  39
  40 typedef struct
  41 {
  42   u32 sw_if_index;
  43   u32 next_index;
  44   u32 session_index;
  45   u32 is_slow_path;
  46   u32 is_hairpinning;
  47 } snat_in2out_trace_t;
  48
  49 /* packet trace format function */
  50 static u8 *
  51 format_snat_in2out_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  56   char *tag;
  57
  58   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  59
  60   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  61               t->sw_if_index, t->next_index, t->session_index);
  62   if (t->is_hairpinning)
  63     {
  64       s = format (s, ", with-hairpinning");
  65     }
  66
  67   return s;
  68 }
  69
  70 static u8 *
  71 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  72 {
  73   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  74   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  75   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  76
  77   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  78               t->sw_if_index, t->next_index);
  79
  80   return s;
  81 }
  82
  83 #define foreach_snat_in2out_error                       \
  84 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  85 _(IN2OUT_PACKETS, "good in2out packets processed")      \
  86 _(OUT_OF_PORTS, "out of ports")                         \
  87 _(BAD_OUTSIDE_FIB, "outside VRF ID not found")          \
  88 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  89 _(NO_TRANSLATION, "no translation")                     \
  90 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  91 _(DROP_FRAGMENT, "drop fragment")                       \
  92 _(TCP_PACKETS, "TCP packets")                           \
  93 _(UDP_PACKETS, "UDP packets")                           \
  94 _(ICMP_PACKETS, "ICMP packets")                         \
  95 _(OTHER_PACKETS, "other protocol packets")              \
  96 _(FRAGMENTS, "fragments")                               \
  97 _(CANNOT_CREATE_USER, "cannot create NAT user")
  98
  99 typedef enum
 100 {
 101 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 102   foreach_snat_in2out_error
 103 #undef _
 104     SNAT_IN2OUT_N_ERROR,
 105 } snat_in2out_error_t;
 106
 107 static char *snat_in2out_error_strings[] = {
 108 #define _(sym,string) string,
 109   foreach_snat_in2out_error
 110 #undef _
 111 };
 112
 113 typedef enum
 114 {
 115   SNAT_IN2OUT_NEXT_LOOKUP,
 116   SNAT_IN2OUT_NEXT_DROP,
 117   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 118   SNAT_IN2OUT_NEXT_SLOW_PATH,
 119   SNAT_IN2OUT_N_NEXT,
 120 } snat_in2out_next_t;
 121
 122 static inline int
 123 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 124                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 125                     u32 rx_fib_index0, u32 thread_index)
 126 {
 127   udp_header_t *udp0 = ip4_next_header (ip0);
 128   clib_bihash_kv_8_8_t kv0, value0;
 129
 130   init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, sm->outside_fib_index,
 131               proto0);
 132
 133   /* NAT packet aimed at external address if */
 134   /* has active sessions */
 135   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 136                               &value0))
 137     {
 138       /* or is static mappings */
 139       ip4_address_t placeholder_addr;
 140       u16 placeholder_port;
 141       u32 placeholder_fib_index;
 142       if (!snat_static_mapping_match
 143           (sm, ip0->dst_address, udp0->dst_port, sm->outside_fib_index,
 144            proto0, &placeholder_addr, &placeholder_port,
 145            &placeholder_fib_index, 1, 0, 0, 0, 0, 0))
 146         return 0;
 147     }
 148   else
 149     return 0;
 150
 151   if (sm->forwarding_enabled)
 152     return 1;
 153
 154   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 155                                   rx_fib_index0);
 156 }
 157
 158 static inline int
 159 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 160                                   u32 proto0, u16 src_port, u16 dst_port,
 161                                   u32 thread_index, u32 sw_if_index)
 162 {
 163   clib_bihash_kv_8_8_t kv0, value0;
 164   snat_interface_t *i;
 165
 166   /* src NAT check */
 167   init_nat_k (&kv0, ip0->src_address, src_port,
 168               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 169
 170   if (!clib_bihash_search_8_8
 171       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 172     return 1;
 173
 174   /* dst NAT check */
 175   init_nat_k (&kv0, ip0->dst_address, dst_port,
 176               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 177   if (!clib_bihash_search_8_8
 178       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 179     {
 180       /* hairpinning */
 181     /* *INDENT-OFF* */
 182     pool_foreach (i, sm->output_feature_interfaces,
 183     ({
 184       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 185         return 0;
 186     }));
 187     /* *INDENT-ON* */
 188       return 1;
 189     }
 190
 191   return 0;
 192 }
 193
 194 #ifndef CLIB_MARCH_VARIANT
 195 int
 196 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 197 {
 198   snat_main_t *sm = &snat_main;
 199   nat44_is_idle_session_ctx_t *ctx = arg;
 200   snat_session_t *s;
 201   u64 sess_timeout_time;
 202   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 203                                                        ctx->thread_index);
 204   clib_bihash_kv_8_8_t s_kv;
 205
 206   s = pool_elt_at_index (tsm->sessions, kv->value);
 207   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 208   if (ctx->now >= sess_timeout_time)
 209     {
 210       init_nat_o2i_k (&s_kv, s);
 211       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 212         nat_elog_warn ("out2in key del failed");
 213
 214       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 215                                            s->in2out.addr.as_u32,
 216                                            s->out2in.addr.as_u32,
 217                                            s->nat_proto,
 218                                            s->in2out.port,
 219                                            s->out2in.port,
 220                                            s->in2out.fib_index);
 221
 222       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 223                                &s->in2out.addr, s->in2out.port,
 224                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 225
 226       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 227                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 228                    ctx->thread_index);
 229
 230       if (!snat_is_session_static (s))
 231         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 232                                             &s->out2in.addr,
 233                                             s->out2in.port, s->nat_proto);
 234
 235       nat44_delete_session (sm, s, ctx->thread_index);
 236       return 1;
 237     }
 238
 239   return 0;
 240 }
 241 #endif
 242
 243 static u32
 244 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 245            ip4_header_t * ip0,
 246            ip4_address_t i2o_addr,
 247            u16 i2o_port,
 248            u32 rx_fib_index0,
 249            nat_protocol_t nat_proto,
 250            snat_session_t ** sessionp,
 251            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 252 {
 253   snat_user_t *u;
 254   snat_session_t *s = 0;
 255   clib_bihash_kv_8_8_t kv0;
 256   u8 is_sm = 0;
 257   nat_outside_fib_t *outside_fib;
 258   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 259   u8 identity_nat;
 260   fib_prefix_t pfx = {
 261     .fp_proto = FIB_PROTOCOL_IP4,
 262     .fp_len = 32,
 263     .fp_addr = {
 264                 .ip4.as_u32 = ip0->dst_address.as_u32,
 265                 },
 266   };
 267   nat44_is_idle_session_ctx_t ctx0;
 268   ip4_address_t sm_addr;
 269   u16 sm_port;
 270   u32 sm_fib_index;
 271
 272   if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 273     {
 274       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 275       nat_ipfix_logging_max_sessions (thread_index,
 276                                       sm->max_translations_per_thread);
 277       nat_elog_notice ("maximum sessions exceeded");
 278       return SNAT_IN2OUT_NEXT_DROP;
 279     }
 280
 281   /* First try to match static mapping by local address and port */
 282   if (snat_static_mapping_match
 283       (sm, i2o_addr, i2o_port, rx_fib_index0, nat_proto, &sm_addr,
 284        &sm_port, &sm_fib_index, 0, 0, 0, 0, 0, &identity_nat))
 285     {
 286       /* Try to create dynamic translation */
 287       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 288                                                thread_index,
 289                                                nat_proto,
 290                                                &sm_addr, &sm_port,
 291                                                sm->port_per_thread,
 292                                                sm->per_thread_data
 293                                                [thread_index].snat_thread_index))
 294         {
 295           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 296           return SNAT_IN2OUT_NEXT_DROP;
 297         }
 298     }
 299   else
 300     {
 301       if (PREDICT_FALSE (identity_nat))
 302         {
 303           *sessionp = s;
 304           return next0;
 305         }
 306
 307       is_sm = 1;
 308     }
 309
 310   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 311                               thread_index);
 312   if (!u)
 313     {
 314       b0->error = node->errors[SNAT_IN2OUT_ERROR_CANNOT_CREATE_USER];
 315       return SNAT_IN2OUT_NEXT_DROP;
 316     }
 317
 318   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 319   if (!s)
 320     {
 321       nat44_delete_user_with_no_session (sm, u, thread_index);
 322       nat_elog_warn ("create NAT session failed");
 323       return SNAT_IN2OUT_NEXT_DROP;
 324     }
 325
 326   if (is_sm)
 327     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 328   user_session_increment (sm, u, is_sm);
 329   s->in2out.addr = i2o_addr;
 330   s->in2out.port = i2o_port;
 331   s->in2out.fib_index = rx_fib_index0;
 332   s->nat_proto = nat_proto;
 333   s->out2in.addr = sm_addr;
 334   s->out2in.port = sm_port;
 335   s->out2in.fib_index = sm->outside_fib_index;
 336   switch (vec_len (sm->outside_fibs))
 337     {
 338     case 0:
 339       s->out2in.fib_index = sm->outside_fib_index;
 340       break;
 341     case 1:
 342       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 343       break;
 344     default:
 345       /* *INDENT-OFF* */
 346       vec_foreach (outside_fib, sm->outside_fibs)
 347         {
 348           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 349           if (FIB_NODE_INDEX_INVALID != fei)
 350             {
 351               if (fib_entry_get_resolving_interface (fei) != ~0)
 352                 {
 353                   s->out2in.fib_index = outside_fib->fib_index;
 354                   break;
 355                 }
 356             }
 357         }
 358       /* *INDENT-ON* */
 359       break;
 360     }
 361   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 362   s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
 363   *sessionp = s;
 364
 365   /* Add to translation hashes */
 366   ctx0.now = now;
 367   ctx0.thread_index = thread_index;
 368   init_nat_i2o_kv (&kv0, s, s - sm->per_thread_data[thread_index].sessions);
 369   if (clib_bihash_add_or_overwrite_stale_8_8
 370       (&sm->per_thread_data[thread_index].in2out, &kv0,
 371        nat44_i2o_is_idle_session_cb, &ctx0))
 372     nat_elog_notice ("in2out key add failed");
 373
 374   init_nat_o2i_kv (&kv0, s, s - sm->per_thread_data[thread_index].sessions);
 375   if (clib_bihash_add_or_overwrite_stale_8_8
 376       (&sm->per_thread_data[thread_index].out2in, &kv0,
 377        nat44_o2i_is_idle_session_cb, &ctx0))
 378     nat_elog_notice ("out2in key add failed");
 379
 380   /* log NAT event */
 381   snat_ipfix_logging_nat44_ses_create (thread_index,
 382                                        s->in2out.addr.as_u32,
 383                                        s->out2in.addr.as_u32,
 384                                        s->nat_proto,
 385                                        s->in2out.port,
 386                                        s->out2in.port, s->in2out.fib_index);
 387
 388   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 389                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 390                            s->out2in.port, s->nat_proto);
 391
 392   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 393                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 394                &s->ext_host_nat_addr, s->ext_host_nat_port,
 395                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 396
 397   return next0;
 398 }
 399
 400 #ifndef CLIB_MARCH_VARIANT
 401 static_always_inline snat_in2out_error_t
 402 icmp_get_key (vlib_buffer_t * b, ip4_header_t * ip0,
 403               ip4_address_t * addr, u16 * port, nat_protocol_t * nat_proto)
 404 {
 405   icmp46_header_t *icmp0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 413
 414   if (!icmp_type_is_error_message
 415       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 416     {
 417       *nat_proto = NAT_PROTOCOL_ICMP;
 418       *addr = ip0->src_address;
 419       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 420     }
 421   else
 422     {
 423       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 424       l4_header = ip4_next_header (inner_ip0);
 425       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 426       *addr = inner_ip0->dst_address;
 427       switch (*nat_proto)
 428         {
 429         case NAT_PROTOCOL_ICMP:
 430           inner_icmp0 = (icmp46_header_t *) l4_header;
 431           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 432           *port = inner_echo0->identifier;
 433           break;
 434         case NAT_PROTOCOL_UDP:
 435         case NAT_PROTOCOL_TCP:
 436           *port = ((tcp_udp_header_t *) l4_header)->dst_port;
 437           break;
 438         default:
 439           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 440         }
 441     }
 442   return -1;                    /* success */
 443 }
 444
 445 /**
 446  * Get address and port values to be used for ICMP packet translation
 447  * and create session if needed
 448  *
 449  * @param[in,out] sm             NAT main
 450  * @param[in,out] node           NAT node runtime
 451  * @param[in] thread_index       thread index
 452  * @param[in,out] b0             buffer containing packet to be translated
 453  * @param[in,out] ip0            ip header
 454  * @param[out] p_proto           protocol used for matching
 455  * @param[out] p_value           address and port after NAT translation
 456  * @param[out] p_dont_translate  if packet should not be translated
 457  * @param d                      optional parameter
 458  * @param e                      optional parameter
 459  */
 460 u32
 461 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 462                         u32 thread_index, vlib_buffer_t * b0,
 463                         ip4_header_t * ip0, ip4_address_t * addr, u16 * port,
 464                         u32 * fib_index, nat_protocol_t * proto, void *d,
 465                         void *e, u8 * dont_translate)
 466 {
 467   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 468   u32 sw_if_index0;
 469   snat_session_t *s0 = 0;
 470   clib_bihash_kv_8_8_t kv0, value0;
 471   u32 next0 = ~0;
 472   int err;
 473   vlib_main_t *vm = vlib_get_main ();
 474   *dont_translate = 0;
 475
 476   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 477   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 478
 479   err = icmp_get_key (b0, ip0, addr, port, proto);
 480   if (err != -1)
 481     {
 482       b0->error = node->errors[err];
 483       next0 = SNAT_IN2OUT_NEXT_DROP;
 484       goto out;
 485     }
 486
 487   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 488   if (clib_bihash_search_8_8 (&tsm->in2out, &kv0, &value0))
 489     {
 490       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 491         {
 492           if (PREDICT_FALSE
 493               (nat_not_translate_output_feature
 494                (sm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
 495             {
 496               *dont_translate = 1;
 497               goto out;
 498             }
 499         }
 500       else
 501         {
 502           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 503                                                  ip0, NAT_PROTOCOL_ICMP,
 504                                                  *fib_index, thread_index)))
 505             {
 506               *dont_translate = 1;
 507               goto out;
 508             }
 509         }
 510
 511       if (PREDICT_FALSE
 512           (icmp_type_is_error_message
 513            (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
 514         {
 515           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 516           next0 = SNAT_IN2OUT_NEXT_DROP;
 517           goto out;
 518         }
 519
 520       next0 =
 521         slow_path (sm, b0, ip0, *addr, *port, *fib_index, *proto, &s0, node,
 522                    next0, thread_index, vlib_time_now (vm));
 523
 524       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 525         goto out;
 526
 527       if (!s0)
 528         {
 529           *dont_translate = 1;
 530           goto out;
 531         }
 532     }
 533   else
 534     {
 535       if (PREDICT_FALSE
 536           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 537            ICMP4_echo_request
 538            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 539            ICMP4_echo_reply
 540            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 541                                            reass.icmp_type_or_tcp_flags)))
 542         {
 543           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 544           next0 = SNAT_IN2OUT_NEXT_DROP;
 545           goto out;
 546         }
 547
 548       s0 = pool_elt_at_index (tsm->sessions, value0.value);
 549     }
 550
 551 out:
 552   if (s0)
 553     {
 554       *addr = s0->out2in.addr;
 555       *port = s0->out2in.port;
 556       *fib_index = s0->out2in.fib_index;
 557     }
 558   if (d)
 559     *(snat_session_t **) (d) = s0;
 560   return next0;
 561 }
 562 #endif
 563
 564 #ifndef CLIB_MARCH_VARIANT
 565 /**
 566  * Get address and port values to be used for ICMP packet translation
 567  *
 568  * @param[in] sm                 NAT main
 569  * @param[in,out] node           NAT node runtime
 570  * @param[in] thread_index       thread index
 571  * @param[in,out] b0             buffer containing packet to be translated
 572  * @param[in,out] ip0            ip header
 573  * @param[out] p_proto           protocol used for matching
 574  * @param[out] p_value           address and port after NAT translation
 575  * @param[out] p_dont_translate  if packet should not be translated
 576  * @param d                      optional parameter
 577  * @param e                      optional parameter
 578  */
 579 u32
 580 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 581                         u32 thread_index, vlib_buffer_t * b0,
 582                         ip4_header_t * ip0, ip4_address_t * addr, u16 * port,
 583                         u32 * fib_index, nat_protocol_t * proto, void *d,
 584                         void *e, u8 * dont_translate)
 585 {
 586   u32 sw_if_index0;
 587   u8 is_addr_only;
 588   u32 next0 = ~0;
 589   int err;
 590   *dont_translate = 0;
 591
 592   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 593   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 594
 595   err = icmp_get_key (b0, ip0, addr, port, proto);
 596   if (err != -1)
 597     {
 598       b0->error = node->errors[err];
 599       next0 = SNAT_IN2OUT_NEXT_DROP;
 600       goto out;
 601     }
 602
 603   ip4_address_t sm_addr;
 604   u16 sm_port;
 605   u32 sm_fib_index;
 606
 607   if (snat_static_mapping_match
 608       (sm, *addr, *port, *fib_index, *proto, &sm_addr, &sm_port,
 609        &sm_fib_index, 0, &is_addr_only, 0, 0, 0, 0))
 610     {
 611       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 612                                                   IP_PROTOCOL_ICMP,
 613                                                   *fib_index)))
 614         {
 615           *dont_translate = 1;
 616           goto out;
 617         }
 618
 619       if (icmp_type_is_error_message
 620           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 621         {
 622           next0 = SNAT_IN2OUT_NEXT_DROP;
 623           goto out;
 624         }
 625
 626       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 627       next0 = SNAT_IN2OUT_NEXT_DROP;
 628       goto out;
 629     }
 630
 631   if (PREDICT_FALSE
 632       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
 633        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 634            ICMP4_echo_reply || !is_addr_only)
 635        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 636                                        reass.icmp_type_or_tcp_flags)))
 637     {
 638       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 639       next0 = SNAT_IN2OUT_NEXT_DROP;
 640       goto out;
 641     }
 642
 643 out:
 644   return next0;
 645 }
 646 #endif
 647
 648 #ifndef CLIB_MARCH_VARIANT
 649 u32
 650 icmp_in2out (snat_main_t * sm,
 651              vlib_buffer_t * b0,
 652              ip4_header_t * ip0,
 653              icmp46_header_t * icmp0,
 654              u32 sw_if_index0,
 655              u32 rx_fib_index0,
 656              vlib_node_runtime_t * node,
 657              u32 next0, u32 thread_index, void *d, void *e)
 658 {
 659   vlib_main_t *vm = vlib_get_main ();
 660   ip4_address_t addr;
 661   u16 port;
 662   u32 fib_index;
 663   nat_protocol_t protocol;
 664   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 665   ip4_header_t *inner_ip0;
 666   void *l4_header = 0;
 667   icmp46_header_t *inner_icmp0;
 668   u8 dont_translate;
 669   u32 new_addr0, old_addr0;
 670   u16 old_id0, new_id0;
 671   u16 old_checksum0, new_checksum0;
 672   ip_csum_t sum0;
 673   u16 checksum0;
 674   u32 next0_tmp;
 675
 676   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 677
 678   next0_tmp =
 679     sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0, &addr, &port,
 680                               &fib_index, &protocol, d, e, &dont_translate);
 681   if (next0_tmp != ~0)
 682     next0 = next0_tmp;
 683   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 684     goto out;
 685
 686   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 687     {
 688       sum0 =
 689         ip_incremental_checksum_buffer (vm, b0,
 690                                         (u8 *) icmp0 -
 691                                         (u8 *) vlib_buffer_get_current (b0),
 692                                         ntohs (ip0->length) -
 693                                         ip4_header_bytes (ip0), 0);
 694       checksum0 = ~ip_csum_fold (sum0);
 695       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 696         {
 697           next0 = SNAT_IN2OUT_NEXT_DROP;
 698           goto out;
 699         }
 700     }
 701
 702   old_addr0 = ip0->src_address.as_u32;
 703   new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
 704
 705   sum0 = ip0->checksum;
 706   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 707                          src_address /* changed member */ );
 708   ip0->checksum = ip_csum_fold (sum0);
 709
 710   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 711     {
 712       if (icmp0->checksum == 0)
 713         icmp0->checksum = 0xffff;
 714
 715       if (!icmp_type_is_error_message (icmp0->type))
 716         {
 717           new_id0 = port;
 718           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 719             {
 720               old_id0 = echo0->identifier;
 721               new_id0 = port;
 722               echo0->identifier = new_id0;
 723
 724               sum0 = icmp0->checksum;
 725               sum0 =
 726                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 727                                 identifier);
 728               icmp0->checksum = ip_csum_fold (sum0);
 729             }
 730         }
 731       else
 732         {
 733           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 734           l4_header = ip4_next_header (inner_ip0);
 735
 736           if (!ip4_header_checksum_is_valid (inner_ip0))
 737             {
 738               next0 = SNAT_IN2OUT_NEXT_DROP;
 739               goto out;
 740             }
 741
 742           /* update inner destination IP address */
 743           old_addr0 = inner_ip0->dst_address.as_u32;
 744           inner_ip0->dst_address = addr;
 745           new_addr0 = inner_ip0->dst_address.as_u32;
 746           sum0 = icmp0->checksum;
 747           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 748                                  dst_address /* changed member */ );
 749           icmp0->checksum = ip_csum_fold (sum0);
 750
 751           /* update inner IP header checksum */
 752           old_checksum0 = inner_ip0->checksum;
 753           sum0 = inner_ip0->checksum;
 754           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 755                                  dst_address /* changed member */ );
 756           inner_ip0->checksum = ip_csum_fold (sum0);
 757           new_checksum0 = inner_ip0->checksum;
 758           sum0 = icmp0->checksum;
 759           sum0 =
 760             ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 761                             checksum);
 762           icmp0->checksum = ip_csum_fold (sum0);
 763
 764           switch (protocol)
 765             {
 766             case NAT_PROTOCOL_ICMP:
 767               inner_icmp0 = (icmp46_header_t *) l4_header;
 768               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 769
 770               old_id0 = inner_echo0->identifier;
 771               new_id0 = port;
 772               inner_echo0->identifier = new_id0;
 773
 774               sum0 = icmp0->checksum;
 775               sum0 =
 776                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 777                                 identifier);
 778               icmp0->checksum = ip_csum_fold (sum0);
 779               break;
 780             case NAT_PROTOCOL_UDP:
 781             case NAT_PROTOCOL_TCP:
 782               old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 783               new_id0 = port;
 784               ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 785
 786               sum0 = icmp0->checksum;
 787               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 788                                      dst_port);
 789               icmp0->checksum = ip_csum_fold (sum0);
 790               break;
 791             default:
 792               ASSERT (0);
 793             }
 794         }
 795     }
 796
 797   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 798     {
 799       if (sm->deterministic ||
 800           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 801                                       sm->endpoint_dependent))
 802         vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
 803     }
 804
 805 out:
 806   return next0;
 807 }
 808 #endif
 809
 810 static inline u32
 811 icmp_in2out_slow_path (snat_main_t * sm,
 812                        vlib_buffer_t * b0,
 813                        ip4_header_t * ip0,
 814                        icmp46_header_t * icmp0,
 815                        u32 sw_if_index0,
 816                        u32 rx_fib_index0,
 817                        vlib_node_runtime_t * node,
 818                        u32 next0,
 819                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 820 {
 821   vlib_main_t *vm = vlib_get_main ();
 822
 823   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 824                        next0, thread_index, p_s0, 0);
 825   snat_session_t *s0 = *p_s0;
 826   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 827     {
 828       /* Accounting */
 829       nat44_session_update_counters (s0, now,
 830                                      vlib_buffer_length_in_chain
 831                                      (vm, b0), thread_index);
 832       /* Per-user LRU list maintenance */
 833       nat44_session_update_lru (sm, s0, thread_index);
 834     }
 835   return next0;
 836 }
 837
 838 static int
 839 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 840                              vlib_buffer_t * b,
 841                              ip4_header_t * ip, u32 rx_fib_index)
 842 {
 843   clib_bihash_kv_8_8_t kv, value;
 844   snat_static_mapping_t *m;
 845   u32 old_addr, new_addr;
 846   ip_csum_t sum;
 847
 848   init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 849   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 850     return 1;
 851
 852   m = pool_elt_at_index (sm->static_mappings, value.value);
 853
 854   old_addr = ip->src_address.as_u32;
 855   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 856   sum = ip->checksum;
 857   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 858   ip->checksum = ip_csum_fold (sum);
 859
 860
 861   /* Hairpinning */
 862   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 863     {
 864       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 865       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 866     }
 867
 868   return 0;
 869 }
 870
 871 static inline uword
 872 snat_in2out_node_fn_inline (vlib_main_t * vm,
 873                             vlib_node_runtime_t * node,
 874                             vlib_frame_t * frame, int is_slow_path,
 875                             int is_output_feature)
 876 {
 877   u32 n_left_from, *from;
 878   u32 pkts_processed = 0;
 879   snat_main_t *sm = &snat_main;
 880   f64 now = vlib_time_now (vm);
 881   u32 stats_node_index;
 882   u32 thread_index = vm->thread_index;
 883   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 884     0, fragments = 0;
 885
 886   stats_node_index = is_slow_path ? sm->in2out_slowpath_node_index :
 887     sm->in2out_node_index;
 888
 889   from = vlib_frame_vector_args (frame);
 890   n_left_from = frame->n_vectors;
 891
 892   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 893   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 894   vlib_get_buffers (vm, from, b, n_left_from);
 895
 896   while (n_left_from >= 2)
 897     {
 898       vlib_buffer_t *b0, *b1;
 899       u32 next0, next1;
 900       u32 sw_if_index0, sw_if_index1;
 901       ip4_header_t *ip0, *ip1;
 902       ip_csum_t sum0, sum1;
 903       u32 new_addr0, old_addr0, new_addr1, old_addr1;
 904       u16 old_port0, new_port0, old_port1, new_port1;
 905       udp_header_t *udp0, *udp1;
 906       tcp_header_t *tcp0, *tcp1;
 907       icmp46_header_t *icmp0, *icmp1;
 908       u32 rx_fib_index0, rx_fib_index1;
 909       u32 proto0, proto1;
 910       snat_session_t *s0 = 0, *s1 = 0;
 911       clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 912       u32 iph_offset0 = 0, iph_offset1 = 0;
 913
 914       b0 = *b;
 915       b++;
 916       b1 = *b;
 917       b++;
 918
 919       /* Prefetch next iteration. */
 920       if (PREDICT_TRUE (n_left_from >= 4))
 921         {
 922           vlib_buffer_t *p2, *p3;
 923
 924           p2 = *b;
 925           p3 = *(b + 1);
 926
 927           vlib_prefetch_buffer_header (p2, LOAD);
 928           vlib_prefetch_buffer_header (p3, LOAD);
 929
 930           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 931           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
 932         }
 933
 934       if (is_output_feature)
 935         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 936
 937       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 938                               iph_offset0);
 939
 940       udp0 = ip4_next_header (ip0);
 941       tcp0 = (tcp_header_t *) udp0;
 942       icmp0 = (icmp46_header_t *) udp0;
 943
 944       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 945       rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 946                                sw_if_index0);
 947
 948       next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 949
 950       if (PREDICT_FALSE (ip0->ttl == 1))
 951         {
 952           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 953           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 954                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 955                                        0);
 956           next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 957           goto trace00;
 958         }
 959
 960       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 961
 962       /* Next configured feature, probably ip4-lookup */
 963       if (is_slow_path)
 964         {
 965           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 966             {
 967               if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
 968                 {
 969                   next0 = SNAT_IN2OUT_NEXT_DROP;
 970                   b0->error =
 971                     node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 972                 }
 973               other_packets++;
 974               goto trace00;
 975             }
 976
 977           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 978             {
 979               next0 = icmp_in2out_slow_path
 980                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 981                  node, next0, now, thread_index, &s0);
 982               icmp_packets++;
 983               goto trace00;
 984             }
 985         }
 986       else
 987         {
 988           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 989             {
 990               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 991               goto trace00;
 992             }
 993
 994           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 995             {
 996               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 997               goto trace00;
 998             }
 999         }
1000
1001       init_nat_k (&kv0, ip0->src_address,
1002                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1003                   proto0);
1004       if (PREDICT_FALSE
1005           (clib_bihash_search_8_8
1006            (&sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1007         {
1008           if (is_slow_path)
1009             {
1010               if (is_output_feature)
1011                 {
1012                   if (PREDICT_FALSE
1013                       (nat_not_translate_output_feature
1014                        (sm, ip0, proto0,
1015                         vnet_buffer (b0)->ip.reass.l4_src_port,
1016                         vnet_buffer (b0)->ip.reass.l4_dst_port,
1017                         thread_index, sw_if_index0)))
1018                     goto trace00;
1019
1020                   /*
1021                    * Send DHCP packets to the ipv4 stack, or we won't
1022                    * be able to use dhcp client on the outside interface
1023                    */
1024                   if (PREDICT_FALSE
1025                       (proto0 == NAT_PROTOCOL_UDP
1026                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1027                            clib_host_to_net_u16
1028                            (UDP_DST_PORT_dhcp_to_server))
1029                        && ip0->dst_address.as_u32 == 0xffffffff))
1030                     goto trace00;
1031                 }
1032               else
1033                 {
1034                   if (PREDICT_FALSE
1035                       (snat_not_translate
1036                        (sm, node, sw_if_index0, ip0, proto0,
1037                         rx_fib_index0, thread_index)))
1038                     goto trace00;
1039                 }
1040
1041               next0 = slow_path (sm, b0, ip0,
1042                                  ip0->src_address,
1043                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1044                                  rx_fib_index0,
1045                                  proto0, &s0, node, next0, thread_index, now);
1046               if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1047                 goto trace00;
1048
1049               if (PREDICT_FALSE (!s0))
1050                 goto trace00;
1051             }
1052           else
1053             {
1054               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1055               goto trace00;
1056             }
1057         }
1058       else
1059         s0 =
1060           pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1061                              value0.value);
1062
1063       b0->flags |= VNET_BUFFER_F_IS_NATED;
1064
1065       old_addr0 = ip0->src_address.as_u32;
1066       ip0->src_address = s0->out2in.addr;
1067       new_addr0 = ip0->src_address.as_u32;
1068       if (!is_output_feature)
1069         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1070
1071       sum0 = ip0->checksum;
1072       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1073                              ip4_header_t, src_address /* changed member */ );
1074       ip0->checksum = ip_csum_fold (sum0);
1075
1076
1077       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1078         {
1079           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1080             {
1081               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1082               new_port0 = udp0->src_port = s0->out2in.port;
1083               sum0 = tcp0->checksum;
1084               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1085                                      ip4_header_t,
1086                                      dst_address /* changed member */ );
1087               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1088                                      ip4_header_t /* cheat */ ,
1089                                      length /* changed member */ );
1090               mss_clamping (sm->mss_clamping, tcp0, &sum0);
1091               tcp0->checksum = ip_csum_fold (sum0);
1092             }
1093           tcp_packets++;
1094         }
1095       else
1096         {
1097           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1098             {
1099               udp0->src_port = s0->out2in.port;
1100               if (PREDICT_FALSE (udp0->checksum))
1101                 {
1102                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1103                   new_port0 = udp0->src_port;
1104                   sum0 = udp0->checksum;
1105                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address  /* changed member */
1106                     );
1107                   sum0 =
1108                     ip_csum_update (sum0, old_port0, new_port0,
1109                                     ip4_header_t /* cheat */ ,
1110                                     length /* changed member */ );
1111                   udp0->checksum = ip_csum_fold (sum0);
1112                 }
1113             }
1114           udp_packets++;
1115         }
1116
1117       /* Accounting */
1118       nat44_session_update_counters (s0, now,
1119                                      vlib_buffer_length_in_chain (vm, b0),
1120                                      thread_index);
1121       /* Per-user LRU list maintenance */
1122       nat44_session_update_lru (sm, s0, thread_index);
1123     trace00:
1124
1125       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1126                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1127         {
1128           snat_in2out_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
1129           t->is_slow_path = is_slow_path;
1130           t->sw_if_index = sw_if_index0;
1131           t->next_index = next0;
1132           t->session_index = ~0;
1133           if (s0)
1134             t->session_index =
1135               s0 - sm->per_thread_data[thread_index].sessions;
1136         }
1137
1138       pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1139
1140       if (is_output_feature)
1141         iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1142
1143       ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1144                               iph_offset1);
1145
1146       udp1 = ip4_next_header (ip1);
1147       tcp1 = (tcp_header_t *) udp1;
1148       icmp1 = (icmp46_header_t *) udp1;
1149
1150       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1151       rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1152                                sw_if_index1);
1153
1154       if (PREDICT_FALSE (ip1->ttl == 1))
1155         {
1156           vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1157           icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1158                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1159                                        0);
1160           next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1161           goto trace01;
1162         }
1163
1164       proto1 = ip_proto_to_nat_proto (ip1->protocol);
1165
1166       /* Next configured feature, probably ip4-lookup */
1167       if (is_slow_path)
1168         {
1169           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1170             {
1171               if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1172                 {
1173                   next1 = SNAT_IN2OUT_NEXT_DROP;
1174                   b1->error =
1175                     node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1176                 }
1177               other_packets++;
1178               goto trace01;
1179             }
1180
1181           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1182             {
1183               next1 = icmp_in2out_slow_path
1184                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1185                  next1, now, thread_index, &s1);
1186               icmp_packets++;
1187               goto trace01;
1188             }
1189         }
1190       else
1191         {
1192           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1193             {
1194               next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1195               goto trace01;
1196             }
1197
1198           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1199             {
1200               next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1201               goto trace01;
1202             }
1203         }
1204
1205       init_nat_k (&kv1, ip1->src_address,
1206                   vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1207                   proto1);
1208       if (PREDICT_FALSE
1209           (clib_bihash_search_8_8
1210            (&sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1211         {
1212           if (is_slow_path)
1213             {
1214               if (is_output_feature)
1215                 {
1216                   if (PREDICT_FALSE
1217                       (nat_not_translate_output_feature
1218                        (sm, ip1, proto1,
1219                         vnet_buffer (b1)->ip.reass.l4_src_port,
1220                         vnet_buffer (b1)->ip.reass.l4_dst_port,
1221                         thread_index, sw_if_index1)))
1222                     goto trace01;
1223
1224                   /*
1225                    * Send DHCP packets to the ipv4 stack, or we won't
1226                    * be able to use dhcp client on the outside interface
1227                    */
1228                   if (PREDICT_FALSE
1229                       (proto1 == NAT_PROTOCOL_UDP
1230                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1231                            clib_host_to_net_u16
1232                            (UDP_DST_PORT_dhcp_to_server))
1233                        && ip1->dst_address.as_u32 == 0xffffffff))
1234                     goto trace01;
1235                 }
1236               else
1237                 {
1238                   if (PREDICT_FALSE
1239                       (snat_not_translate
1240                        (sm, node, sw_if_index1, ip1, proto1,
1241                         rx_fib_index1, thread_index)))
1242                     goto trace01;
1243                 }
1244
1245               next1 =
1246                 slow_path (sm, b1, ip1, ip1->src_address,
1247                            vnet_buffer (b1)->ip.reass.l4_src_port,
1248                            rx_fib_index1, proto1, &s1, node, next1,
1249                            thread_index, now);
1250               if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1251                 goto trace01;
1252
1253               if (PREDICT_FALSE (!s1))
1254                 goto trace01;
1255             }
1256           else
1257             {
1258               next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1259               goto trace01;
1260             }
1261         }
1262       else
1263         s1 =
1264           pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1265                              value1.value);
1266
1267       b1->flags |= VNET_BUFFER_F_IS_NATED;
1268
1269       old_addr1 = ip1->src_address.as_u32;
1270       ip1->src_address = s1->out2in.addr;
1271       new_addr1 = ip1->src_address.as_u32;
1272       if (!is_output_feature)
1273         vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1274
1275       sum1 = ip1->checksum;
1276       sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1277                              ip4_header_t, src_address /* changed member */ );
1278       ip1->checksum = ip_csum_fold (sum1);
1279
1280       if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1281         {
1282           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1283             {
1284               old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1285               new_port1 = udp1->src_port = s1->out2in.port;
1286               sum1 = tcp1->checksum;
1287               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1288                                      ip4_header_t,
1289                                      dst_address /* changed member */ );
1290               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1291                                      ip4_header_t /* cheat */ ,
1292                                      length /* changed member */ );
1293               mss_clamping (sm->mss_clamping, tcp1, &sum1);
1294               tcp1->checksum = ip_csum_fold (sum1);
1295             }
1296           tcp_packets++;
1297         }
1298       else
1299         {
1300           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1301             {
1302               udp1->src_port = s1->out2in.port;
1303               if (PREDICT_FALSE (udp1->checksum))
1304                 {
1305                   old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1306                   new_port1 = udp1->src_port;
1307                   sum1 = udp1->checksum;
1308                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address  /* changed member */
1309                     );
1310                   sum1 =
1311                     ip_csum_update (sum1, old_port1, new_port1,
1312                                     ip4_header_t /* cheat */ ,
1313                                     length /* changed member */ );
1314                   udp1->checksum = ip_csum_fold (sum1);
1315                 }
1316             }
1317           udp_packets++;
1318         }
1319
1320       /* Accounting */
1321       nat44_session_update_counters (s1, now,
1322                                      vlib_buffer_length_in_chain (vm, b1),
1323                                      thread_index);
1324       /* Per-user LRU list maintenance */
1325       nat44_session_update_lru (sm, s1, thread_index);
1326     trace01:
1327
1328       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1329                          && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1330         {
1331           snat_in2out_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t));
1332           t->sw_if_index = sw_if_index1;
1333           t->next_index = next1;
1334           t->session_index = ~0;
1335           if (s1)
1336             t->session_index =
1337               s1 - sm->per_thread_data[thread_index].sessions;
1338         }
1339
1340       pkts_processed += next1 == SNAT_IN2OUT_NEXT_LOOKUP;
1341
1342       n_left_from -= 2;
1343       next[0] = next0;
1344       next[1] = next1;
1345       next += 2;
1346     }
1347
1348   while (n_left_from > 0)
1349     {
1350       vlib_buffer_t *b0;
1351       u32 next0;
1352       u32 sw_if_index0;
1353       ip4_header_t *ip0;
1354       ip_csum_t sum0;
1355       u32 new_addr0, old_addr0;
1356       u16 old_port0, new_port0;
1357       udp_header_t *udp0;
1358       tcp_header_t *tcp0;
1359       icmp46_header_t *icmp0;
1360       u32 rx_fib_index0;
1361       u32 proto0;
1362       snat_session_t *s0 = 0;
1363       clib_bihash_kv_8_8_t kv0, value0;
1364       u32 iph_offset0 = 0;
1365
1366       b0 = *b;
1367       b++;
1368       next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1369
1370       if (is_output_feature)
1371         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1372
1373       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1374                               iph_offset0);
1375
1376       udp0 = ip4_next_header (ip0);
1377       tcp0 = (tcp_header_t *) udp0;
1378       icmp0 = (icmp46_header_t *) udp0;
1379
1380       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1381       rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1382                                sw_if_index0);
1383
1384       if (PREDICT_FALSE (ip0->ttl == 1))
1385         {
1386           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1387           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1388                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1389                                        0);
1390           next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1391           goto trace0;
1392         }
1393
1394       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1395
1396       /* Next configured feature, probably ip4-lookup */
1397       if (is_slow_path)
1398         {
1399           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1400             {
1401               if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1402                 {
1403                   next0 = SNAT_IN2OUT_NEXT_DROP;
1404                   b0->error =
1405                     node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1406                 }
1407               other_packets++;
1408               goto trace0;
1409             }
1410
1411           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1412             {
1413               next0 = icmp_in2out_slow_path
1414                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1415                  next0, now, thread_index, &s0);
1416               icmp_packets++;
1417               goto trace0;
1418             }
1419         }
1420       else
1421         {
1422           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1423             {
1424               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1425               goto trace0;
1426             }
1427
1428           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1429             {
1430               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1431               goto trace0;
1432             }
1433         }
1434
1435       init_nat_k (&kv0, ip0->src_address,
1436                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1437                   proto0);
1438
1439       if (clib_bihash_search_8_8
1440           (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1441         {
1442           if (is_slow_path)
1443             {
1444               if (is_output_feature)
1445                 {
1446                   if (PREDICT_FALSE
1447                       (nat_not_translate_output_feature
1448                        (sm, ip0, proto0,
1449                         vnet_buffer (b0)->ip.reass.l4_src_port,
1450                         vnet_buffer (b0)->ip.reass.l4_dst_port,
1451                         thread_index, sw_if_index0)))
1452                     goto trace0;
1453
1454                   /*
1455                    * Send DHCP packets to the ipv4 stack, or we won't
1456                    * be able to use dhcp client on the outside interface
1457                    */
1458                   if (PREDICT_FALSE
1459                       (proto0 == NAT_PROTOCOL_UDP
1460                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1461                            clib_host_to_net_u16
1462                            (UDP_DST_PORT_dhcp_to_server))
1463                        && ip0->dst_address.as_u32 == 0xffffffff))
1464                     goto trace0;
1465                 }
1466               else
1467                 {
1468                   if (PREDICT_FALSE
1469                       (snat_not_translate
1470                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1471                         thread_index)))
1472                     goto trace0;
1473                 }
1474
1475               next0 =
1476                 slow_path (sm, b0, ip0, ip0->src_address,
1477                            vnet_buffer (b0)->ip.reass.l4_src_port,
1478                            rx_fib_index0, proto0, &s0, node, next0,
1479                            thread_index, now);
1480
1481               if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1482                 goto trace0;
1483
1484               if (PREDICT_FALSE (!s0))
1485                 goto trace0;
1486             }
1487           else
1488             {
1489               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1490               goto trace0;
1491             }
1492         }
1493       else
1494         s0 =
1495           pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1496                              value0.value);
1497
1498       b0->flags |= VNET_BUFFER_F_IS_NATED;
1499
1500       old_addr0 = ip0->src_address.as_u32;
1501       ip0->src_address = s0->out2in.addr;
1502       new_addr0 = ip0->src_address.as_u32;
1503       if (!is_output_feature)
1504         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1505
1506       sum0 = ip0->checksum;
1507       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1508                              ip4_header_t, src_address /* changed member */ );
1509       ip0->checksum = ip_csum_fold (sum0);
1510
1511       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1512         {
1513           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1514             {
1515               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1516               new_port0 = udp0->src_port = s0->out2in.port;
1517               sum0 = tcp0->checksum;
1518               sum0 =
1519                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1520                                 dst_address /* changed member */ );
1521               sum0 =
1522                 ip_csum_update (sum0, old_port0, new_port0,
1523                                 ip4_header_t /* cheat */ ,
1524                                 length /* changed member */ );
1525               mss_clamping (sm->mss_clamping, tcp0, &sum0);
1526               tcp0->checksum = ip_csum_fold (sum0);
1527             }
1528           tcp_packets++;
1529         }
1530       else
1531         {
1532           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1533             {
1534               udp0->src_port = s0->out2in.port;
1535               if (PREDICT_FALSE (udp0->checksum))
1536                 {
1537                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1538                   new_port0 = udp0->src_port;
1539                   sum0 = udp0->checksum;
1540                   sum0 =
1541                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1542                                     dst_address /* changed member */ );
1543                   sum0 =
1544                     ip_csum_update (sum0, old_port0, new_port0,
1545                                     ip4_header_t /* cheat */ ,
1546                                     length /* changed member */ );
1547                   udp0->checksum = ip_csum_fold (sum0);
1548                 }
1549             }
1550           udp_packets++;
1551         }
1552
1553       /* Accounting */
1554       nat44_session_update_counters (s0, now,
1555                                      vlib_buffer_length_in_chain (vm, b0),
1556                                      thread_index);
1557       /* Per-user LRU list maintenance */
1558       nat44_session_update_lru (sm, s0, thread_index);
1559
1560     trace0:
1561       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1562                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1563         {
1564           snat_in2out_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
1565           t->is_slow_path = is_slow_path;
1566           t->sw_if_index = sw_if_index0;
1567           t->next_index = next0;
1568           t->session_index = ~0;
1569           if (s0)
1570             t->session_index =
1571               s0 - sm->per_thread_data[thread_index].sessions;
1572         }
1573
1574       pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1575
1576       n_left_from--;
1577       next[0] = next0;
1578       next++;
1579     }
1580
1581   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1582                                frame->n_vectors);
1583
1584   vlib_node_increment_counter (vm, stats_node_index,
1585                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1586                                pkts_processed);
1587   vlib_node_increment_counter (vm, stats_node_index,
1588                                SNAT_IN2OUT_ERROR_TCP_PACKETS, tcp_packets);
1589   vlib_node_increment_counter (vm, stats_node_index,
1590                                SNAT_IN2OUT_ERROR_UDP_PACKETS, udp_packets);
1591   vlib_node_increment_counter (vm, stats_node_index,
1592                                SNAT_IN2OUT_ERROR_ICMP_PACKETS, icmp_packets);
1593   vlib_node_increment_counter (vm, stats_node_index,
1594                                SNAT_IN2OUT_ERROR_OTHER_PACKETS,
1595                                other_packets);
1596   vlib_node_increment_counter (vm, stats_node_index,
1597                                SNAT_IN2OUT_ERROR_FRAGMENTS, fragments);
1598
1599   return frame->n_vectors;
1600 }
1601
1602 VLIB_NODE_FN (snat_in2out_node) (vlib_main_t * vm,
1603                                  vlib_node_runtime_t * node,
1604                                  vlib_frame_t * frame)
1605 {
1606   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1607                                      0);
1608 }
1609
1610 /* *INDENT-OFF* */
1611 VLIB_REGISTER_NODE (snat_in2out_node) = {
1612   .name = "nat44-in2out",
1613   .vector_size = sizeof (u32),
1614   .format_trace = format_snat_in2out_trace,
1615   .type = VLIB_NODE_TYPE_INTERNAL,
1616
1617   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1618   .error_strings = snat_in2out_error_strings,
1619
1620   .runtime_data_bytes = sizeof (snat_runtime_t),
1621
1622   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1623
1624   /* edit / add dispositions here */
1625   .next_nodes = {
1626     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1627     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1628     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1629     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1630   },
1631 };
1632 /* *INDENT-ON* */
1633
1634 VLIB_NODE_FN (snat_in2out_output_node) (vlib_main_t * vm,
1635                                         vlib_node_runtime_t * node,
1636                                         vlib_frame_t * frame)
1637 {
1638   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1639                                      1);
1640 }
1641
1642 /* *INDENT-OFF* */
1643 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1644   .name = "nat44-in2out-output",
1645   .vector_size = sizeof (u32),
1646   .format_trace = format_snat_in2out_trace,
1647   .type = VLIB_NODE_TYPE_INTERNAL,
1648
1649   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1650   .error_strings = snat_in2out_error_strings,
1651
1652   .runtime_data_bytes = sizeof (snat_runtime_t),
1653
1654   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1655
1656   /* edit / add dispositions here */
1657   .next_nodes = {
1658     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1659     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1660     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1661     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1662   },
1663 };
1664 /* *INDENT-ON* */
1665
1666 VLIB_NODE_FN (snat_in2out_slowpath_node) (vlib_main_t * vm,
1667                                           vlib_node_runtime_t * node,
1668                                           vlib_frame_t * frame)
1669 {
1670   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1671                                      0);
1672 }
1673
1674 /* *INDENT-OFF* */
1675 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1676   .name = "nat44-in2out-slowpath",
1677   .vector_size = sizeof (u32),
1678   .format_trace = format_snat_in2out_trace,
1679   .type = VLIB_NODE_TYPE_INTERNAL,
1680
1681   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1682   .error_strings = snat_in2out_error_strings,
1683
1684   .runtime_data_bytes = sizeof (snat_runtime_t),
1685
1686   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1687
1688   /* edit / add dispositions here */
1689   .next_nodes = {
1690     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1691     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1692     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1693     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1694   },
1695 };
1696 /* *INDENT-ON* */
1697
1698 VLIB_NODE_FN (snat_in2out_output_slowpath_node) (vlib_main_t * vm,
1699                                                  vlib_node_runtime_t * node,
1700                                                  vlib_frame_t * frame)
1701 {
1702   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1703                                      1);
1704 }
1705
1706 /* *INDENT-OFF* */
1707 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1708   .name = "nat44-in2out-output-slowpath",
1709   .vector_size = sizeof (u32),
1710   .format_trace = format_snat_in2out_trace,
1711   .type = VLIB_NODE_TYPE_INTERNAL,
1712
1713   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1714   .error_strings = snat_in2out_error_strings,
1715
1716   .runtime_data_bytes = sizeof (snat_runtime_t),
1717
1718   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1719
1720   /* edit / add dispositions here */
1721   .next_nodes = {
1722     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1723     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1724     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1725     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1726   },
1727 };
1728 /* *INDENT-ON* */
1729
1730 VLIB_NODE_FN (snat_in2out_fast_node) (vlib_main_t * vm,
1731                                       vlib_node_runtime_t * node,
1732                                       vlib_frame_t * frame)
1733 {
1734   u32 n_left_from, *from, *to_next;
1735   snat_in2out_next_t next_index;
1736   u32 pkts_processed = 0;
1737   snat_main_t *sm = &snat_main;
1738   u32 stats_node_index;
1739   int is_hairpinning = 0;
1740
1741   stats_node_index = sm->in2out_fast_node_index;
1742
1743   from = vlib_frame_vector_args (frame);
1744   n_left_from = frame->n_vectors;
1745   next_index = node->cached_next_index;
1746
1747   while (n_left_from > 0)
1748     {
1749       u32 n_left_to_next;
1750
1751       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1752
1753       while (n_left_from > 0 && n_left_to_next > 0)
1754         {
1755           u32 bi0;
1756           vlib_buffer_t *b0;
1757           u32 next0;
1758           u32 sw_if_index0;
1759           ip4_header_t *ip0;
1760           ip_csum_t sum0;
1761           u32 new_addr0, old_addr0;
1762           u16 old_port0, new_port0;
1763           udp_header_t *udp0;
1764           tcp_header_t *tcp0;
1765           icmp46_header_t *icmp0;
1766           u32 proto0;
1767           u32 rx_fib_index0;
1768           ip4_address_t sm0_addr;
1769           u16 sm0_port;
1770           u32 sm0_fib_index;
1771
1772
1773           /* speculatively enqueue b0 to the current next frame */
1774           bi0 = from[0];
1775           to_next[0] = bi0;
1776           from += 1;
1777           to_next += 1;
1778           n_left_from -= 1;
1779           n_left_to_next -= 1;
1780
1781           b0 = vlib_get_buffer (vm, bi0);
1782           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1783
1784           ip0 = vlib_buffer_get_current (b0);
1785           udp0 = ip4_next_header (ip0);
1786           tcp0 = (tcp_header_t *) udp0;
1787           icmp0 = (icmp46_header_t *) udp0;
1788
1789           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1790           rx_fib_index0 =
1791             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1792
1793           if (PREDICT_FALSE (ip0->ttl == 1))
1794             {
1795               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1796               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1797                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1798                                            0);
1799               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1800               goto trace0;
1801             }
1802
1803           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1804
1805           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1806             goto trace0;
1807
1808           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1809             {
1810               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
1811                                    rx_fib_index0, node, next0, ~0, 0, 0);
1812               goto trace0;
1813             }
1814
1815           if (snat_static_mapping_match
1816               (sm, ip0->src_address, udp0->src_port, rx_fib_index0, proto0,
1817                &sm0_addr, &sm0_port, &sm0_fib_index, 0, 0, 0, 0, 0, 0))
1818             {
1819               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1820               next0 = SNAT_IN2OUT_NEXT_DROP;
1821               goto trace0;
1822             }
1823
1824           new_addr0 = sm0_addr.as_u32;
1825           new_port0 = sm0_port;
1826           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1827           old_addr0 = ip0->src_address.as_u32;
1828           ip0->src_address.as_u32 = new_addr0;
1829
1830           sum0 = ip0->checksum;
1831           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1832                                  ip4_header_t,
1833                                  src_address /* changed member */ );
1834           ip0->checksum = ip_csum_fold (sum0);
1835
1836           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1837             {
1838               old_port0 = udp0->src_port;
1839               udp0->src_port = new_port0;
1840
1841               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1842                 {
1843                   sum0 = tcp0->checksum;
1844                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1845                                          ip4_header_t,
1846                                          dst_address /* changed member */ );
1847                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1848                                          ip4_header_t /* cheat */ ,
1849                                          length /* changed member */ );
1850                   mss_clamping (sm->mss_clamping, tcp0, &sum0);
1851                   tcp0->checksum = ip_csum_fold (sum0);
1852                 }
1853               else if (udp0->checksum)
1854                 {
1855                   sum0 = udp0->checksum;
1856                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1857                                          ip4_header_t,
1858                                          dst_address /* changed member */ );
1859                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1860                                          ip4_header_t /* cheat */ ,
1861                                          length /* changed member */ );
1862                   udp0->checksum = ip_csum_fold (sum0);
1863                 }
1864             }
1865           else
1866             {
1867               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1868                 {
1869                   sum0 = tcp0->checksum;
1870                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1871                                          ip4_header_t,
1872                                          dst_address /* changed member */ );
1873                   mss_clamping (sm->mss_clamping, tcp0, &sum0);
1874                   tcp0->checksum = ip_csum_fold (sum0);
1875                 }
1876               else if (udp0->checksum)
1877                 {
1878                   sum0 = udp0->checksum;
1879                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1880                                          ip4_header_t,
1881                                          dst_address /* changed member */ );
1882                   udp0->checksum = ip_csum_fold (sum0);
1883                 }
1884             }
1885
1886           /* Hairpinning */
1887           snat_hairpinning (vm, node, sm, b0, ip0, udp0, tcp0, proto0, 0,
1888                             0 /* do_trace */ );
1889           is_hairpinning = 1;
1890
1891         trace0:
1892           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1893                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1894             {
1895               snat_in2out_trace_t *t =
1896                 vlib_add_trace (vm, node, b0, sizeof (*t));
1897               t->sw_if_index = sw_if_index0;
1898               t->next_index = next0;
1899               t->is_hairpinning = is_hairpinning;
1900             }
1901
1902           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1903
1904           /* verify speculative enqueue, maybe switch current next frame */
1905           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1906                                            to_next, n_left_to_next,
1907                                            bi0, next0);
1908         }
1909
1910       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1911     }
1912
1913   vlib_node_increment_counter (vm, stats_node_index,
1914                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1915                                pkts_processed);
1916   return frame->n_vectors;
1917 }
1918
1919
1920 /* *INDENT-OFF* */
1921 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
1922   .name = "nat44-in2out-fast",
1923   .vector_size = sizeof (u32),
1924   .format_trace = format_snat_in2out_fast_trace,
1925   .type = VLIB_NODE_TYPE_INTERNAL,
1926
1927   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1928   .error_strings = snat_in2out_error_strings,
1929
1930   .runtime_data_bytes = sizeof (snat_runtime_t),
1931
1932   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1933
1934   /* edit / add dispositions here */
1935   .next_nodes = {
1936     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1937     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1938     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1939     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1940   },
1941 };
1942 /* *INDENT-ON* */
1943
1944 /*
1945  * fd.io coding-style-patch-verification: ON
1946  *
1947  * Local Variables:
1948  * eval: (c-set-style "gnu")
1949  * End:
1950  */