src/plugins/nat/nat44-ei/nat44_ei_in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp_local.h>
  27 #include <nat/nat.h>
  28 #include <nat/lib/ipfix_logging.h>
  29 #include <nat/nat_inlines.h>
  30 #include <nat/lib/nat_syslog.h>
  31 #include <nat/nat44-ei/nat44_ei_inlines.h>
  32 #include <nat/nat44-ei/nat44_ei.h>
  33
  34 #include <vppinfra/hash.h>
  35 #include <vppinfra/error.h>
  36 #include <vppinfra/elog.h>
  37 #include <nat/lib/nat_inlines.h>
  38 #include <nat/nat44_hairpinning.h>
  39
  40 typedef struct
  41 {
  42   u32 sw_if_index;
  43   u32 next_index;
  44   u32 session_index;
  45   u32 is_slow_path;
  46   u32 is_hairpinning;
  47 } snat_in2out_trace_t;
  48
  49 /* packet trace format function */
  50 static u8 *
  51 format_snat_in2out_trace (u8 * s, va_list * args)
  52 {
  53   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  54   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  55   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  56   char *tag;
  57
  58   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  59
  60   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  61               t->sw_if_index, t->next_index, t->session_index);
  62   if (t->is_hairpinning)
  63     {
  64       s = format (s, ", with-hairpinning");
  65     }
  66
  67   return s;
  68 }
  69
  70 static u8 *
  71 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  72 {
  73   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  74   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  75   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  76
  77   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  78               t->sw_if_index, t->next_index);
  79
  80   return s;
  81 }
  82
  83 #define foreach_snat_in2out_error                       \
  84 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  85 _(OUT_OF_PORTS, "out of ports")                         \
  86 _(BAD_OUTSIDE_FIB, "outside VRF ID not found")          \
  87 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  88 _(NO_TRANSLATION, "no translation")                     \
  89 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  90 _(CANNOT_CREATE_USER, "cannot create NAT user")
  91
  92 typedef enum
  93 {
  94 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  95   foreach_snat_in2out_error
  96 #undef _
  97     SNAT_IN2OUT_N_ERROR,
  98 } snat_in2out_error_t;
  99
 100 static char *snat_in2out_error_strings[] = {
 101 #define _(sym,string) string,
 102   foreach_snat_in2out_error
 103 #undef _
 104 };
 105
 106 typedef enum
 107 {
 108   SNAT_IN2OUT_NEXT_LOOKUP,
 109   SNAT_IN2OUT_NEXT_DROP,
 110   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 111   SNAT_IN2OUT_NEXT_SLOW_PATH,
 112   SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF,
 113   SNAT_IN2OUT_N_NEXT,
 114 } snat_in2out_next_t;
 115
 116 typedef enum
 117 {
 118   NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP,
 119   NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP,
 120   NAT44_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
 121 } nat44_in2out_hairpinnig_finish_next_t;
 122
 123 static inline int
 124 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 125                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 126                     u32 rx_fib_index0, u32 thread_index)
 127 {
 128   udp_header_t *udp0 = ip4_next_header (ip0);
 129   clib_bihash_kv_8_8_t kv0, value0;
 130
 131   init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, sm->outside_fib_index,
 132               proto0);
 133
 134   /* NAT packet aimed at external address if */
 135   /* has active sessions */
 136   if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 137     {
 138       /* or is static mappings */
 139       ip4_address_t placeholder_addr;
 140       u16 placeholder_port;
 141       u32 placeholder_fib_index;
 142       if (!nat44_ei_static_mapping_match (ip0->dst_address, udp0->dst_port,
 143                                           sm->outside_fib_index, proto0,
 144                                           &placeholder_addr, &placeholder_port,
 145                                           &placeholder_fib_index, 1, 0, 0))
 146         return 0;
 147     }
 148   else
 149     return 0;
 150
 151   if (sm->forwarding_enabled)
 152     return 1;
 153
 154   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 155                                   rx_fib_index0);
 156 }
 157
 158 static inline int
 159 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 160                                   u32 proto0, u16 src_port, u16 dst_port,
 161                                   u32 thread_index, u32 sw_if_index)
 162 {
 163   clib_bihash_kv_8_8_t kv0, value0;
 164   snat_interface_t *i;
 165
 166   /* src NAT check */
 167   init_nat_k (&kv0, ip0->src_address, src_port,
 168               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 169
 170   if (!clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
 171     return 1;
 172
 173   /* dst NAT check */
 174   init_nat_k (&kv0, ip0->dst_address, dst_port,
 175               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 176   if (!clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 177     {
 178       /* hairpinning */
 179       pool_foreach (i, sm->output_feature_interfaces)
 180         {
 181           if ((nat_interface_is_inside (i)) && (sw_if_index == i->sw_if_index))
 182             return 0;
 183         }
 184       return 1;
 185     }
 186
 187   return 0;
 188 }
 189
 190 #ifndef CLIB_MARCH_VARIANT
 191 int
 192 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 193 {
 194   snat_main_t *sm = &snat_main;
 195   nat44_is_idle_session_ctx_t *ctx = arg;
 196   snat_session_t *s;
 197   u64 sess_timeout_time;
 198   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 199                                                        ctx->thread_index);
 200   clib_bihash_kv_8_8_t s_kv;
 201
 202   s = pool_elt_at_index (tsm->sessions, kv->value);
 203   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 204   if (ctx->now >= sess_timeout_time)
 205     {
 206       init_nat_o2i_k (&s_kv, s);
 207       if (clib_bihash_add_del_8_8 (&sm->out2in, &s_kv, 0))
 208         nat_elog_warn ("out2in key del failed");
 209
 210       nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 211                                           s->in2out.addr.as_u32,
 212                                           s->out2in.addr.as_u32,
 213                                           s->nat_proto,
 214                                           s->in2out.port,
 215                                           s->out2in.port,
 216                                           s->in2out.fib_index);
 217
 218       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 219                                &s->in2out.addr, s->in2out.port,
 220                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 221
 222       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 223                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 224                    ctx->thread_index);
 225
 226       if (!snat_is_session_static (s))
 227         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 228                                             &s->out2in.addr,
 229                                             s->out2in.port, s->nat_proto);
 230
 231       nat44_delete_session (sm, s, ctx->thread_index);
 232       return 1;
 233     }
 234
 235   return 0;
 236 }
 237 #endif
 238
 239 static u32
 240 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 241            ip4_header_t * ip0,
 242            ip4_address_t i2o_addr,
 243            u16 i2o_port,
 244            u32 rx_fib_index0,
 245            nat_protocol_t nat_proto,
 246            snat_session_t ** sessionp,
 247            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 248 {
 249   snat_user_t *u;
 250   snat_session_t *s = 0;
 251   clib_bihash_kv_8_8_t kv0;
 252   u8 is_sm = 0;
 253   nat_outside_fib_t *outside_fib;
 254   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 255   u8 identity_nat;
 256   fib_prefix_t pfx = {
 257     .fp_proto = FIB_PROTOCOL_IP4,
 258     .fp_len = 32,
 259     .fp_addr = {
 260                 .ip4.as_u32 = ip0->dst_address.as_u32,
 261                 },
 262   };
 263   nat44_is_idle_session_ctx_t ctx0;
 264   ip4_address_t sm_addr;
 265   u16 sm_port;
 266   u32 sm_fib_index;
 267
 268   if (PREDICT_FALSE (nat44_ei_maximum_sessions_exceeded (sm, thread_index)))
 269     {
 270       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 271       nat_ipfix_logging_max_sessions (thread_index,
 272                                       sm->max_translations_per_thread);
 273       nat_elog_notice ("maximum sessions exceeded");
 274       return SNAT_IN2OUT_NEXT_DROP;
 275     }
 276
 277   /* First try to match static mapping by local address and port */
 278   if (nat44_ei_static_mapping_match (i2o_addr, i2o_port, rx_fib_index0,
 279                                      nat_proto, &sm_addr, &sm_port,
 280                                      &sm_fib_index, 0, 0, &identity_nat))
 281     {
 282       /* Try to create dynamic translation */
 283       if (sm->alloc_addr_and_port (
 284             sm->addresses, rx_fib_index0, thread_index, nat_proto, &sm_addr,
 285             &sm_port, sm->port_per_thread,
 286             sm->per_thread_data[thread_index].snat_thread_index))
 287         {
 288           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 289           return SNAT_IN2OUT_NEXT_DROP;
 290         }
 291     }
 292   else
 293     {
 294       if (PREDICT_FALSE (identity_nat))
 295         {
 296           *sessionp = s;
 297           return next0;
 298         }
 299
 300       is_sm = 1;
 301     }
 302
 303   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 304                               thread_index);
 305   if (!u)
 306     {
 307       b0->error = node->errors[SNAT_IN2OUT_ERROR_CANNOT_CREATE_USER];
 308       return SNAT_IN2OUT_NEXT_DROP;
 309     }
 310
 311   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 312   if (!s)
 313     {
 314       nat44_delete_user_with_no_session (sm, u, thread_index);
 315       nat_elog_warn ("create NAT session failed");
 316       return SNAT_IN2OUT_NEXT_DROP;
 317     }
 318
 319   if (is_sm)
 320     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 321   user_session_increment (sm, u, is_sm);
 322   s->in2out.addr = i2o_addr;
 323   s->in2out.port = i2o_port;
 324   s->in2out.fib_index = rx_fib_index0;
 325   s->nat_proto = nat_proto;
 326   s->out2in.addr = sm_addr;
 327   s->out2in.port = sm_port;
 328   s->out2in.fib_index = sm->outside_fib_index;
 329   switch (vec_len (sm->outside_fibs))
 330     {
 331     case 0:
 332       s->out2in.fib_index = sm->outside_fib_index;
 333       break;
 334     case 1:
 335       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 336       break;
 337     default:
 338       vec_foreach (outside_fib, sm->outside_fibs)
 339         {
 340           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 341           if (FIB_NODE_INDEX_INVALID != fei)
 342             {
 343               if (fib_entry_get_resolving_interface (fei) != ~0)
 344                 {
 345                   s->out2in.fib_index = outside_fib->fib_index;
 346                   break;
 347                 }
 348             }
 349         }
 350       break;
 351     }
 352   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 353   s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
 354   *sessionp = s;
 355
 356   /* Add to translation hashes */
 357   ctx0.now = now;
 358   ctx0.thread_index = thread_index;
 359   init_nat_i2o_kv (&kv0, s, thread_index,
 360                    s - sm->per_thread_data[thread_index].sessions);
 361   if (clib_bihash_add_or_overwrite_stale_8_8 (
 362         &sm->in2out, &kv0, nat44_i2o_is_idle_session_cb, &ctx0))
 363     nat_elog_notice ("in2out key add failed");
 364
 365   init_nat_o2i_kv (&kv0, s, thread_index,
 366                    s - sm->per_thread_data[thread_index].sessions);
 367   if (clib_bihash_add_or_overwrite_stale_8_8 (
 368         &sm->out2in, &kv0, nat44_o2i_is_idle_session_cb, &ctx0))
 369     nat_elog_notice ("out2in key add failed");
 370
 371   /* log NAT event */
 372   nat_ipfix_logging_nat44_ses_create (thread_index,
 373                                       s->in2out.addr.as_u32,
 374                                       s->out2in.addr.as_u32,
 375                                       s->nat_proto,
 376                                       s->in2out.port,
 377                                       s->out2in.port, s->in2out.fib_index);
 378
 379   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 380                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 381                            s->out2in.port, s->nat_proto);
 382
 383   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 384                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 385                &s->ext_host_nat_addr, s->ext_host_nat_port,
 386                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 387
 388   return next0;
 389 }
 390
 391 #ifndef CLIB_MARCH_VARIANT
 392 static_always_inline snat_in2out_error_t
 393 icmp_get_key (vlib_buffer_t * b, ip4_header_t * ip0,
 394               ip4_address_t * addr, u16 * port, nat_protocol_t * nat_proto)
 395 {
 396   icmp46_header_t *icmp0;
 397   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 398   ip4_header_t *inner_ip0 = 0;
 399   void *l4_header = 0;
 400   icmp46_header_t *inner_icmp0;
 401
 402   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 403   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 404
 405   if (!icmp_type_is_error_message
 406       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 407     {
 408       *nat_proto = NAT_PROTOCOL_ICMP;
 409       *addr = ip0->src_address;
 410       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 411     }
 412   else
 413     {
 414       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 415       l4_header = ip4_next_header (inner_ip0);
 416       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 417       *addr = inner_ip0->dst_address;
 418       switch (*nat_proto)
 419         {
 420         case NAT_PROTOCOL_ICMP:
 421           inner_icmp0 = (icmp46_header_t *) l4_header;
 422           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 423           *port = inner_echo0->identifier;
 424           break;
 425         case NAT_PROTOCOL_UDP:
 426         case NAT_PROTOCOL_TCP:
 427           *port = ((tcp_udp_header_t *) l4_header)->dst_port;
 428           break;
 429         default:
 430           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 431         }
 432     }
 433   return -1;                    /* success */
 434 }
 435
 436 /**
 437  * Get address and port values to be used for ICMP packet translation
 438  * and create session if needed
 439  *
 440  * @param[in,out] sm             NAT main
 441  * @param[in,out] node           NAT node runtime
 442  * @param[in] thread_index       thread index
 443  * @param[in,out] b0             buffer containing packet to be translated
 444  * @param[in,out] ip0            ip header
 445  * @param[out] p_proto           protocol used for matching
 446  * @param[out] p_value           address and port after NAT translation
 447  * @param[out] p_dont_translate  if packet should not be translated
 448  * @param d                      optional parameter
 449  * @param e                      optional parameter
 450  */
 451 u32
 452 icmp_match_in2out_slow (snat_main_t *sm, vlib_node_runtime_t *node,
 453                         u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0,
 454                         ip4_address_t *addr, u16 *port, u32 *fib_index,
 455                         nat_protocol_t *proto, snat_session_t **p_s0,
 456                         u8 *dont_translate)
 457 {
 458   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 459   u32 sw_if_index0;
 460   snat_session_t *s0 = 0;
 461   clib_bihash_kv_8_8_t kv0, value0;
 462   u32 next0 = ~0;
 463   int err;
 464   vlib_main_t *vm = vlib_get_main ();
 465   *dont_translate = 0;
 466
 467   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 468   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 469
 470   err = icmp_get_key (b0, ip0, addr, port, proto);
 471   if (err != -1)
 472     {
 473       b0->error = node->errors[err];
 474       next0 = SNAT_IN2OUT_NEXT_DROP;
 475       goto out;
 476     }
 477
 478   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 479   if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
 480     {
 481       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 482         {
 483           if (PREDICT_FALSE
 484               (nat_not_translate_output_feature
 485                (sm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
 486             {
 487               *dont_translate = 1;
 488               goto out;
 489             }
 490         }
 491       else
 492         {
 493           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 494                                                  ip0, NAT_PROTOCOL_ICMP,
 495                                                  *fib_index, thread_index)))
 496             {
 497               *dont_translate = 1;
 498               goto out;
 499             }
 500         }
 501
 502       if (PREDICT_FALSE
 503           (icmp_type_is_error_message
 504            (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
 505         {
 506           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 507           next0 = SNAT_IN2OUT_NEXT_DROP;
 508           goto out;
 509         }
 510
 511       next0 =
 512         slow_path (sm, b0, ip0, *addr, *port, *fib_index, *proto, &s0, node,
 513                    next0, thread_index, vlib_time_now (vm));
 514
 515       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 516         goto out;
 517
 518       if (!s0)
 519         {
 520           *dont_translate = 1;
 521           goto out;
 522         }
 523     }
 524   else
 525     {
 526       if (PREDICT_FALSE
 527           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 528            ICMP4_echo_request
 529            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 530            ICMP4_echo_reply
 531            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 532                                            reass.icmp_type_or_tcp_flags)))
 533         {
 534           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 535           next0 = SNAT_IN2OUT_NEXT_DROP;
 536           goto out;
 537         }
 538
 539       s0 = pool_elt_at_index (tsm->sessions,
 540                               nat_value_get_session_index (&value0));
 541     }
 542
 543 out:
 544   if (s0)
 545     {
 546       *addr = s0->out2in.addr;
 547       *port = s0->out2in.port;
 548       *fib_index = s0->out2in.fib_index;
 549     }
 550   if (p_s0)
 551     *p_s0 = s0;
 552   return next0;
 553 }
 554 #endif
 555
 556 #ifndef CLIB_MARCH_VARIANT
 557 /**
 558  * Get address and port values to be used for ICMP packet translation
 559  *
 560  * @param[in] sm                 NAT main
 561  * @param[in,out] node           NAT node runtime
 562  * @param[in] thread_index       thread index
 563  * @param[in,out] b0             buffer containing packet to be translated
 564  * @param[in,out] ip0            ip header
 565  * @param[out] p_proto           protocol used for matching
 566  * @param[out] p_value           address and port after NAT translation
 567  * @param[out] p_dont_translate  if packet should not be translated
 568  * @param d                      optional parameter
 569  * @param e                      optional parameter
 570  */
 571 u32
 572 icmp_match_in2out_fast (snat_main_t *sm, vlib_node_runtime_t *node,
 573                         u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0,
 574                         ip4_address_t *addr, u16 *port, u32 *fib_index,
 575                         nat_protocol_t *proto, snat_session_t **s0,
 576                         u8 *dont_translate)
 577 {
 578   u32 sw_if_index0;
 579   u8 is_addr_only;
 580   u32 next0 = ~0;
 581   int err;
 582   *dont_translate = 0;
 583
 584   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 585   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 586
 587   err = icmp_get_key (b0, ip0, addr, port, proto);
 588   if (err != -1)
 589     {
 590       b0->error = node->errors[err];
 591       next0 = SNAT_IN2OUT_NEXT_DROP;
 592       goto out;
 593     }
 594
 595   ip4_address_t sm_addr;
 596   u16 sm_port;
 597   u32 sm_fib_index;
 598
 599   if (nat44_ei_static_mapping_match (*addr, *port, *fib_index, *proto,
 600                                      &sm_addr, &sm_port, &sm_fib_index, 0,
 601                                      &is_addr_only, 0))
 602     {
 603       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 604                                                   IP_PROTOCOL_ICMP,
 605                                                   *fib_index)))
 606         {
 607           *dont_translate = 1;
 608           goto out;
 609         }
 610
 611       if (icmp_type_is_error_message
 612           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 613         {
 614           next0 = SNAT_IN2OUT_NEXT_DROP;
 615           goto out;
 616         }
 617
 618       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 619       next0 = SNAT_IN2OUT_NEXT_DROP;
 620       goto out;
 621     }
 622
 623   if (PREDICT_FALSE
 624       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
 625        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 626            ICMP4_echo_reply || !is_addr_only)
 627        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 628                                        reass.icmp_type_or_tcp_flags)))
 629     {
 630       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 631       next0 = SNAT_IN2OUT_NEXT_DROP;
 632       goto out;
 633     }
 634
 635 out:
 636   return next0;
 637 }
 638 #endif
 639
 640 u32 icmp_in2out (snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0,
 641                  icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0,
 642                  vlib_node_runtime_t *node, u32 next0, u32 thread_index,
 643                  snat_session_t **p_s0);
 644
 645 #ifndef CLIB_MARCH_VARIANT
 646 u32
 647 icmp_in2out (snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0,
 648              icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0,
 649              vlib_node_runtime_t *node, u32 next0, u32 thread_index,
 650              snat_session_t **p_s0)
 651 {
 652   vlib_main_t *vm = vlib_get_main ();
 653   ip4_address_t addr;
 654   u16 port;
 655   u32 fib_index;
 656   nat_protocol_t protocol;
 657   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 658   ip4_header_t *inner_ip0;
 659   void *l4_header = 0;
 660   icmp46_header_t *inner_icmp0;
 661   u8 dont_translate;
 662   u32 new_addr0, old_addr0;
 663   u16 old_id0, new_id0;
 664   u16 old_checksum0, new_checksum0;
 665   ip_csum_t sum0;
 666   u16 checksum0;
 667   u32 next0_tmp;
 668   u32 required_thread_index = thread_index;
 669
 670   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 671
 672   next0_tmp =
 673     sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0, &addr, &port,
 674                               &fib_index, &protocol, p_s0, &dont_translate);
 675   if (next0_tmp != ~0)
 676     next0 = next0_tmp;
 677   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 678     goto out;
 679
 680   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 681     {
 682       sum0 =
 683         ip_incremental_checksum_buffer (vm, b0,
 684                                         (u8 *) icmp0 -
 685                                         (u8 *) vlib_buffer_get_current (b0),
 686                                         ntohs (ip0->length) -
 687                                         ip4_header_bytes (ip0), 0);
 688       checksum0 = ~ip_csum_fold (sum0);
 689       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 690         {
 691           next0 = SNAT_IN2OUT_NEXT_DROP;
 692           goto out;
 693         }
 694     }
 695
 696   old_addr0 = ip0->src_address.as_u32;
 697   new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
 698
 699   sum0 = ip0->checksum;
 700   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 701                          src_address /* changed member */ );
 702   ip0->checksum = ip_csum_fold (sum0);
 703
 704   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 705     {
 706       if (icmp0->checksum == 0)
 707         icmp0->checksum = 0xffff;
 708
 709       if (!icmp_type_is_error_message (icmp0->type))
 710         {
 711           new_id0 = port;
 712           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 713             {
 714               old_id0 = echo0->identifier;
 715               new_id0 = port;
 716               echo0->identifier = new_id0;
 717
 718               sum0 = icmp0->checksum;
 719               sum0 =
 720                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 721                                 identifier);
 722               icmp0->checksum = ip_csum_fold (sum0);
 723             }
 724         }
 725       else
 726         {
 727           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 728           l4_header = ip4_next_header (inner_ip0);
 729
 730           if (!ip4_header_checksum_is_valid (inner_ip0))
 731             {
 732               next0 = SNAT_IN2OUT_NEXT_DROP;
 733               goto out;
 734             }
 735
 736           /* update inner destination IP address */
 737           old_addr0 = inner_ip0->dst_address.as_u32;
 738           inner_ip0->dst_address = addr;
 739           new_addr0 = inner_ip0->dst_address.as_u32;
 740           sum0 = icmp0->checksum;
 741           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 742                                  dst_address /* changed member */ );
 743           icmp0->checksum = ip_csum_fold (sum0);
 744
 745           /* update inner IP header checksum */
 746           old_checksum0 = inner_ip0->checksum;
 747           sum0 = inner_ip0->checksum;
 748           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 749                                  dst_address /* changed member */ );
 750           inner_ip0->checksum = ip_csum_fold (sum0);
 751           new_checksum0 = inner_ip0->checksum;
 752           sum0 = icmp0->checksum;
 753           sum0 =
 754             ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 755                             checksum);
 756           icmp0->checksum = ip_csum_fold (sum0);
 757
 758           switch (protocol)
 759             {
 760             case NAT_PROTOCOL_ICMP:
 761               inner_icmp0 = (icmp46_header_t *) l4_header;
 762               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 763
 764               old_id0 = inner_echo0->identifier;
 765               new_id0 = port;
 766               inner_echo0->identifier = new_id0;
 767
 768               sum0 = icmp0->checksum;
 769               sum0 =
 770                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 771                                 identifier);
 772               icmp0->checksum = ip_csum_fold (sum0);
 773               break;
 774             case NAT_PROTOCOL_UDP:
 775             case NAT_PROTOCOL_TCP:
 776               old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 777               new_id0 = port;
 778               ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 779
 780               sum0 = icmp0->checksum;
 781               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 782                                      dst_port);
 783               icmp0->checksum = ip_csum_fold (sum0);
 784               break;
 785             default:
 786               ASSERT (0);
 787             }
 788         }
 789     }
 790
 791   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 792     {
 793       if (0 != snat_icmp_hairpinning (sm, b0, thread_index, ip0, icmp0,
 794                                       &required_thread_index))
 795         vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
 796       if (thread_index != required_thread_index)
 797         {
 798           vnet_buffer (b0)->snat.required_thread_index = required_thread_index;
 799           next0 = SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
 800         }
 801     }
 802
 803 out:
 804   return next0;
 805 }
 806 #endif
 807
 808 static inline u32
 809 icmp_in2out_slow_path (snat_main_t * sm,
 810                        vlib_buffer_t * b0,
 811                        ip4_header_t * ip0,
 812                        icmp46_header_t * icmp0,
 813                        u32 sw_if_index0,
 814                        u32 rx_fib_index0,
 815                        vlib_node_runtime_t * node,
 816                        u32 next0,
 817                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 818 {
 819   vlib_main_t *vm = vlib_get_main ();
 820
 821   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 822                        next0, thread_index, p_s0);
 823   snat_session_t *s0 = *p_s0;
 824   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 825     {
 826       /* Accounting */
 827       nat44_ei_session_update_counters (
 828         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
 829       /* Per-user LRU list maintenance */
 830       nat44_session_update_lru (sm, s0, thread_index);
 831     }
 832   return next0;
 833 }
 834
 835 static int
 836 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 837                              vlib_buffer_t * b,
 838                              ip4_header_t * ip, u32 rx_fib_index)
 839 {
 840   clib_bihash_kv_8_8_t kv, value;
 841   snat_static_mapping_t *m;
 842   u32 old_addr, new_addr;
 843   ip_csum_t sum;
 844
 845   init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 846   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 847     return 1;
 848
 849   m = pool_elt_at_index (sm->static_mappings, value.value);
 850
 851   old_addr = ip->src_address.as_u32;
 852   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 853   sum = ip->checksum;
 854   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 855   ip->checksum = ip_csum_fold (sum);
 856
 857
 858   /* Hairpinning */
 859   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 860     {
 861       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 862       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 863     }
 864
 865   return 0;
 866 }
 867
 868 static inline uword
 869 snat_in2out_node_fn_inline (vlib_main_t * vm,
 870                             vlib_node_runtime_t * node,
 871                             vlib_frame_t * frame, int is_slow_path,
 872                             int is_output_feature)
 873 {
 874   u32 n_left_from, *from;
 875   snat_main_t *sm = &snat_main;
 876   f64 now = vlib_time_now (vm);
 877   u32 thread_index = vm->thread_index;
 878
 879   from = vlib_frame_vector_args (frame);
 880   n_left_from = frame->n_vectors;
 881
 882   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 883   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 884   vlib_get_buffers (vm, from, b, n_left_from);
 885
 886   while (n_left_from >= 2)
 887     {
 888       vlib_buffer_t *b0, *b1;
 889       u32 next0, next1;
 890       u32 sw_if_index0, sw_if_index1;
 891       ip4_header_t *ip0, *ip1;
 892       ip_csum_t sum0, sum1;
 893       u32 new_addr0, old_addr0, new_addr1, old_addr1;
 894       u16 old_port0, new_port0, old_port1, new_port1;
 895       udp_header_t *udp0, *udp1;
 896       tcp_header_t *tcp0, *tcp1;
 897       icmp46_header_t *icmp0, *icmp1;
 898       u32 rx_fib_index0, rx_fib_index1;
 899       u32 proto0, proto1;
 900       snat_session_t *s0 = 0, *s1 = 0;
 901       clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 902       u32 iph_offset0 = 0, iph_offset1 = 0;
 903
 904       b0 = *b;
 905       b++;
 906       b1 = *b;
 907       b++;
 908
 909       /* Prefetch next iteration. */
 910       if (PREDICT_TRUE (n_left_from >= 4))
 911         {
 912           vlib_buffer_t *p2, *p3;
 913
 914           p2 = *b;
 915           p3 = *(b + 1);
 916
 917           vlib_prefetch_buffer_header (p2, LOAD);
 918           vlib_prefetch_buffer_header (p3, LOAD);
 919
 920           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 921           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
 922         }
 923
 924       if (is_output_feature)
 925         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 926
 927       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 928                               iph_offset0);
 929
 930       udp0 = ip4_next_header (ip0);
 931       tcp0 = (tcp_header_t *) udp0;
 932       icmp0 = (icmp46_header_t *) udp0;
 933
 934       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 935       rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 936                                sw_if_index0);
 937
 938       next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 939
 940       if (PREDICT_FALSE (ip0->ttl == 1))
 941         {
 942           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 943           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 944                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 945                                        0);
 946           next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 947           goto trace00;
 948         }
 949
 950       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 951
 952       /* Next configured feature, probably ip4-lookup */
 953       if (is_slow_path)
 954         {
 955           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 956             {
 957               if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
 958                 {
 959                   next0 = SNAT_IN2OUT_NEXT_DROP;
 960                   b0->error =
 961                     node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 962                 }
 963               vlib_increment_simple_counter (is_slow_path ? &sm->
 964                                              counters.slowpath.in2out.
 965                                              other : &sm->counters.fastpath.
 966                                              in2out.other, thread_index,
 967                                              sw_if_index0, 1);
 968               goto trace00;
 969             }
 970
 971           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 972             {
 973               next0 = icmp_in2out_slow_path
 974                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 975                  node, next0, now, thread_index, &s0);
 976               vlib_increment_simple_counter (is_slow_path ? &sm->
 977                                              counters.slowpath.in2out.
 978                                              icmp : &sm->counters.fastpath.
 979                                              in2out.icmp, thread_index,
 980                                              sw_if_index0, 1);
 981               goto trace00;
 982             }
 983         }
 984       else
 985         {
 986           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 987             {
 988               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 989               goto trace00;
 990             }
 991
 992           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 993             {
 994               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 995               goto trace00;
 996             }
 997         }
 998
 999       init_nat_k (&kv0, ip0->src_address,
1000                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1001                   proto0);
1002       if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) !=
1003                          0))
1004         {
1005           if (is_slow_path)
1006             {
1007               if (is_output_feature)
1008                 {
1009                   if (PREDICT_FALSE
1010                       (nat_not_translate_output_feature
1011                        (sm, ip0, proto0,
1012                         vnet_buffer (b0)->ip.reass.l4_src_port,
1013                         vnet_buffer (b0)->ip.reass.l4_dst_port,
1014                         thread_index, sw_if_index0)))
1015                     goto trace00;
1016
1017                   /*
1018                    * Send DHCP packets to the ipv4 stack, or we won't
1019                    * be able to use dhcp client on the outside interface
1020                    */
1021                   if (PREDICT_FALSE
1022                       (proto0 == NAT_PROTOCOL_UDP
1023                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1024                            clib_host_to_net_u16
1025                            (UDP_DST_PORT_dhcp_to_server))
1026                        && ip0->dst_address.as_u32 == 0xffffffff))
1027                     goto trace00;
1028                 }
1029               else
1030                 {
1031                   if (PREDICT_FALSE
1032                       (snat_not_translate
1033                        (sm, node, sw_if_index0, ip0, proto0,
1034                         rx_fib_index0, thread_index)))
1035                     goto trace00;
1036                 }
1037
1038               next0 = slow_path (sm, b0, ip0,
1039                                  ip0->src_address,
1040                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1041                                  rx_fib_index0,
1042                                  proto0, &s0, node, next0, thread_index, now);
1043               if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1044                 goto trace00;
1045
1046               if (PREDICT_FALSE (!s0))
1047                 goto trace00;
1048             }
1049           else
1050             {
1051               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1052               goto trace00;
1053             }
1054         }
1055       else
1056         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1057                                 nat_value_get_session_index (&value0));
1058
1059       b0->flags |= VNET_BUFFER_F_IS_NATED;
1060
1061       old_addr0 = ip0->src_address.as_u32;
1062       ip0->src_address = s0->out2in.addr;
1063       new_addr0 = ip0->src_address.as_u32;
1064       if (!is_output_feature)
1065         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1066
1067       sum0 = ip0->checksum;
1068       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1069                              ip4_header_t, src_address /* changed member */ );
1070       ip0->checksum = ip_csum_fold (sum0);
1071
1072
1073       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1074         {
1075           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1076             {
1077               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1078               new_port0 = udp0->src_port = s0->out2in.port;
1079               sum0 = tcp0->checksum;
1080               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1081                                      ip4_header_t,
1082                                      dst_address /* changed member */ );
1083               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1084                                      ip4_header_t /* cheat */ ,
1085                                      length /* changed member */ );
1086               mss_clamping (sm->mss_clamping, tcp0, &sum0);
1087               tcp0->checksum = ip_csum_fold (sum0);
1088             }
1089           vlib_increment_simple_counter (is_slow_path ? &sm->
1090                                          counters.slowpath.in2out.tcp : &sm->
1091                                          counters.fastpath.in2out.tcp,
1092                                          thread_index, sw_if_index0, 1);
1093         }
1094       else
1095         {
1096           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1097             {
1098               udp0->src_port = s0->out2in.port;
1099               if (PREDICT_FALSE (udp0->checksum))
1100                 {
1101                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1102                   new_port0 = udp0->src_port;
1103                   sum0 = udp0->checksum;
1104                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address  /* changed member */
1105                     );
1106                   sum0 =
1107                     ip_csum_update (sum0, old_port0, new_port0,
1108                                     ip4_header_t /* cheat */ ,
1109                                     length /* changed member */ );
1110                   udp0->checksum = ip_csum_fold (sum0);
1111                 }
1112             }
1113           vlib_increment_simple_counter (is_slow_path ? &sm->
1114                                          counters.slowpath.in2out.udp : &sm->
1115                                          counters.fastpath.in2out.udp,
1116                                          thread_index, sw_if_index0, 1);
1117         }
1118
1119       /* Accounting */
1120       nat44_ei_session_update_counters (
1121         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1122       /* Per-user LRU list maintenance */
1123       nat44_session_update_lru (sm, s0, thread_index);
1124     trace00:
1125
1126       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1127                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1128         {
1129           snat_in2out_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
1130           t->is_slow_path = is_slow_path;
1131           t->sw_if_index = sw_if_index0;
1132           t->next_index = next0;
1133           t->session_index = ~0;
1134           if (s0)
1135             t->session_index =
1136               s0 - sm->per_thread_data[thread_index].sessions;
1137         }
1138
1139       if (next0 == SNAT_IN2OUT_NEXT_DROP)
1140         {
1141           vlib_increment_simple_counter (is_slow_path ? &sm->
1142                                          counters.slowpath.in2out.
1143                                          drops : &sm->counters.fastpath.
1144                                          in2out.drops, thread_index,
1145                                          sw_if_index0, 1);
1146         }
1147
1148       if (is_output_feature)
1149         iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1150
1151       ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1152                               iph_offset1);
1153
1154       udp1 = ip4_next_header (ip1);
1155       tcp1 = (tcp_header_t *) udp1;
1156       icmp1 = (icmp46_header_t *) udp1;
1157
1158       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1159       rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1160                                sw_if_index1);
1161
1162       if (PREDICT_FALSE (ip1->ttl == 1))
1163         {
1164           vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1165           icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1166                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1167                                        0);
1168           next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1169           goto trace01;
1170         }
1171
1172       proto1 = ip_proto_to_nat_proto (ip1->protocol);
1173
1174       /* Next configured feature, probably ip4-lookup */
1175       if (is_slow_path)
1176         {
1177           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1178             {
1179               if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1180                 {
1181                   next1 = SNAT_IN2OUT_NEXT_DROP;
1182                   b1->error =
1183                     node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1184                 }
1185               vlib_increment_simple_counter (is_slow_path ? &sm->
1186                                              counters.slowpath.in2out.
1187                                              other : &sm->counters.fastpath.
1188                                              in2out.other, thread_index,
1189                                              sw_if_index1, 1);
1190               goto trace01;
1191             }
1192
1193           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1194             {
1195               next1 = icmp_in2out_slow_path
1196                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1197                  next1, now, thread_index, &s1);
1198               vlib_increment_simple_counter (is_slow_path ? &sm->
1199                                              counters.slowpath.in2out.
1200                                              icmp : &sm->counters.fastpath.
1201                                              in2out.icmp, thread_index,
1202                                              sw_if_index1, 1);
1203               goto trace01;
1204             }
1205         }
1206       else
1207         {
1208           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1209             {
1210               next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1211               goto trace01;
1212             }
1213
1214           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1215             {
1216               next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1217               goto trace01;
1218             }
1219         }
1220
1221       init_nat_k (&kv1, ip1->src_address,
1222                   vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1223                   proto1);
1224       if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) !=
1225                          0))
1226         {
1227           if (is_slow_path)
1228             {
1229               if (is_output_feature)
1230                 {
1231                   if (PREDICT_FALSE
1232                       (nat_not_translate_output_feature
1233                        (sm, ip1, proto1,
1234                         vnet_buffer (b1)->ip.reass.l4_src_port,
1235                         vnet_buffer (b1)->ip.reass.l4_dst_port,
1236                         thread_index, sw_if_index1)))
1237                     goto trace01;
1238
1239                   /*
1240                    * Send DHCP packets to the ipv4 stack, or we won't
1241                    * be able to use dhcp client on the outside interface
1242                    */
1243                   if (PREDICT_FALSE
1244                       (proto1 == NAT_PROTOCOL_UDP
1245                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1246                            clib_host_to_net_u16
1247                            (UDP_DST_PORT_dhcp_to_server))
1248                        && ip1->dst_address.as_u32 == 0xffffffff))
1249                     goto trace01;
1250                 }
1251               else
1252                 {
1253                   if (PREDICT_FALSE
1254                       (snat_not_translate
1255                        (sm, node, sw_if_index1, ip1, proto1,
1256                         rx_fib_index1, thread_index)))
1257                     goto trace01;
1258                 }
1259
1260               next1 =
1261                 slow_path (sm, b1, ip1, ip1->src_address,
1262                            vnet_buffer (b1)->ip.reass.l4_src_port,
1263                            rx_fib_index1, proto1, &s1, node, next1,
1264                            thread_index, now);
1265               if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1266                 goto trace01;
1267
1268               if (PREDICT_FALSE (!s1))
1269                 goto trace01;
1270             }
1271           else
1272             {
1273               next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1274               goto trace01;
1275             }
1276         }
1277       else
1278         s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1279                                 nat_value_get_session_index (&value1));
1280
1281       b1->flags |= VNET_BUFFER_F_IS_NATED;
1282
1283       old_addr1 = ip1->src_address.as_u32;
1284       ip1->src_address = s1->out2in.addr;
1285       new_addr1 = ip1->src_address.as_u32;
1286       if (!is_output_feature)
1287         vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1288
1289       sum1 = ip1->checksum;
1290       sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1291                              ip4_header_t, src_address /* changed member */ );
1292       ip1->checksum = ip_csum_fold (sum1);
1293
1294       if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1295         {
1296           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1297             {
1298               old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1299               new_port1 = udp1->src_port = s1->out2in.port;
1300               sum1 = tcp1->checksum;
1301               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1302                                      ip4_header_t,
1303                                      dst_address /* changed member */ );
1304               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1305                                      ip4_header_t /* cheat */ ,
1306                                      length /* changed member */ );
1307               mss_clamping (sm->mss_clamping, tcp1, &sum1);
1308               tcp1->checksum = ip_csum_fold (sum1);
1309             }
1310           vlib_increment_simple_counter (is_slow_path ? &sm->
1311                                          counters.slowpath.in2out.tcp : &sm->
1312                                          counters.fastpath.in2out.tcp,
1313                                          thread_index, sw_if_index1, 1);
1314         }
1315       else
1316         {
1317           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1318             {
1319               udp1->src_port = s1->out2in.port;
1320               if (PREDICT_FALSE (udp1->checksum))
1321                 {
1322                   old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1323                   new_port1 = udp1->src_port;
1324                   sum1 = udp1->checksum;
1325                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address  /* changed member */
1326                     );
1327                   sum1 =
1328                     ip_csum_update (sum1, old_port1, new_port1,
1329                                     ip4_header_t /* cheat */ ,
1330                                     length /* changed member */ );
1331                   udp1->checksum = ip_csum_fold (sum1);
1332                 }
1333             }
1334           vlib_increment_simple_counter (is_slow_path ? &sm->
1335                                          counters.slowpath.in2out.udp : &sm->
1336                                          counters.fastpath.in2out.udp,
1337                                          thread_index, sw_if_index1, 1);
1338         }
1339
1340       /* Accounting */
1341       nat44_ei_session_update_counters (
1342         s1, now, vlib_buffer_length_in_chain (vm, b1), thread_index);
1343       /* Per-user LRU list maintenance */
1344       nat44_session_update_lru (sm, s1, thread_index);
1345     trace01:
1346
1347       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1348                          && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1349         {
1350           snat_in2out_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t));
1351           t->sw_if_index = sw_if_index1;
1352           t->next_index = next1;
1353           t->session_index = ~0;
1354           if (s1)
1355             t->session_index =
1356               s1 - sm->per_thread_data[thread_index].sessions;
1357         }
1358
1359       if (next1 == SNAT_IN2OUT_NEXT_DROP)
1360         {
1361           vlib_increment_simple_counter (is_slow_path ? &sm->
1362                                          counters.slowpath.in2out.
1363                                          drops : &sm->counters.fastpath.
1364                                          in2out.drops, thread_index,
1365                                          sw_if_index1, 1);
1366         }
1367
1368       n_left_from -= 2;
1369       next[0] = next0;
1370       next[1] = next1;
1371       next += 2;
1372     }
1373
1374   while (n_left_from > 0)
1375     {
1376       vlib_buffer_t *b0;
1377       u32 next0;
1378       u32 sw_if_index0;
1379       ip4_header_t *ip0;
1380       ip_csum_t sum0;
1381       u32 new_addr0, old_addr0;
1382       u16 old_port0, new_port0;
1383       udp_header_t *udp0;
1384       tcp_header_t *tcp0;
1385       icmp46_header_t *icmp0;
1386       u32 rx_fib_index0;
1387       u32 proto0;
1388       snat_session_t *s0 = 0;
1389       clib_bihash_kv_8_8_t kv0, value0;
1390       u32 iph_offset0 = 0;
1391
1392       b0 = *b;
1393       b++;
1394       next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1395
1396       if (is_output_feature)
1397         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1398
1399       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1400                               iph_offset0);
1401
1402       udp0 = ip4_next_header (ip0);
1403       tcp0 = (tcp_header_t *) udp0;
1404       icmp0 = (icmp46_header_t *) udp0;
1405
1406       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1407       rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1408                                sw_if_index0);
1409
1410       if (PREDICT_FALSE (ip0->ttl == 1))
1411         {
1412           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1413           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1414                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1415                                        0);
1416           next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1417           goto trace0;
1418         }
1419
1420       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1421
1422       /* Next configured feature, probably ip4-lookup */
1423       if (is_slow_path)
1424         {
1425           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1426             {
1427               if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1428                 {
1429                   next0 = SNAT_IN2OUT_NEXT_DROP;
1430                   b0->error =
1431                     node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1432                 }
1433               vlib_increment_simple_counter (is_slow_path ? &sm->
1434                                              counters.slowpath.in2out.
1435                                              other : &sm->counters.fastpath.
1436                                              in2out.other, thread_index,
1437                                              sw_if_index0, 1);
1438               goto trace0;
1439             }
1440
1441           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1442             {
1443               next0 = icmp_in2out_slow_path
1444                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1445                  next0, now, thread_index, &s0);
1446               vlib_increment_simple_counter (is_slow_path ? &sm->
1447                                              counters.slowpath.in2out.
1448                                              icmp : &sm->counters.fastpath.
1449                                              in2out.icmp, thread_index,
1450                                              sw_if_index0, 1);
1451               goto trace0;
1452             }
1453         }
1454       else
1455         {
1456           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1457             {
1458               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1459               goto trace0;
1460             }
1461
1462           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1463             {
1464               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1465               goto trace0;
1466             }
1467         }
1468
1469       init_nat_k (&kv0, ip0->src_address,
1470                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1471                   proto0);
1472
1473       if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
1474         {
1475           if (is_slow_path)
1476             {
1477               if (is_output_feature)
1478                 {
1479                   if (PREDICT_FALSE
1480                       (nat_not_translate_output_feature
1481                        (sm, ip0, proto0,
1482                         vnet_buffer (b0)->ip.reass.l4_src_port,
1483                         vnet_buffer (b0)->ip.reass.l4_dst_port,
1484                         thread_index, sw_if_index0)))
1485                     goto trace0;
1486
1487                   /*
1488                    * Send DHCP packets to the ipv4 stack, or we won't
1489                    * be able to use dhcp client on the outside interface
1490                    */
1491                   if (PREDICT_FALSE
1492                       (proto0 == NAT_PROTOCOL_UDP
1493                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1494                            clib_host_to_net_u16
1495                            (UDP_DST_PORT_dhcp_to_server))
1496                        && ip0->dst_address.as_u32 == 0xffffffff))
1497                     goto trace0;
1498                 }
1499               else
1500                 {
1501                   if (PREDICT_FALSE
1502                       (snat_not_translate
1503                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1504                         thread_index)))
1505                     goto trace0;
1506                 }
1507
1508               next0 =
1509                 slow_path (sm, b0, ip0, ip0->src_address,
1510                            vnet_buffer (b0)->ip.reass.l4_src_port,
1511                            rx_fib_index0, proto0, &s0, node, next0,
1512                            thread_index, now);
1513
1514               if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1515                 goto trace0;
1516
1517               if (PREDICT_FALSE (!s0))
1518                 goto trace0;
1519             }
1520           else
1521             {
1522               next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1523               goto trace0;
1524             }
1525         }
1526       else
1527         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1528                                 nat_value_get_session_index (&value0));
1529
1530       b0->flags |= VNET_BUFFER_F_IS_NATED;
1531
1532       old_addr0 = ip0->src_address.as_u32;
1533       ip0->src_address = s0->out2in.addr;
1534       new_addr0 = ip0->src_address.as_u32;
1535       if (!is_output_feature)
1536         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1537
1538       sum0 = ip0->checksum;
1539       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1540                              ip4_header_t, src_address /* changed member */ );
1541       ip0->checksum = ip_csum_fold (sum0);
1542
1543       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1544         {
1545           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1546             {
1547               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1548               new_port0 = udp0->src_port = s0->out2in.port;
1549               sum0 = tcp0->checksum;
1550               sum0 =
1551                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1552                                 dst_address /* changed member */ );
1553               sum0 =
1554                 ip_csum_update (sum0, old_port0, new_port0,
1555                                 ip4_header_t /* cheat */ ,
1556                                 length /* changed member */ );
1557               mss_clamping (sm->mss_clamping, tcp0, &sum0);
1558               tcp0->checksum = ip_csum_fold (sum0);
1559             }
1560           vlib_increment_simple_counter (is_slow_path ? &sm->
1561                                          counters.slowpath.in2out.tcp : &sm->
1562                                          counters.fastpath.in2out.tcp,
1563                                          thread_index, sw_if_index0, 1);
1564         }
1565       else
1566         {
1567           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1568             {
1569               udp0->src_port = s0->out2in.port;
1570               if (PREDICT_FALSE (udp0->checksum))
1571                 {
1572                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1573                   new_port0 = udp0->src_port;
1574                   sum0 = udp0->checksum;
1575                   sum0 =
1576                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1577                                     dst_address /* changed member */ );
1578                   sum0 =
1579                     ip_csum_update (sum0, old_port0, new_port0,
1580                                     ip4_header_t /* cheat */ ,
1581                                     length /* changed member */ );
1582                   udp0->checksum = ip_csum_fold (sum0);
1583                 }
1584             }
1585           vlib_increment_simple_counter (is_slow_path ? &sm->
1586                                          counters.slowpath.in2out.udp : &sm->
1587                                          counters.fastpath.in2out.udp,
1588                                          thread_index, sw_if_index0, 1);
1589         }
1590
1591       /* Accounting */
1592       nat44_ei_session_update_counters (
1593         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1594       /* Per-user LRU list maintenance */
1595       nat44_session_update_lru (sm, s0, thread_index);
1596
1597     trace0:
1598       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1599                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1600         {
1601           snat_in2out_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
1602           t->is_slow_path = is_slow_path;
1603           t->sw_if_index = sw_if_index0;
1604           t->next_index = next0;
1605           t->session_index = ~0;
1606           if (s0)
1607             t->session_index =
1608               s0 - sm->per_thread_data[thread_index].sessions;
1609         }
1610
1611       if (next0 == SNAT_IN2OUT_NEXT_DROP)
1612         {
1613           vlib_increment_simple_counter (is_slow_path ? &sm->
1614                                          counters.slowpath.in2out.
1615                                          drops : &sm->counters.fastpath.
1616                                          in2out.drops, thread_index,
1617                                          sw_if_index0, 1);
1618         }
1619
1620       n_left_from--;
1621       next[0] = next0;
1622       next++;
1623     }
1624
1625   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1626                                frame->n_vectors);
1627   return frame->n_vectors;
1628 }
1629
1630 VLIB_NODE_FN (snat_in2out_node) (vlib_main_t * vm,
1631                                  vlib_node_runtime_t * node,
1632                                  vlib_frame_t * frame)
1633 {
1634   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1635                                      0);
1636 }
1637
1638 VLIB_REGISTER_NODE (snat_in2out_node) = {
1639   .name = "nat44-in2out",
1640   .vector_size = sizeof (u32),
1641   .format_trace = format_snat_in2out_trace,
1642   .type = VLIB_NODE_TYPE_INTERNAL,
1643
1644   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1645   .error_strings = snat_in2out_error_strings,
1646
1647   .runtime_data_bytes = sizeof (snat_runtime_t),
1648
1649   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1650
1651   /* edit / add dispositions here */
1652   .next_nodes = {
1653     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1654     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1655     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1656     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1657     [SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-in2out-hairpinning-handoff-ip4-lookup",
1658   },
1659 };
1660
1661 VLIB_NODE_FN (snat_in2out_output_node) (vlib_main_t * vm,
1662                                         vlib_node_runtime_t * node,
1663                                         vlib_frame_t * frame)
1664 {
1665   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1666                                      1);
1667 }
1668
1669 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1670   .name = "nat44-in2out-output",
1671   .vector_size = sizeof (u32),
1672   .format_trace = format_snat_in2out_trace,
1673   .type = VLIB_NODE_TYPE_INTERNAL,
1674
1675   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1676   .error_strings = snat_in2out_error_strings,
1677
1678   .runtime_data_bytes = sizeof (snat_runtime_t),
1679
1680   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1681
1682   /* edit / add dispositions here */
1683   .next_nodes = {
1684     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1685     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1686     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1687     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1688     [SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-in2out-hairpinning-handoff-interface-output",
1689   },
1690 };
1691
1692 VLIB_NODE_FN (snat_in2out_slowpath_node) (vlib_main_t * vm,
1693                                           vlib_node_runtime_t * node,
1694                                           vlib_frame_t * frame)
1695 {
1696   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1697                                      0);
1698 }
1699
1700 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1701   .name = "nat44-in2out-slowpath",
1702   .vector_size = sizeof (u32),
1703   .format_trace = format_snat_in2out_trace,
1704   .type = VLIB_NODE_TYPE_INTERNAL,
1705
1706   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1707   .error_strings = snat_in2out_error_strings,
1708
1709   .runtime_data_bytes = sizeof (snat_runtime_t),
1710
1711   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1712
1713   /* edit / add dispositions here */
1714   .next_nodes = {
1715     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1716     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1717     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1718     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1719     [SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-in2out-hairpinning-handoff-ip4-lookup",
1720   },
1721 };
1722
1723 VLIB_NODE_FN (snat_in2out_output_slowpath_node) (vlib_main_t * vm,
1724                                                  vlib_node_runtime_t * node,
1725                                                  vlib_frame_t * frame)
1726 {
1727   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1728                                      1);
1729 }
1730
1731 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1732   .name = "nat44-in2out-output-slowpath",
1733   .vector_size = sizeof (u32),
1734   .format_trace = format_snat_in2out_trace,
1735   .type = VLIB_NODE_TYPE_INTERNAL,
1736
1737   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1738   .error_strings = snat_in2out_error_strings,
1739
1740   .runtime_data_bytes = sizeof (snat_runtime_t),
1741
1742   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1743
1744   /* edit / add dispositions here */
1745   .next_nodes = {
1746     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1747     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1748     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1749     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1750     [SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-in2out-hairpinning-handoff-interface-output",
1751   },
1752 };
1753
1754 VLIB_NODE_FN (snat_in2out_fast_node) (vlib_main_t * vm,
1755                                       vlib_node_runtime_t * node,
1756                                       vlib_frame_t * frame)
1757 {
1758   u32 n_left_from, *from, *to_next;
1759   u32 thread_index = vm->thread_index;
1760   snat_in2out_next_t next_index;
1761   snat_main_t *sm = &snat_main;
1762   int is_hairpinning = 0;
1763
1764   from = vlib_frame_vector_args (frame);
1765   n_left_from = frame->n_vectors;
1766   next_index = node->cached_next_index;
1767
1768   while (n_left_from > 0)
1769     {
1770       u32 n_left_to_next;
1771
1772       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1773
1774       while (n_left_from > 0 && n_left_to_next > 0)
1775         {
1776           u32 bi0;
1777           vlib_buffer_t *b0;
1778           u32 next0;
1779           u32 sw_if_index0;
1780           ip4_header_t *ip0;
1781           ip_csum_t sum0;
1782           u32 new_addr0, old_addr0;
1783           u16 old_port0, new_port0;
1784           udp_header_t *udp0;
1785           tcp_header_t *tcp0;
1786           icmp46_header_t *icmp0;
1787           u32 proto0;
1788           u32 rx_fib_index0;
1789           ip4_address_t sm0_addr;
1790           u16 sm0_port;
1791           u32 sm0_fib_index;
1792           u32 required_thread_index = thread_index;
1793
1794           /* speculatively enqueue b0 to the current next frame */
1795           bi0 = from[0];
1796           to_next[0] = bi0;
1797           from += 1;
1798           to_next += 1;
1799           n_left_from -= 1;
1800           n_left_to_next -= 1;
1801
1802           b0 = vlib_get_buffer (vm, bi0);
1803           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1804
1805           ip0 = vlib_buffer_get_current (b0);
1806           udp0 = ip4_next_header (ip0);
1807           tcp0 = (tcp_header_t *) udp0;
1808           icmp0 = (icmp46_header_t *) udp0;
1809
1810           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1811           rx_fib_index0 =
1812             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1813
1814           if (PREDICT_FALSE (ip0->ttl == 1))
1815             {
1816               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1817               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1818                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1819                                            0);
1820               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1821               goto trace0;
1822             }
1823
1824           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1825
1826           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1827             goto trace0;
1828
1829           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1830             {
1831               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
1832                                    rx_fib_index0, node, next0, ~0, 0);
1833               goto trace0;
1834             }
1835
1836           if (nat44_ei_static_mapping_match (
1837                 ip0->src_address, udp0->src_port, rx_fib_index0, proto0,
1838                 &sm0_addr, &sm0_port, &sm0_fib_index, 0, 0, 0))
1839             {
1840               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
1841               next0 = SNAT_IN2OUT_NEXT_DROP;
1842               goto trace0;
1843             }
1844
1845           new_addr0 = sm0_addr.as_u32;
1846           new_port0 = sm0_port;
1847           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1848           old_addr0 = ip0->src_address.as_u32;
1849           ip0->src_address.as_u32 = new_addr0;
1850
1851           sum0 = ip0->checksum;
1852           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1853                                  ip4_header_t,
1854                                  src_address /* changed member */ );
1855           ip0->checksum = ip_csum_fold (sum0);
1856
1857           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1858             {
1859               old_port0 = udp0->src_port;
1860               udp0->src_port = new_port0;
1861
1862               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1863                 {
1864                   sum0 = tcp0->checksum;
1865                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1866                                          ip4_header_t,
1867                                          dst_address /* changed member */ );
1868                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1869                                          ip4_header_t /* cheat */ ,
1870                                          length /* changed member */ );
1871                   mss_clamping (sm->mss_clamping, tcp0, &sum0);
1872                   tcp0->checksum = ip_csum_fold (sum0);
1873                 }
1874               else if (udp0->checksum)
1875                 {
1876                   sum0 = udp0->checksum;
1877                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1878                                          ip4_header_t,
1879                                          dst_address /* changed member */ );
1880                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1881                                          ip4_header_t /* cheat */ ,
1882                                          length /* changed member */ );
1883                   udp0->checksum = ip_csum_fold (sum0);
1884                 }
1885             }
1886           else
1887             {
1888               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1889                 {
1890                   sum0 = tcp0->checksum;
1891                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1892                                          ip4_header_t,
1893                                          dst_address /* changed member */ );
1894                   mss_clamping (sm->mss_clamping, tcp0, &sum0);
1895                   tcp0->checksum = ip_csum_fold (sum0);
1896                 }
1897               else if (udp0->checksum)
1898                 {
1899                   sum0 = udp0->checksum;
1900                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1901                                          ip4_header_t,
1902                                          dst_address /* changed member */ );
1903                   udp0->checksum = ip_csum_fold (sum0);
1904                 }
1905             }
1906
1907           /* Hairpinning */
1908           is_hairpinning = snat_hairpinning (
1909             vm, node, sm, thread_index, b0, ip0, udp0, tcp0, proto0,
1910             0 /* do_trace */, &required_thread_index);
1911
1912           if (thread_index != required_thread_index)
1913             {
1914               vnet_buffer (b0)->snat.required_thread_index =
1915                 required_thread_index;
1916               next0 = SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
1917             }
1918
1919         trace0:
1920           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1921                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1922             {
1923               snat_in2out_trace_t *t =
1924                 vlib_add_trace (vm, node, b0, sizeof (*t));
1925               t->sw_if_index = sw_if_index0;
1926               t->next_index = next0;
1927               t->is_hairpinning = is_hairpinning;
1928             }
1929
1930           if (next0 != SNAT_IN2OUT_NEXT_DROP)
1931             {
1932
1933               vlib_increment_simple_counter (&sm->counters.fastpath.
1934                                              in2out.other, sw_if_index0,
1935                                              vm->thread_index, 1);
1936             }
1937
1938           /* verify speculative enqueue, maybe switch current next frame */
1939           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1940                                            to_next, n_left_to_next,
1941                                            bi0, next0);
1942         }
1943
1944       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1945     }
1946
1947   return frame->n_vectors;
1948 }
1949
1950 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
1951   .name = "nat44-in2out-fast",
1952   .vector_size = sizeof (u32),
1953   .format_trace = format_snat_in2out_fast_trace,
1954   .type = VLIB_NODE_TYPE_INTERNAL,
1955
1956   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1957   .error_strings = snat_in2out_error_strings,
1958
1959   .runtime_data_bytes = sizeof (snat_runtime_t),
1960
1961   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1962
1963   /* edit / add dispositions here */
1964   .next_nodes = {
1965     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1966     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1967     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1968     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1969     [SNAT_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-in2out-hairpinning-handoff-ip4-lookup",
1970   },
1971 };
1972
1973 VLIB_NODE_FN (nat44_in2out_hairpinning_handoff_ip4_lookup_node)
1974 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1975 {
1976   return nat44_hairpinning_handoff_fn_inline (
1977     vm, node, frame,
1978     snat_main.nat44_in2out_hairpinning_finish_ip4_lookup_node_fq_index);
1979 }
1980
1981 VLIB_REGISTER_NODE (nat44_in2out_hairpinning_handoff_ip4_lookup_node) = {
1982   .name = "nat44-in2out-hairpinning-handoff-ip4-lookup",
1983   .vector_size = sizeof (u32),
1984   .n_errors = ARRAY_LEN(nat44_hairpinning_handoff_error_strings),
1985   .error_strings = nat44_hairpinning_handoff_error_strings,
1986   .format_trace = format_nat44_hairpinning_handoff_trace,
1987
1988   .n_next_nodes = 1,
1989
1990   .next_nodes = {
1991     [0] = "error-drop",
1992   },
1993 };
1994
1995 VLIB_NODE_FN (nat44_in2out_hairpinning_handoff_interface_output_node)
1996 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1997 {
1998   return nat44_hairpinning_handoff_fn_inline (
1999     vm, node, frame,
2000     snat_main.nat44_in2out_hairpinning_finish_interface_output_node_fq_index);
2001 }
2002
2003 VLIB_REGISTER_NODE (nat44_in2out_hairpinning_handoff_interface_output_node) = {
2004   .name = "nat44-in2out-hairpinning-handoff-interface-output",
2005   .vector_size = sizeof (u32),
2006   .n_errors = ARRAY_LEN(nat44_hairpinning_handoff_error_strings),
2007   .error_strings = nat44_hairpinning_handoff_error_strings,
2008   .format_trace = format_nat44_hairpinning_handoff_trace,
2009
2010   .n_next_nodes = 1,
2011
2012   .next_nodes = {
2013     [0] = "error-drop",
2014   },
2015 };
2016
2017 static_always_inline int
2018 nat44_in2out_hairpinning_finish_inline (vlib_main_t *vm,
2019                                         vlib_node_runtime_t *node,
2020                                         vlib_frame_t *frame)
2021 {
2022   u32 n_left_from, *from, *to_next;
2023   u32 thread_index = vm->thread_index;
2024   snat_in2out_next_t next_index;
2025   snat_main_t *sm = &snat_main;
2026   int is_hairpinning = 0;
2027
2028   from = vlib_frame_vector_args (frame);
2029   n_left_from = frame->n_vectors;
2030   next_index = node->cached_next_index;
2031
2032   while (n_left_from > 0)
2033     {
2034       u32 n_left_to_next;
2035
2036       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2037
2038       while (n_left_from > 0 && n_left_to_next > 0)
2039         {
2040           u32 bi0;
2041           vlib_buffer_t *b0;
2042           u32 next0;
2043           u32 sw_if_index0;
2044           ip4_header_t *ip0;
2045           udp_header_t *udp0;
2046           tcp_header_t *tcp0;
2047           icmp46_header_t *icmp0;
2048           u32 proto0;
2049           u32 required_thread_index = thread_index;
2050
2051           /* speculatively enqueue b0 to the current next frame */
2052           bi0 = from[0];
2053           to_next[0] = bi0;
2054           from += 1;
2055           to_next += 1;
2056           n_left_from -= 1;
2057           n_left_to_next -= 1;
2058
2059           b0 = vlib_get_buffer (vm, bi0);
2060           next0 = NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP;
2061
2062           ip0 = vlib_buffer_get_current (b0);
2063           udp0 = ip4_next_header (ip0);
2064           tcp0 = (tcp_header_t *) udp0;
2065           icmp0 = (icmp46_header_t *) udp0;
2066
2067           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2068           proto0 = ip_proto_to_nat_proto (ip0->protocol);
2069
2070           switch (proto0)
2071             {
2072             case NAT_PROTOCOL_TCP:
2073               // fallthrough
2074             case NAT_PROTOCOL_UDP:
2075               is_hairpinning = snat_hairpinning (
2076                 vm, node, sm, thread_index, b0, ip0, udp0, tcp0, proto0,
2077                 0 /* do_trace */, &required_thread_index);
2078               break;
2079             case NAT_PROTOCOL_ICMP:
2080               is_hairpinning =
2081                 (0 == snat_icmp_hairpinning (sm, b0, thread_index, ip0, icmp0,
2082                                              &required_thread_index));
2083               break;
2084             case NAT_PROTOCOL_OTHER:
2085               // this should never happen
2086               next0 = NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2087               break;
2088             }
2089
2090           if (thread_index != required_thread_index)
2091             {
2092               // but we already did a handoff ...
2093               next0 = NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2094             }
2095
2096           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2097                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
2098             {
2099               snat_in2out_trace_t *t =
2100                 vlib_add_trace (vm, node, b0, sizeof (*t));
2101               t->sw_if_index = sw_if_index0;
2102               t->next_index = next0;
2103               t->is_hairpinning = is_hairpinning;
2104             }
2105
2106           if (next0 != NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP)
2107             {
2108               vlib_increment_simple_counter (
2109                 &sm->counters.fastpath.in2out.other, sw_if_index0,
2110                 vm->thread_index, 1);
2111             }
2112
2113           /* verify speculative enqueue, maybe switch current next frame */
2114           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2115                                            n_left_to_next, bi0, next0);
2116         }
2117
2118       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2119     }
2120
2121   return frame->n_vectors;
2122 }
2123
2124 VLIB_NODE_FN (nat44_in2out_hairpinning_finish_ip4_lookup_node)
2125 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2126 {
2127   return nat44_in2out_hairpinning_finish_inline (vm, node, frame);
2128 }
2129
2130 VLIB_REGISTER_NODE (nat44_in2out_hairpinning_finish_ip4_lookup_node) = {
2131   .name = "nat44-in2out-hairpinning-finish-ip4-lookup",
2132   .vector_size = sizeof (u32),
2133   .format_trace = format_snat_in2out_fast_trace,
2134   .type = VLIB_NODE_TYPE_INTERNAL,
2135
2136   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2137   .error_strings = snat_in2out_error_strings,
2138
2139   .runtime_data_bytes = sizeof (snat_runtime_t),
2140
2141   .n_next_nodes = NAT44_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2142
2143   /* edit / add dispositions here */
2144   .next_nodes = {
2145     [NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2146     [NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "ip4-lookup",
2147   },
2148 };
2149
2150 VLIB_NODE_FN (nat44_in2out_hairpinning_finish_interface_output_node)
2151 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2152 {
2153   return nat44_in2out_hairpinning_finish_inline (vm, node, frame);
2154 }
2155
2156 VLIB_REGISTER_NODE (nat44_in2out_hairpinning_finish_interface_output_node) = {
2157   .name = "nat44-in2out-hairpinning-finish-interface-output",
2158   .vector_size = sizeof (u32),
2159   .format_trace = format_snat_in2out_fast_trace,
2160   .type = VLIB_NODE_TYPE_INTERNAL,
2161
2162   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2163   .error_strings = snat_in2out_error_strings,
2164
2165   .runtime_data_bytes = sizeof (snat_runtime_t),
2166
2167   .n_next_nodes = NAT44_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2168
2169   /* edit / add dispositions here */
2170   .next_nodes = {
2171     [NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2172     [NAT44_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "interface-output",
2173   },
2174 };
2175
2176 /*
2177  * fd.io coding-style-patch-verification: ON
2178  *
2179  * Local Variables:
2180  * eval: (c-set-style "gnu")
2181  * End:
2182  */