src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <vnet/udp/udp.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_reass.h>
  31 #include <nat/nat_inlines.h>
  32 #include <nat/nat44_inlines.h>
  33 #include <nat/nat_syslog.h>
  34 #include <nat/nat_ha.h>
  35
  36 #include <vppinfra/hash.h>
  37 #include <vppinfra/error.h>
  38 #include <vppinfra/elog.h>
  39
  40 typedef struct
  41 {
  42   u32 sw_if_index;
  43   u32 next_index;
  44   u32 session_index;
  45   u32 is_slow_path;
  46 } snat_in2out_trace_t;
  47
  48 /* packet trace format function */
  49 static u8 *
  50 format_snat_in2out_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  55   char *tag;
  56
  57   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  58
  59   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  60               t->sw_if_index, t->next_index, t->session_index);
  61
  62   return s;
  63 }
  64
  65 static u8 *
  66 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  71
  72   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  73               t->sw_if_index, t->next_index);
  74
  75   return s;
  76 }
  77
  78 #define foreach_snat_in2out_error                       \
  79 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  80 _(IN2OUT_PACKETS, "good in2out packets processed")      \
  81 _(OUT_OF_PORTS, "out of ports")                         \
  82 _(BAD_OUTSIDE_FIB, "outside VRF ID not found")          \
  83 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  84 _(NO_TRANSLATION, "no translation")                     \
  85 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  86 _(DROP_FRAGMENT, "drop fragment")                       \
  87 _(MAX_REASS, "maximum reassemblies exceeded")           \
  88 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  89 _(TCP_PACKETS, "TCP packets")                           \
  90 _(UDP_PACKETS, "UDP packets")                           \
  91 _(ICMP_PACKETS, "ICMP packets")                         \
  92 _(OTHER_PACKETS, "other protocol packets")              \
  93 _(FRAGMENTS, "fragments")                               \
  94 _(CACHED_FRAGMENTS, "cached fragments")                 \
  95 _(PROCESSED_FRAGMENTS, "processed fragments")
  96
  97 typedef enum
  98 {
  99 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 100   foreach_snat_in2out_error
 101 #undef _
 102     SNAT_IN2OUT_N_ERROR,
 103 } snat_in2out_error_t;
 104
 105 static char *snat_in2out_error_strings[] = {
 106 #define _(sym,string) string,
 107   foreach_snat_in2out_error
 108 #undef _
 109 };
 110
 111 typedef enum
 112 {
 113   SNAT_IN2OUT_NEXT_LOOKUP,
 114   SNAT_IN2OUT_NEXT_DROP,
 115   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 116   SNAT_IN2OUT_NEXT_SLOW_PATH,
 117   SNAT_IN2OUT_NEXT_REASS,
 118   SNAT_IN2OUT_N_NEXT,
 119 } snat_in2out_next_t;
 120
 121 static inline int
 122 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 123                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 124                     u32 rx_fib_index0, u32 thread_index)
 125 {
 126   udp_header_t *udp0 = ip4_next_header (ip0);
 127   snat_session_key_t key0, sm0;
 128   clib_bihash_kv_8_8_t kv0, value0;
 129
 130   key0.addr = ip0->dst_address;
 131   key0.port = udp0->dst_port;
 132   key0.protocol = proto0;
 133   key0.fib_index = sm->outside_fib_index;
 134   kv0.key = key0.as_u64;
 135
 136   /* NAT packet aimed at external address if */
 137   /* has active sessions */
 138   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 139                               &value0))
 140     {
 141       /* or is static mappings */
 142       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 143         return 0;
 144     }
 145   else
 146     return 0;
 147
 148   if (sm->forwarding_enabled)
 149     return 1;
 150
 151   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 152                                   rx_fib_index0);
 153 }
 154
 155 static inline int
 156 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 157                                   u32 proto0, u16 src_port, u16 dst_port,
 158                                   u32 thread_index, u32 sw_if_index)
 159 {
 160   snat_session_key_t key0;
 161   clib_bihash_kv_8_8_t kv0, value0;
 162   snat_interface_t *i;
 163
 164   /* src NAT check */
 165   key0.addr = ip0->src_address;
 166   key0.port = src_port;
 167   key0.protocol = proto0;
 168   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 169   kv0.key = key0.as_u64;
 170
 171   if (!clib_bihash_search_8_8
 172       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 173     return 1;
 174
 175   /* dst NAT check */
 176   key0.addr = ip0->dst_address;
 177   key0.port = dst_port;
 178   key0.protocol = proto0;
 179   kv0.key = key0.as_u64;
 180   if (!clib_bihash_search_8_8
 181       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 182     {
 183       /* hairpinning */
 184     /* *INDENT-OFF* */
 185     pool_foreach (i, sm->output_feature_interfaces,
 186     ({
 187       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 188         return 0;
 189     }));
 190     /* *INDENT-ON* */
 191       return 1;
 192     }
 193
 194   return 0;
 195 }
 196
 197 #ifndef CLIB_MARCH_VARIANT
 198 int
 199 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 200 {
 201   snat_main_t *sm = &snat_main;
 202   nat44_is_idle_session_ctx_t *ctx = arg;
 203   snat_session_t *s;
 204   u64 sess_timeout_time;
 205   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 206                                                        ctx->thread_index);
 207   clib_bihash_kv_8_8_t s_kv;
 208
 209   s = pool_elt_at_index (tsm->sessions, kv->value);
 210   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 211   if (ctx->now >= sess_timeout_time)
 212     {
 213       s_kv.key = s->out2in.as_u64;
 214       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 215         nat_elog_warn ("out2in key del failed");
 216
 217       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 218                                            s->in2out.addr.as_u32,
 219                                            s->out2in.addr.as_u32,
 220                                            s->in2out.protocol,
 221                                            s->in2out.port,
 222                                            s->out2in.port,
 223                                            s->in2out.fib_index);
 224
 225       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 226                                &s->in2out.addr, s->in2out.port,
 227                                &s->out2in.addr, s->out2in.port,
 228                                s->in2out.protocol);
 229
 230       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 231                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 232                    ctx->thread_index);
 233
 234       if (!snat_is_session_static (s))
 235         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 236                                             &s->out2in);
 237
 238       nat44_delete_session (sm, s, ctx->thread_index);
 239       return 1;
 240     }
 241
 242   return 0;
 243 }
 244 #endif
 245
 246 static u32
 247 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 248            ip4_header_t * ip0,
 249            u32 rx_fib_index0,
 250            snat_session_key_t * key0,
 251            snat_session_t ** sessionp,
 252            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 253 {
 254   snat_user_t *u;
 255   snat_session_t *s = 0;
 256   clib_bihash_kv_8_8_t kv0;
 257   snat_session_key_t key1;
 258   udp_header_t *udp0 = ip4_next_header (ip0);
 259   u8 is_sm = 0;
 260   nat_outside_fib_t *outside_fib;
 261   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 262   u8 identity_nat;
 263   fib_prefix_t pfx = {
 264     .fp_proto = FIB_PROTOCOL_IP4,
 265     .fp_len = 32,
 266     .fp_addr = {
 267                 .ip4.as_u32 = ip0->dst_address.as_u32,
 268                 },
 269   };
 270   nat44_is_idle_session_ctx_t ctx0;
 271
 272   nat44_session_try_cleanup (&ip0->src_address, rx_fib_index0, thread_index,
 273                              now);
 274
 275   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 276     {
 277       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 278       nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 279       nat_elog_notice ("maximum sessions exceeded");
 280       return SNAT_IN2OUT_NEXT_DROP;
 281     }
 282
 283   key1.protocol = key0->protocol;
 284
 285   /* First try to match static mapping by local address and port */
 286   if (snat_static_mapping_match
 287       (sm, *key0, &key1, 0, 0, 0, 0, 0, &identity_nat))
 288     {
 289       /* Try to create dynamic translation */
 290       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 291                                                thread_index, &key1,
 292                                                sm->port_per_thread,
 293                                                sm->per_thread_data
 294                                                [thread_index].snat_thread_index))
 295         {
 296           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 297           return SNAT_IN2OUT_NEXT_DROP;
 298         }
 299     }
 300   else
 301     {
 302       if (PREDICT_FALSE (identity_nat))
 303         {
 304           *sessionp = s;
 305           return next0;
 306         }
 307
 308       is_sm = 1;
 309     }
 310
 311   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 312                               thread_index);
 313   if (!u)
 314     {
 315       nat_elog_warn ("create NAT user failed");
 316       return SNAT_IN2OUT_NEXT_DROP;
 317     }
 318
 319   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 320   if (!s)
 321     {
 322       nat44_delete_user_with_no_session (sm, u, thread_index);
 323       nat_elog_warn ("create NAT session failed");
 324       return SNAT_IN2OUT_NEXT_DROP;
 325     }
 326
 327   if (is_sm)
 328     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 329   user_session_increment (sm, u, is_sm);
 330   s->in2out = *key0;
 331   s->out2in = key1;
 332   s->out2in.protocol = key0->protocol;
 333   s->out2in.fib_index = sm->outside_fib_index;
 334   switch (vec_len (sm->outside_fibs))
 335     {
 336     case 0:
 337       s->out2in.fib_index = sm->outside_fib_index;
 338       break;
 339     case 1:
 340       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 341       break;
 342     default:
 343       /* *INDENT-OFF* */
 344       vec_foreach (outside_fib, sm->outside_fibs)
 345         {
 346           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 347           if (FIB_NODE_INDEX_INVALID != fei)
 348             {
 349               if (fib_entry_get_resolving_interface (fei) != ~0)
 350                 {
 351                   s->out2in.fib_index = outside_fib->fib_index;
 352                   break;
 353                 }
 354             }
 355         }
 356       /* *INDENT-ON* */
 357       break;
 358     }
 359   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 360   s->ext_host_port = udp0->dst_port;
 361   *sessionp = s;
 362
 363   /* Add to translation hashes */
 364   ctx0.now = now;
 365   ctx0.thread_index = thread_index;
 366   kv0.key = s->in2out.as_u64;
 367   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 368   if (clib_bihash_add_or_overwrite_stale_8_8
 369       (&sm->per_thread_data[thread_index].in2out, &kv0,
 370        nat44_i2o_is_idle_session_cb, &ctx0))
 371     nat_elog_notice ("in2out key add failed");
 372
 373   kv0.key = s->out2in.as_u64;
 374   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 375
 376   if (clib_bihash_add_or_overwrite_stale_8_8
 377       (&sm->per_thread_data[thread_index].out2in, &kv0,
 378        nat44_o2i_is_idle_session_cb, &ctx0))
 379     nat_elog_notice ("out2in key add failed");
 380
 381   /* log NAT event */
 382   snat_ipfix_logging_nat44_ses_create (thread_index,
 383                                        s->in2out.addr.as_u32,
 384                                        s->out2in.addr.as_u32,
 385                                        s->in2out.protocol,
 386                                        s->in2out.port,
 387                                        s->out2in.port, s->in2out.fib_index);
 388
 389   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 390                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 391                            s->out2in.port, s->in2out.protocol);
 392
 393   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 394                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 395                &s->ext_host_nat_addr, s->ext_host_nat_port,
 396                s->in2out.protocol, s->in2out.fib_index, s->flags,
 397                thread_index, 0);
 398
 399   return next0;
 400 }
 401
 402 #ifndef CLIB_MARCH_VARIANT
 403 static_always_inline
 404   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 405                                     snat_session_key_t * p_key0)
 406 {
 407   icmp46_header_t *icmp0;
 408   snat_session_key_t key0;
 409   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 410   ip4_header_t *inner_ip0 = 0;
 411   void *l4_header = 0;
 412   icmp46_header_t *inner_icmp0;
 413
 414   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 415   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 416
 417   if (!icmp_is_error_message (icmp0))
 418     {
 419       key0.protocol = SNAT_PROTOCOL_ICMP;
 420       key0.addr = ip0->src_address;
 421       key0.port = echo0->identifier;
 422     }
 423   else
 424     {
 425       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 426       l4_header = ip4_next_header (inner_ip0);
 427       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 428       key0.addr = inner_ip0->dst_address;
 429       switch (key0.protocol)
 430         {
 431         case SNAT_PROTOCOL_ICMP:
 432           inner_icmp0 = (icmp46_header_t *) l4_header;
 433           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 434           key0.port = inner_echo0->identifier;
 435           break;
 436         case SNAT_PROTOCOL_UDP:
 437         case SNAT_PROTOCOL_TCP:
 438           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 439           break;
 440         default:
 441           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 442         }
 443     }
 444   *p_key0 = key0;
 445   return -1;                    /* success */
 446 }
 447
 448 /**
 449  * Get address and port values to be used for ICMP packet translation
 450  * and create session if needed
 451  *
 452  * @param[in,out] sm             NAT main
 453  * @param[in,out] node           NAT node runtime
 454  * @param[in] thread_index       thread index
 455  * @param[in,out] b0             buffer containing packet to be translated
 456  * @param[out] p_proto           protocol used for matching
 457  * @param[out] p_value           address and port after NAT translation
 458  * @param[out] p_dont_translate  if packet should not be translated
 459  * @param d                      optional parameter
 460  * @param e                      optional parameter
 461  */
 462 u32
 463 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 464                         u32 thread_index, vlib_buffer_t * b0,
 465                         ip4_header_t * ip0, u8 * p_proto,
 466                         snat_session_key_t * p_value,
 467                         u8 * p_dont_translate, void *d, void *e)
 468 {
 469   icmp46_header_t *icmp0;
 470   u32 sw_if_index0;
 471   u32 rx_fib_index0;
 472   snat_session_key_t key0;
 473   snat_session_t *s0 = 0;
 474   u8 dont_translate = 0;
 475   clib_bihash_kv_8_8_t kv0, value0;
 476   u32 next0 = ~0;
 477   int err;
 478
 479   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 480   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 481   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 482
 483   err = icmp_get_key (ip0, &key0);
 484   if (err != -1)
 485     {
 486       b0->error = node->errors[err];
 487       next0 = SNAT_IN2OUT_NEXT_DROP;
 488       goto out;
 489     }
 490   key0.fib_index = rx_fib_index0;
 491
 492   kv0.key = key0.as_u64;
 493
 494   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 495                               &value0))
 496     {
 497       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 498         {
 499           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 500                                                                key0.protocol,
 501                                                                key0.port,
 502                                                                key0.port,
 503                                                                thread_index,
 504                                                                sw_if_index0)))
 505             {
 506               dont_translate = 1;
 507               goto out;
 508             }
 509         }
 510       else
 511         {
 512           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 513                                                  ip0, SNAT_PROTOCOL_ICMP,
 514                                                  rx_fib_index0,
 515                                                  thread_index)))
 516             {
 517               dont_translate = 1;
 518               goto out;
 519             }
 520         }
 521
 522       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 523         {
 524           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 525           next0 = SNAT_IN2OUT_NEXT_DROP;
 526           goto out;
 527         }
 528
 529       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 530                          thread_index, vlib_time_now (sm->vlib_main));
 531
 532       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 533         goto out;
 534
 535       if (!s0)
 536         {
 537           dont_translate = 1;
 538           goto out;
 539         }
 540     }
 541   else
 542     {
 543       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 544                          icmp0->type != ICMP4_echo_reply &&
 545                          !icmp_is_error_message (icmp0)))
 546         {
 547           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 548           next0 = SNAT_IN2OUT_NEXT_DROP;
 549           goto out;
 550         }
 551
 552       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 553                               value0.value);
 554     }
 555
 556 out:
 557   *p_proto = key0.protocol;
 558   if (s0)
 559     *p_value = s0->out2in;
 560   *p_dont_translate = dont_translate;
 561   if (d)
 562     *(snat_session_t **) d = s0;
 563   return next0;
 564 }
 565 #endif
 566
 567 #ifndef CLIB_MARCH_VARIANT
 568 /**
 569  * Get address and port values to be used for ICMP packet translation
 570  *
 571  * @param[in] sm                 NAT main
 572  * @param[in,out] node           NAT node runtime
 573  * @param[in] thread_index       thread index
 574  * @param[in,out] b0             buffer containing packet to be translated
 575  * @param[out] p_proto           protocol used for matching
 576  * @param[out] p_value           address and port after NAT translation
 577  * @param[out] p_dont_translate  if packet should not be translated
 578  * @param d                      optional parameter
 579  * @param e                      optional parameter
 580  */
 581 u32
 582 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 583                         u32 thread_index, vlib_buffer_t * b0,
 584                         ip4_header_t * ip0, u8 * p_proto,
 585                         snat_session_key_t * p_value,
 586                         u8 * p_dont_translate, void *d, void *e)
 587 {
 588   icmp46_header_t *icmp0;
 589   u32 sw_if_index0;
 590   u32 rx_fib_index0;
 591   snat_session_key_t key0;
 592   snat_session_key_t sm0;
 593   u8 dont_translate = 0;
 594   u8 is_addr_only;
 595   u32 next0 = ~0;
 596   int err;
 597
 598   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 599   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 600   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 601
 602   err = icmp_get_key (ip0, &key0);
 603   if (err != -1)
 604     {
 605       b0->error = node->errors[err];
 606       next0 = SNAT_IN2OUT_NEXT_DROP;
 607       goto out2;
 608     }
 609   key0.fib_index = rx_fib_index0;
 610
 611   if (snat_static_mapping_match
 612       (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0, 0))
 613     {
 614       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 615                                                   IP_PROTOCOL_ICMP,
 616                                                   rx_fib_index0)))
 617         {
 618           dont_translate = 1;
 619           goto out;
 620         }
 621
 622       if (icmp_is_error_message (icmp0))
 623         {
 624           next0 = SNAT_IN2OUT_NEXT_DROP;
 625           goto out;
 626         }
 627
 628       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 629       next0 = SNAT_IN2OUT_NEXT_DROP;
 630       goto out;
 631     }
 632
 633   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 634                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 635                      !icmp_is_error_message (icmp0)))
 636     {
 637       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 638       next0 = SNAT_IN2OUT_NEXT_DROP;
 639       goto out;
 640     }
 641
 642 out:
 643   *p_value = sm0;
 644 out2:
 645   *p_proto = key0.protocol;
 646   *p_dont_translate = dont_translate;
 647   return next0;
 648 }
 649 #endif
 650
 651 #ifndef CLIB_MARCH_VARIANT
 652 u32
 653 icmp_in2out (snat_main_t * sm,
 654              vlib_buffer_t * b0,
 655              ip4_header_t * ip0,
 656              icmp46_header_t * icmp0,
 657              u32 sw_if_index0,
 658              u32 rx_fib_index0,
 659              vlib_node_runtime_t * node,
 660              u32 next0, u32 thread_index, void *d, void *e)
 661 {
 662   snat_session_key_t sm0;
 663   u8 protocol;
 664   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 665   ip4_header_t *inner_ip0;
 666   void *l4_header = 0;
 667   icmp46_header_t *inner_icmp0;
 668   u8 dont_translate;
 669   u32 new_addr0, old_addr0;
 670   u16 old_id0, new_id0;
 671   u16 old_checksum0, new_checksum0;
 672   ip_csum_t sum0;
 673   u16 checksum0;
 674   u32 next0_tmp;
 675
 676   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 677
 678   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 679                                         &protocol, &sm0, &dont_translate, d,
 680                                         e);
 681   if (next0_tmp != ~0)
 682     next0 = next0_tmp;
 683   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 684     goto out;
 685
 686   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 687     {
 688       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 689                                              (u8 *)
 690                                              vlib_buffer_get_current (b0),
 691                                              ntohs (ip0->length) -
 692                                              ip4_header_bytes (ip0), 0);
 693       checksum0 = ~ip_csum_fold (sum0);
 694       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 695         {
 696           next0 = SNAT_IN2OUT_NEXT_DROP;
 697           goto out;
 698         }
 699     }
 700
 701   old_addr0 = ip0->src_address.as_u32;
 702   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 703
 704   sum0 = ip0->checksum;
 705   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 706                          src_address /* changed member */ );
 707   ip0->checksum = ip_csum_fold (sum0);
 708
 709   if (icmp0->checksum == 0)
 710     icmp0->checksum = 0xffff;
 711
 712   if (!icmp_is_error_message (icmp0))
 713     {
 714       new_id0 = sm0.port;
 715       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 716         {
 717           old_id0 = echo0->identifier;
 718           new_id0 = sm0.port;
 719           echo0->identifier = new_id0;
 720
 721           sum0 = icmp0->checksum;
 722           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 723                                  identifier);
 724           icmp0->checksum = ip_csum_fold (sum0);
 725         }
 726     }
 727   else
 728     {
 729       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 730       l4_header = ip4_next_header (inner_ip0);
 731
 732       if (!ip4_header_checksum_is_valid (inner_ip0))
 733         {
 734           next0 = SNAT_IN2OUT_NEXT_DROP;
 735           goto out;
 736         }
 737
 738       /* update inner destination IP address */
 739       old_addr0 = inner_ip0->dst_address.as_u32;
 740       inner_ip0->dst_address = sm0.addr;
 741       new_addr0 = inner_ip0->dst_address.as_u32;
 742       sum0 = icmp0->checksum;
 743       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 744                              dst_address /* changed member */ );
 745       icmp0->checksum = ip_csum_fold (sum0);
 746
 747       /* update inner IP header checksum */
 748       old_checksum0 = inner_ip0->checksum;
 749       sum0 = inner_ip0->checksum;
 750       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 751                              dst_address /* changed member */ );
 752       inner_ip0->checksum = ip_csum_fold (sum0);
 753       new_checksum0 = inner_ip0->checksum;
 754       sum0 = icmp0->checksum;
 755       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 756                              checksum);
 757       icmp0->checksum = ip_csum_fold (sum0);
 758
 759       switch (protocol)
 760         {
 761         case SNAT_PROTOCOL_ICMP:
 762           inner_icmp0 = (icmp46_header_t *) l4_header;
 763           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 764
 765           old_id0 = inner_echo0->identifier;
 766           new_id0 = sm0.port;
 767           inner_echo0->identifier = new_id0;
 768
 769           sum0 = icmp0->checksum;
 770           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 771                                  identifier);
 772           icmp0->checksum = ip_csum_fold (sum0);
 773           break;
 774         case SNAT_PROTOCOL_UDP:
 775         case SNAT_PROTOCOL_TCP:
 776           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 777           new_id0 = sm0.port;
 778           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 779
 780           sum0 = icmp0->checksum;
 781           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 782                                  dst_port);
 783           icmp0->checksum = ip_csum_fold (sum0);
 784           break;
 785         default:
 786           ASSERT (0);
 787         }
 788     }
 789
 790   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 791     {
 792       if (sm->deterministic ||
 793           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 794                                       sm->endpoint_dependent))
 795         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 796     }
 797
 798 out:
 799   return next0;
 800 }
 801 #endif
 802
 803 static inline u32
 804 icmp_in2out_slow_path (snat_main_t * sm,
 805                        vlib_buffer_t * b0,
 806                        ip4_header_t * ip0,
 807                        icmp46_header_t * icmp0,
 808                        u32 sw_if_index0,
 809                        u32 rx_fib_index0,
 810                        vlib_node_runtime_t * node,
 811                        u32 next0,
 812                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 813 {
 814   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 815                        next0, thread_index, p_s0, 0);
 816   snat_session_t *s0 = *p_s0;
 817   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 818     {
 819       /* Accounting */
 820       nat44_session_update_counters (s0, now,
 821                                      vlib_buffer_length_in_chain
 822                                      (sm->vlib_main, b0), thread_index);
 823       /* Per-user LRU list maintenance */
 824       nat44_session_update_lru (sm, s0, thread_index);
 825     }
 826   return next0;
 827 }
 828
 829 static int
 830 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 831                              vlib_buffer_t * b,
 832                              ip4_header_t * ip, u32 rx_fib_index)
 833 {
 834   clib_bihash_kv_8_8_t kv, value;
 835   snat_static_mapping_t *m;
 836   snat_session_key_t m_key;
 837   u32 old_addr, new_addr;
 838   ip_csum_t sum;
 839
 840   m_key.addr = ip->src_address;
 841   m_key.port = 0;
 842   m_key.protocol = 0;
 843   m_key.fib_index = rx_fib_index;
 844   kv.key = m_key.as_u64;
 845   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 846     return 1;
 847
 848   m = pool_elt_at_index (sm->static_mappings, value.value);
 849
 850   old_addr = ip->src_address.as_u32;
 851   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 852   sum = ip->checksum;
 853   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 854   ip->checksum = ip_csum_fold (sum);
 855
 856
 857   /* Hairpinning */
 858   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 859     {
 860       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 861       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 862     }
 863
 864   return 0;
 865 }
 866
 867 static inline uword
 868 snat_in2out_node_fn_inline (vlib_main_t * vm,
 869                             vlib_node_runtime_t * node,
 870                             vlib_frame_t * frame, int is_slow_path,
 871                             int is_output_feature)
 872 {
 873   u32 n_left_from, *from, *to_next;
 874   snat_in2out_next_t next_index;
 875   u32 pkts_processed = 0;
 876   snat_main_t *sm = &snat_main;
 877   f64 now = vlib_time_now (vm);
 878   u32 stats_node_index;
 879   u32 thread_index = vm->thread_index;
 880   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 881     0, fragments = 0;
 882
 883   stats_node_index = is_slow_path ? sm->in2out_slowpath_node_index :
 884     sm->in2out_node_index;
 885
 886   from = vlib_frame_vector_args (frame);
 887   n_left_from = frame->n_vectors;
 888   next_index = node->cached_next_index;
 889
 890   while (n_left_from > 0)
 891     {
 892       u32 n_left_to_next;
 893
 894       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 895
 896       while (n_left_from >= 4 && n_left_to_next >= 2)
 897         {
 898           u32 bi0, bi1;
 899           vlib_buffer_t *b0, *b1;
 900           u32 next0, next1;
 901           u32 sw_if_index0, sw_if_index1;
 902           ip4_header_t *ip0, *ip1;
 903           ip_csum_t sum0, sum1;
 904           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 905           u16 old_port0, new_port0, old_port1, new_port1;
 906           udp_header_t *udp0, *udp1;
 907           tcp_header_t *tcp0, *tcp1;
 908           icmp46_header_t *icmp0, *icmp1;
 909           snat_session_key_t key0, key1;
 910           u32 rx_fib_index0, rx_fib_index1;
 911           u32 proto0, proto1;
 912           snat_session_t *s0 = 0, *s1 = 0;
 913           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 914           u32 iph_offset0 = 0, iph_offset1 = 0;
 915
 916           /* Prefetch next iteration. */
 917           {
 918             vlib_buffer_t *p2, *p3;
 919
 920             p2 = vlib_get_buffer (vm, from[2]);
 921             p3 = vlib_get_buffer (vm, from[3]);
 922
 923             vlib_prefetch_buffer_header (p2, LOAD);
 924             vlib_prefetch_buffer_header (p3, LOAD);
 925
 926             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 927             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 928           }
 929
 930           /* speculatively enqueue b0 and b1 to the current next frame */
 931           to_next[0] = bi0 = from[0];
 932           to_next[1] = bi1 = from[1];
 933           from += 2;
 934           to_next += 2;
 935           n_left_from -= 2;
 936           n_left_to_next -= 2;
 937
 938           b0 = vlib_get_buffer (vm, bi0);
 939           b1 = vlib_get_buffer (vm, bi1);
 940
 941           if (is_output_feature)
 942             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 943
 944           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 945                                   iph_offset0);
 946
 947           udp0 = ip4_next_header (ip0);
 948           tcp0 = (tcp_header_t *) udp0;
 949           icmp0 = (icmp46_header_t *) udp0;
 950
 951           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 952           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 953                                    sw_if_index0);
 954
 955           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 956
 957           if (PREDICT_FALSE (ip0->ttl == 1))
 958             {
 959               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 960               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 961                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 962                                            0);
 963               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 964               goto trace00;
 965             }
 966
 967           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 968
 969           /* Next configured feature, probably ip4-lookup */
 970           if (is_slow_path)
 971             {
 972               if (PREDICT_FALSE (proto0 == ~0))
 973                 {
 974                   if (nat_in2out_sm_unknown_proto
 975                       (sm, b0, ip0, rx_fib_index0))
 976                     {
 977                       next0 = SNAT_IN2OUT_NEXT_DROP;
 978                       b0->error =
 979                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 980                     }
 981                   other_packets++;
 982                   goto trace00;
 983                 }
 984
 985               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 986                 {
 987                   next0 = icmp_in2out_slow_path
 988                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 989                      node, next0, now, thread_index, &s0);
 990                   icmp_packets++;
 991                   goto trace00;
 992                 }
 993             }
 994           else
 995             {
 996               if (PREDICT_FALSE (proto0 == ~0))
 997                 {
 998                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 999                   goto trace00;
1000                 }
1001
1002               if (ip4_is_fragment (ip0))
1003                 {
1004                   next0 = SNAT_IN2OUT_NEXT_REASS;
1005                   fragments++;
1006                   goto trace00;
1007                 }
1008
1009               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1010                 {
1011                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1012                   goto trace00;
1013                 }
1014             }
1015
1016           key0.addr = ip0->src_address;
1017           key0.port = udp0->src_port;
1018           key0.protocol = proto0;
1019           key0.fib_index = rx_fib_index0;
1020
1021           kv0.key = key0.as_u64;
1022
1023           if (PREDICT_FALSE
1024               (clib_bihash_search_8_8
1025                (&sm->per_thread_data[thread_index].in2out, &kv0,
1026                 &value0) != 0))
1027             {
1028               if (is_slow_path)
1029                 {
1030                   if (is_output_feature)
1031                     {
1032                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1033                                                                            ip0,
1034                                                                            proto0,
1035                                                                            udp0->src_port,
1036                                                                            udp0->dst_port,
1037                                                                            thread_index,
1038                                                                            sw_if_index0)))
1039                         goto trace00;
1040
1041                       /*
1042                        * Send DHCP packets to the ipv4 stack, or we won't
1043                        * be able to use dhcp client on the outside interface
1044                        */
1045                       if (PREDICT_FALSE
1046                           ((b0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1047                            && proto0 == SNAT_PROTOCOL_UDP
1048                            && (udp0->dst_port ==
1049                                clib_host_to_net_u16
1050                                (UDP_DST_PORT_dhcp_to_server))))
1051                         goto trace00;
1052                     }
1053                   else
1054                     {
1055                       if (PREDICT_FALSE
1056                           (snat_not_translate
1057                            (sm, node, sw_if_index0, ip0, proto0,
1058                             rx_fib_index0, thread_index)))
1059                         goto trace00;
1060                     }
1061
1062                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1063                                      &s0, node, next0, thread_index, now);
1064                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1065                     goto trace00;
1066
1067                   if (PREDICT_FALSE (!s0))
1068                     goto trace00;
1069                 }
1070               else
1071                 {
1072                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1073                   goto trace00;
1074                 }
1075             }
1076           else
1077             s0 =
1078               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1079                                  value0.value);
1080
1081           b0->flags |= VNET_BUFFER_F_IS_NATED;
1082
1083           old_addr0 = ip0->src_address.as_u32;
1084           ip0->src_address = s0->out2in.addr;
1085           new_addr0 = ip0->src_address.as_u32;
1086           if (!is_output_feature)
1087             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1088
1089           sum0 = ip0->checksum;
1090           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1091                                  ip4_header_t,
1092                                  src_address /* changed member */ );
1093           ip0->checksum = ip_csum_fold (sum0);
1094
1095           old_port0 = udp0->src_port;
1096           new_port0 = udp0->src_port = s0->out2in.port;
1097
1098           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1099             {
1100               sum0 = tcp0->checksum;
1101               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1102                                      ip4_header_t,
1103                                      dst_address /* changed member */ );
1104               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1105                                      ip4_header_t /* cheat */ ,
1106                                      length /* changed member */ );
1107               mss_clamping (sm, tcp0, &sum0);
1108               tcp0->checksum = ip_csum_fold (sum0);
1109               tcp_packets++;
1110             }
1111           else
1112             {
1113               if (PREDICT_FALSE (udp0->checksum))
1114                 {
1115                   sum0 = udp0->checksum;
1116                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1117                                          ip4_header_t,
1118                                          dst_address /* changed member */ );
1119                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1120                                          ip4_header_t /* cheat */ ,
1121                                          length /* changed member */ );
1122                   udp0->checksum = ip_csum_fold (sum0);
1123                 }
1124               udp_packets++;
1125             }
1126
1127           /* Accounting */
1128           nat44_session_update_counters (s0, now,
1129                                          vlib_buffer_length_in_chain (vm, b0),
1130                                          thread_index);
1131           /* Per-user LRU list maintenance */
1132           nat44_session_update_lru (sm, s0, thread_index);
1133         trace00:
1134
1135           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1136                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1137             {
1138               snat_in2out_trace_t *t =
1139                 vlib_add_trace (vm, node, b0, sizeof (*t));
1140               t->is_slow_path = is_slow_path;
1141               t->sw_if_index = sw_if_index0;
1142               t->next_index = next0;
1143               t->session_index = ~0;
1144               if (s0)
1145                 t->session_index =
1146                   s0 - sm->per_thread_data[thread_index].sessions;
1147             }
1148
1149           pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1150
1151           if (is_output_feature)
1152             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1153
1154           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1155                                   iph_offset1);
1156
1157           udp1 = ip4_next_header (ip1);
1158           tcp1 = (tcp_header_t *) udp1;
1159           icmp1 = (icmp46_header_t *) udp1;
1160
1161           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1162           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1163                                    sw_if_index1);
1164
1165           if (PREDICT_FALSE (ip1->ttl == 1))
1166             {
1167               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1168               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1169                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1170                                            0);
1171               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1172               goto trace01;
1173             }
1174
1175           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1176
1177           /* Next configured feature, probably ip4-lookup */
1178           if (is_slow_path)
1179             {
1180               if (PREDICT_FALSE (proto1 == ~0))
1181                 {
1182                   if (nat_in2out_sm_unknown_proto
1183                       (sm, b1, ip1, rx_fib_index1))
1184                     {
1185                       next1 = SNAT_IN2OUT_NEXT_DROP;
1186                       b1->error =
1187                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1188                     }
1189                   other_packets++;
1190                   goto trace01;
1191                 }
1192
1193               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1194                 {
1195                   next1 = icmp_in2out_slow_path
1196                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1197                      next1, now, thread_index, &s1);
1198                   icmp_packets++;
1199                   goto trace01;
1200                 }
1201             }
1202           else
1203             {
1204               if (PREDICT_FALSE (proto1 == ~0))
1205                 {
1206                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1207                   goto trace01;
1208                 }
1209
1210               if (ip4_is_fragment (ip1))
1211                 {
1212                   next1 = SNAT_IN2OUT_NEXT_REASS;
1213                   fragments++;
1214                   goto trace01;
1215                 }
1216
1217               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1218                 {
1219                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1220                   goto trace01;
1221                 }
1222             }
1223
1224           key1.addr = ip1->src_address;
1225           key1.port = udp1->src_port;
1226           key1.protocol = proto1;
1227           key1.fib_index = rx_fib_index1;
1228
1229           kv1.key = key1.as_u64;
1230
1231           if (PREDICT_FALSE
1232               (clib_bihash_search_8_8
1233                (&sm->per_thread_data[thread_index].in2out, &kv1,
1234                 &value1) != 0))
1235             {
1236               if (is_slow_path)
1237                 {
1238                   if (is_output_feature)
1239                     {
1240                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1241                                                                            ip1,
1242                                                                            proto1,
1243                                                                            udp1->src_port,
1244                                                                            udp1->dst_port,
1245                                                                            thread_index,
1246                                                                            sw_if_index1)))
1247                         goto trace01;
1248
1249                       /*
1250                        * Send DHCP packets to the ipv4 stack, or we won't
1251                        * be able to use dhcp client on the outside interface
1252                        */
1253                       if (PREDICT_FALSE
1254                           ((b1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1255                            && proto1 == SNAT_PROTOCOL_UDP
1256                            && (udp1->dst_port ==
1257                                clib_host_to_net_u16
1258                                (UDP_DST_PORT_dhcp_to_server))))
1259                         goto trace01;
1260                     }
1261                   else
1262                     {
1263                       if (PREDICT_FALSE
1264                           (snat_not_translate
1265                            (sm, node, sw_if_index1, ip1, proto1,
1266                             rx_fib_index1, thread_index)))
1267                         goto trace01;
1268                     }
1269
1270                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1271                                      &s1, node, next1, thread_index, now);
1272                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1273                     goto trace01;
1274
1275                   if (PREDICT_FALSE (!s1))
1276                     goto trace01;
1277                 }
1278               else
1279                 {
1280                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1281                   goto trace01;
1282                 }
1283             }
1284           else
1285             s1 =
1286               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1287                                  value1.value);
1288
1289           b1->flags |= VNET_BUFFER_F_IS_NATED;
1290
1291           old_addr1 = ip1->src_address.as_u32;
1292           ip1->src_address = s1->out2in.addr;
1293           new_addr1 = ip1->src_address.as_u32;
1294           if (!is_output_feature)
1295             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1296
1297           sum1 = ip1->checksum;
1298           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1299                                  ip4_header_t,
1300                                  src_address /* changed member */ );
1301           ip1->checksum = ip_csum_fold (sum1);
1302
1303           old_port1 = udp1->src_port;
1304           new_port1 = udp1->src_port = s1->out2in.port;
1305
1306           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1307             {
1308               sum1 = tcp1->checksum;
1309               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1310                                      ip4_header_t,
1311                                      dst_address /* changed member */ );
1312               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1313                                      ip4_header_t /* cheat */ ,
1314                                      length /* changed member */ );
1315               mss_clamping (sm, tcp1, &sum1);
1316               tcp1->checksum = ip_csum_fold (sum1);
1317               tcp_packets++;
1318             }
1319           else
1320             {
1321               if (PREDICT_FALSE (udp1->checksum))
1322                 {
1323                   sum1 = udp1->checksum;
1324                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1325                                          ip4_header_t,
1326                                          dst_address /* changed member */ );
1327                   sum1 = ip_csum_update (sum1, old_port1, new_port1,
1328                                          ip4_header_t /* cheat */ ,
1329                                          length /* changed member */ );
1330                   udp1->checksum = ip_csum_fold (sum1);
1331                 }
1332               udp_packets++;
1333             }
1334
1335           /* Accounting */
1336           nat44_session_update_counters (s1, now,
1337                                          vlib_buffer_length_in_chain (vm, b1),
1338                                          thread_index);
1339           /* Per-user LRU list maintenance */
1340           nat44_session_update_lru (sm, s1, thread_index);
1341         trace01:
1342
1343           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1344                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1345             {
1346               snat_in2out_trace_t *t =
1347                 vlib_add_trace (vm, node, b1, sizeof (*t));
1348               t->sw_if_index = sw_if_index1;
1349               t->next_index = next1;
1350               t->session_index = ~0;
1351               if (s1)
1352                 t->session_index =
1353                   s1 - sm->per_thread_data[thread_index].sessions;
1354             }
1355
1356           pkts_processed += next1 == SNAT_IN2OUT_NEXT_LOOKUP;
1357
1358           /* verify speculative enqueues, maybe switch current next frame */
1359           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1360                                            to_next, n_left_to_next,
1361                                            bi0, bi1, next0, next1);
1362         }
1363
1364       while (n_left_from > 0 && n_left_to_next > 0)
1365         {
1366           u32 bi0;
1367           vlib_buffer_t *b0;
1368           u32 next0;
1369           u32 sw_if_index0;
1370           ip4_header_t *ip0;
1371           ip_csum_t sum0;
1372           u32 new_addr0, old_addr0;
1373           u16 old_port0, new_port0;
1374           udp_header_t *udp0;
1375           tcp_header_t *tcp0;
1376           icmp46_header_t *icmp0;
1377           snat_session_key_t key0;
1378           u32 rx_fib_index0;
1379           u32 proto0;
1380           snat_session_t *s0 = 0;
1381           clib_bihash_kv_8_8_t kv0, value0;
1382           u32 iph_offset0 = 0;
1383
1384           /* speculatively enqueue b0 to the current next frame */
1385           bi0 = from[0];
1386           to_next[0] = bi0;
1387           from += 1;
1388           to_next += 1;
1389           n_left_from -= 1;
1390           n_left_to_next -= 1;
1391
1392           b0 = vlib_get_buffer (vm, bi0);
1393           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1394
1395           if (is_output_feature)
1396             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1397
1398           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1399                                   iph_offset0);
1400
1401           udp0 = ip4_next_header (ip0);
1402           tcp0 = (tcp_header_t *) udp0;
1403           icmp0 = (icmp46_header_t *) udp0;
1404
1405           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1406           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1407                                    sw_if_index0);
1408
1409           if (PREDICT_FALSE (ip0->ttl == 1))
1410             {
1411               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1412               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1413                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1414                                            0);
1415               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1416               goto trace0;
1417             }
1418
1419           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1420
1421           /* Next configured feature, probably ip4-lookup */
1422           if (is_slow_path)
1423             {
1424               if (PREDICT_FALSE (proto0 == ~0))
1425                 {
1426                   if (nat_in2out_sm_unknown_proto
1427                       (sm, b0, ip0, rx_fib_index0))
1428                     {
1429                       next0 = SNAT_IN2OUT_NEXT_DROP;
1430                       b0->error =
1431                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1432                     }
1433                   other_packets++;
1434                   goto trace0;
1435                 }
1436
1437               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1438                 {
1439                   next0 = icmp_in2out_slow_path
1440                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1441                      next0, now, thread_index, &s0);
1442                   icmp_packets++;
1443                   goto trace0;
1444                 }
1445             }
1446           else
1447             {
1448               if (PREDICT_FALSE (proto0 == ~0))
1449                 {
1450                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1451                   goto trace0;
1452                 }
1453
1454               if (ip4_is_fragment (ip0))
1455                 {
1456                   next0 = SNAT_IN2OUT_NEXT_REASS;
1457                   fragments++;
1458                   goto trace0;
1459                 }
1460
1461               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1462                 {
1463                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1464                   goto trace0;
1465                 }
1466             }
1467
1468           key0.addr = ip0->src_address;
1469           key0.port = udp0->src_port;
1470           key0.protocol = proto0;
1471           key0.fib_index = rx_fib_index0;
1472
1473           kv0.key = key0.as_u64;
1474
1475           if (clib_bihash_search_8_8
1476               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1477             {
1478               if (is_slow_path)
1479                 {
1480                   if (is_output_feature)
1481                     {
1482                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1483                                                                            ip0,
1484                                                                            proto0,
1485                                                                            udp0->src_port,
1486                                                                            udp0->dst_port,
1487                                                                            thread_index,
1488                                                                            sw_if_index0)))
1489                         goto trace0;
1490
1491                       /*
1492                        * Send DHCP packets to the ipv4 stack, or we won't
1493                        * be able to use dhcp client on the outside interface
1494                        */
1495                       if (PREDICT_FALSE
1496                           ((b0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1497                            && proto0 == SNAT_PROTOCOL_UDP
1498                            && (udp0->dst_port ==
1499                                clib_host_to_net_u16
1500                                (UDP_DST_PORT_dhcp_to_server))))
1501                         goto trace0;
1502                     }
1503                   else
1504                     {
1505                       if (PREDICT_FALSE
1506                           (snat_not_translate
1507                            (sm, node, sw_if_index0, ip0, proto0,
1508                             rx_fib_index0, thread_index)))
1509                         goto trace0;
1510                     }
1511
1512                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1513                                      &s0, node, next0, thread_index, now);
1514
1515                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1516                     goto trace0;
1517
1518                   if (PREDICT_FALSE (!s0))
1519                     goto trace0;
1520                 }
1521               else
1522                 {
1523                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1524                   goto trace0;
1525                 }
1526             }
1527           else
1528             s0 =
1529               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1530                                  value0.value);
1531
1532           b0->flags |= VNET_BUFFER_F_IS_NATED;
1533
1534           old_addr0 = ip0->src_address.as_u32;
1535           ip0->src_address = s0->out2in.addr;
1536           new_addr0 = ip0->src_address.as_u32;
1537           if (!is_output_feature)
1538             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1539
1540           sum0 = ip0->checksum;
1541           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1542                                  ip4_header_t,
1543                                  src_address /* changed member */ );
1544           ip0->checksum = ip_csum_fold (sum0);
1545
1546           old_port0 = udp0->src_port;
1547           new_port0 = udp0->src_port = s0->out2in.port;
1548
1549           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1550             {
1551               sum0 = tcp0->checksum;
1552               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1553                                      ip4_header_t,
1554                                      dst_address /* changed member */ );
1555               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1556                                      ip4_header_t /* cheat */ ,
1557                                      length /* changed member */ );
1558               mss_clamping (sm, tcp0, &sum0);
1559               tcp0->checksum = ip_csum_fold (sum0);
1560               tcp_packets++;
1561             }
1562           else
1563             {
1564               if (PREDICT_FALSE (udp0->checksum))
1565                 {
1566                   sum0 = udp0->checksum;
1567                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1568                                          ip4_header_t,
1569                                          dst_address /* changed member */ );
1570                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1571                                          ip4_header_t /* cheat */ ,
1572                                          length /* changed member */ );
1573                   udp0->checksum = ip_csum_fold (sum0);
1574                 }
1575               udp_packets++;
1576             }
1577
1578           /* Accounting */
1579           nat44_session_update_counters (s0, now,
1580                                          vlib_buffer_length_in_chain (vm, b0),
1581                                          thread_index);
1582           /* Per-user LRU list maintenance */
1583           nat44_session_update_lru (sm, s0, thread_index);
1584
1585         trace0:
1586           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1587                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1588             {
1589               snat_in2out_trace_t *t =
1590                 vlib_add_trace (vm, node, b0, sizeof (*t));
1591               t->is_slow_path = is_slow_path;
1592               t->sw_if_index = sw_if_index0;
1593               t->next_index = next0;
1594               t->session_index = ~0;
1595               if (s0)
1596                 t->session_index =
1597                   s0 - sm->per_thread_data[thread_index].sessions;
1598             }
1599
1600           pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1601
1602           /* verify speculative enqueue, maybe switch current next frame */
1603           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1604                                            to_next, n_left_to_next,
1605                                            bi0, next0);
1606         }
1607
1608       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1609     }
1610
1611   vlib_node_increment_counter (vm, stats_node_index,
1612                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1613                                pkts_processed);
1614   vlib_node_increment_counter (vm, stats_node_index,
1615                                SNAT_IN2OUT_ERROR_TCP_PACKETS, tcp_packets);
1616   vlib_node_increment_counter (vm, stats_node_index,
1617                                SNAT_IN2OUT_ERROR_UDP_PACKETS, udp_packets);
1618   vlib_node_increment_counter (vm, stats_node_index,
1619                                SNAT_IN2OUT_ERROR_ICMP_PACKETS, icmp_packets);
1620   vlib_node_increment_counter (vm, stats_node_index,
1621                                SNAT_IN2OUT_ERROR_OTHER_PACKETS,
1622                                other_packets);
1623   vlib_node_increment_counter (vm, stats_node_index,
1624                                SNAT_IN2OUT_ERROR_FRAGMENTS, fragments);
1625
1626   return frame->n_vectors;
1627 }
1628
1629 VLIB_NODE_FN (snat_in2out_node) (vlib_main_t * vm,
1630                                  vlib_node_runtime_t * node,
1631                                  vlib_frame_t * frame)
1632 {
1633   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1634                                      0);
1635 }
1636
1637 /* *INDENT-OFF* */
1638 VLIB_REGISTER_NODE (snat_in2out_node) = {
1639   .name = "nat44-in2out",
1640   .vector_size = sizeof (u32),
1641   .format_trace = format_snat_in2out_trace,
1642   .type = VLIB_NODE_TYPE_INTERNAL,
1643
1644   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1645   .error_strings = snat_in2out_error_strings,
1646
1647   .runtime_data_bytes = sizeof (snat_runtime_t),
1648
1649   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1650
1651   /* edit / add dispositions here */
1652   .next_nodes = {
1653     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1654     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1655     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1656     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1657     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1658   },
1659 };
1660 /* *INDENT-ON* */
1661
1662 VLIB_NODE_FN (snat_in2out_output_node) (vlib_main_t * vm,
1663                                         vlib_node_runtime_t * node,
1664                                         vlib_frame_t * frame)
1665 {
1666   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1667                                      1);
1668 }
1669
1670 /* *INDENT-OFF* */
1671 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1672   .name = "nat44-in2out-output",
1673   .vector_size = sizeof (u32),
1674   .format_trace = format_snat_in2out_trace,
1675   .type = VLIB_NODE_TYPE_INTERNAL,
1676
1677   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1678   .error_strings = snat_in2out_error_strings,
1679
1680   .runtime_data_bytes = sizeof (snat_runtime_t),
1681
1682   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1683
1684   /* edit / add dispositions here */
1685   .next_nodes = {
1686     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1687     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1688     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1689     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1690     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1691   },
1692 };
1693 /* *INDENT-ON* */
1694
1695 VLIB_NODE_FN (snat_in2out_slowpath_node) (vlib_main_t * vm,
1696                                           vlib_node_runtime_t * node,
1697                                           vlib_frame_t * frame)
1698 {
1699   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1700                                      0);
1701 }
1702
1703 /* *INDENT-OFF* */
1704 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1705   .name = "nat44-in2out-slowpath",
1706   .vector_size = sizeof (u32),
1707   .format_trace = format_snat_in2out_trace,
1708   .type = VLIB_NODE_TYPE_INTERNAL,
1709
1710   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1711   .error_strings = snat_in2out_error_strings,
1712
1713   .runtime_data_bytes = sizeof (snat_runtime_t),
1714
1715   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1716
1717   /* edit / add dispositions here */
1718   .next_nodes = {
1719     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1720     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1721     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1722     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1723     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1724   },
1725 };
1726 /* *INDENT-ON* */
1727
1728 VLIB_NODE_FN (snat_in2out_output_slowpath_node) (vlib_main_t * vm,
1729                                                  vlib_node_runtime_t * node,
1730                                                  vlib_frame_t * frame)
1731 {
1732   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1733                                      1);
1734 }
1735
1736 /* *INDENT-OFF* */
1737 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1738   .name = "nat44-in2out-output-slowpath",
1739   .vector_size = sizeof (u32),
1740   .format_trace = format_snat_in2out_trace,
1741   .type = VLIB_NODE_TYPE_INTERNAL,
1742
1743   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1744   .error_strings = snat_in2out_error_strings,
1745
1746   .runtime_data_bytes = sizeof (snat_runtime_t),
1747
1748   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1749
1750   /* edit / add dispositions here */
1751   .next_nodes = {
1752     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1753     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1754     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1755     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1756     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1757   },
1758 };
1759 /* *INDENT-ON* */
1760
1761 VLIB_NODE_FN (nat44_in2out_reass_node) (vlib_main_t * vm,
1762                                         vlib_node_runtime_t * node,
1763                                         vlib_frame_t * frame)
1764 {
1765   u32 n_left_from, *from, *to_next;
1766   snat_in2out_next_t next_index;
1767   u32 pkts_processed = 0, cached_fragments = 0;
1768   snat_main_t *sm = &snat_main;
1769   f64 now = vlib_time_now (vm);
1770   u32 thread_index = vm->thread_index;
1771   snat_main_per_thread_data_t *per_thread_data =
1772     &sm->per_thread_data[thread_index];
1773   u32 *fragments_to_drop = 0;
1774   u32 *fragments_to_loopback = 0;
1775
1776   from = vlib_frame_vector_args (frame);
1777   n_left_from = frame->n_vectors;
1778   next_index = node->cached_next_index;
1779
1780   while (n_left_from > 0)
1781     {
1782       u32 n_left_to_next;
1783
1784       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1785
1786       while (n_left_from > 0 && n_left_to_next > 0)
1787         {
1788           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1789           vlib_buffer_t *b0;
1790           u32 next0;
1791           u8 cached0 = 0;
1792           ip4_header_t *ip0;
1793           nat_reass_ip4_t *reass0;
1794           udp_header_t *udp0;
1795           tcp_header_t *tcp0;
1796           icmp46_header_t *icmp0;
1797           snat_session_key_t key0;
1798           clib_bihash_kv_8_8_t kv0, value0;
1799           snat_session_t *s0 = 0;
1800           u16 old_port0, new_port0;
1801           ip_csum_t sum0;
1802
1803           /* speculatively enqueue b0 to the current next frame */
1804           bi0 = from[0];
1805           to_next[0] = bi0;
1806           from += 1;
1807           to_next += 1;
1808           n_left_from -= 1;
1809           n_left_to_next -= 1;
1810
1811           b0 = vlib_get_buffer (vm, bi0);
1812           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1813
1814           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1815           rx_fib_index0 =
1816             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1817                                                  sw_if_index0);
1818
1819           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1820             {
1821               next0 = SNAT_IN2OUT_NEXT_DROP;
1822               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1823               goto trace0;
1824             }
1825
1826           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1827           udp0 = ip4_next_header (ip0);
1828           tcp0 = (tcp_header_t *) udp0;
1829           icmp0 = (icmp46_header_t *) udp0;
1830           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1831
1832           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1833                                                  ip0->dst_address,
1834                                                  ip0->fragment_id,
1835                                                  ip0->protocol,
1836                                                  1, &fragments_to_drop);
1837
1838           if (PREDICT_FALSE (!reass0))
1839             {
1840               next0 = SNAT_IN2OUT_NEXT_DROP;
1841               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1842               nat_elog_notice ("maximum reassemblies exceeded");
1843               goto trace0;
1844             }
1845
1846           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1847             {
1848               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1849                 {
1850                   next0 = icmp_in2out_slow_path
1851                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1852                      next0, now, thread_index, &s0);
1853
1854                   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP))
1855                     {
1856                       if (s0)
1857                         reass0->sess_index = s0 - per_thread_data->sessions;
1858                       else
1859                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1860                       nat_ip4_reass_get_frags (reass0,
1861                                                &fragments_to_loopback);
1862                     }
1863
1864                   goto trace0;
1865                 }
1866
1867               key0.addr = ip0->src_address;
1868               key0.port = udp0->src_port;
1869               key0.protocol = proto0;
1870               key0.fib_index = rx_fib_index0;
1871               kv0.key = key0.as_u64;
1872
1873               if (clib_bihash_search_8_8
1874                   (&per_thread_data->in2out, &kv0, &value0))
1875                 {
1876                   if (PREDICT_FALSE
1877                       (snat_not_translate
1878                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1879                         thread_index)))
1880                     goto trace0;
1881
1882                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1883                                      &s0, node, next0, thread_index, now);
1884
1885                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1886                     goto trace0;
1887
1888                   if (PREDICT_FALSE (!s0))
1889                     goto trace0;
1890
1891                   reass0->sess_index = s0 - per_thread_data->sessions;
1892                 }
1893               else
1894                 {
1895                   s0 = pool_elt_at_index (per_thread_data->sessions,
1896                                           value0.value);
1897                   reass0->sess_index = value0.value;
1898                 }
1899               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1900             }
1901           else
1902             {
1903               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1904                 {
1905                   if (nat_ip4_reass_add_fragment
1906                       (thread_index, reass0, bi0, &fragments_to_drop))
1907                     {
1908                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1909                       nat_elog_notice
1910                         ("maximum fragments per reassembly exceeded");
1911                       next0 = SNAT_IN2OUT_NEXT_DROP;
1912                       goto trace0;
1913                     }
1914                   cached0 = 1;
1915                   goto trace0;
1916                 }
1917               s0 = pool_elt_at_index (per_thread_data->sessions,
1918                                       reass0->sess_index);
1919             }
1920
1921           old_addr0 = ip0->src_address.as_u32;
1922           ip0->src_address = s0->out2in.addr;
1923           new_addr0 = ip0->src_address.as_u32;
1924           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1925
1926           sum0 = ip0->checksum;
1927           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1928                                  ip4_header_t,
1929                                  src_address /* changed member */ );
1930           ip0->checksum = ip_csum_fold (sum0);
1931
1932           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1933             {
1934               old_port0 = udp0->src_port;
1935               new_port0 = udp0->src_port = s0->out2in.port;
1936
1937               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1938                 {
1939                   sum0 = tcp0->checksum;
1940                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1941                                          ip4_header_t,
1942                                          dst_address /* changed member */ );
1943                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1944                                          ip4_header_t /* cheat */ ,
1945                                          length /* changed member */ );
1946                   tcp0->checksum = ip_csum_fold (sum0);
1947                 }
1948               else if (PREDICT_FALSE (udp0->checksum))
1949                 {
1950                   sum0 = udp0->checksum;
1951                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1952                                          ip4_header_t,
1953                                          dst_address /* changed member */ );
1954                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1955                                          ip4_header_t /* cheat */ ,
1956                                          length /* changed member */ );
1957                   udp0->checksum = ip_csum_fold (sum0);
1958                 }
1959             }
1960
1961           /* Hairpinning */
1962           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1963                                    s0->ext_host_port, proto0, 0);
1964
1965           /* Accounting */
1966           nat44_session_update_counters (s0, now,
1967                                          vlib_buffer_length_in_chain (vm, b0),
1968                                          thread_index);
1969           /* Per-user LRU list maintenance */
1970           nat44_session_update_lru (sm, s0, thread_index);
1971
1972         trace0:
1973           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1974                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1975             {
1976               nat44_reass_trace_t *t =
1977                 vlib_add_trace (vm, node, b0, sizeof (*t));
1978               t->cached = cached0;
1979               t->sw_if_index = sw_if_index0;
1980               t->next_index = next0;
1981             }
1982
1983           if (cached0)
1984             {
1985               n_left_to_next++;
1986               to_next--;
1987               cached_fragments++;
1988             }
1989           else
1990             {
1991               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1992
1993               /* verify speculative enqueue, maybe switch current next frame */
1994               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1995                                                to_next, n_left_to_next,
1996                                                bi0, next0);
1997             }
1998
1999           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2000             {
2001               from = vlib_frame_vector_args (frame);
2002               u32 len = vec_len (fragments_to_loopback);
2003               if (len <= VLIB_FRAME_SIZE)
2004                 {
2005                   clib_memcpy_fast (from, fragments_to_loopback,
2006                                     sizeof (u32) * len);
2007                   n_left_from = len;
2008                   vec_reset_length (fragments_to_loopback);
2009                 }
2010               else
2011                 {
2012                   clib_memcpy_fast (from, fragments_to_loopback +
2013                                     (len - VLIB_FRAME_SIZE),
2014                                     sizeof (u32) * VLIB_FRAME_SIZE);
2015                   n_left_from = VLIB_FRAME_SIZE;
2016                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2017                 }
2018             }
2019         }
2020
2021       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2022     }
2023
2024   vlib_node_increment_counter (vm, sm->in2out_reass_node_index,
2025                                SNAT_IN2OUT_ERROR_PROCESSED_FRAGMENTS,
2026                                pkts_processed);
2027   vlib_node_increment_counter (vm, sm->in2out_reass_node_index,
2028                                SNAT_IN2OUT_ERROR_CACHED_FRAGMENTS,
2029                                cached_fragments);
2030
2031   nat_send_all_to_node (vm, fragments_to_drop, node,
2032                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2033                         SNAT_IN2OUT_NEXT_DROP);
2034
2035   vec_free (fragments_to_drop);
2036   vec_free (fragments_to_loopback);
2037   return frame->n_vectors;
2038 }
2039
2040 /* *INDENT-OFF* */
2041 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2042   .name = "nat44-in2out-reass",
2043   .vector_size = sizeof (u32),
2044   .format_trace = format_nat44_reass_trace,
2045   .type = VLIB_NODE_TYPE_INTERNAL,
2046
2047   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2048   .error_strings = snat_in2out_error_strings,
2049
2050   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2051   .next_nodes = {
2052     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2053     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2054     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2055     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2056     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2057   },
2058 };
2059 /* *INDENT-ON* */
2060
2061 VLIB_NODE_FN (snat_in2out_fast_node) (vlib_main_t * vm,
2062                                       vlib_node_runtime_t * node,
2063                                       vlib_frame_t * frame)
2064 {
2065   u32 n_left_from, *from, *to_next;
2066   snat_in2out_next_t next_index;
2067   u32 pkts_processed = 0;
2068   snat_main_t *sm = &snat_main;
2069   u32 stats_node_index;
2070
2071   stats_node_index = sm->in2out_fast_node_index;
2072
2073   from = vlib_frame_vector_args (frame);
2074   n_left_from = frame->n_vectors;
2075   next_index = node->cached_next_index;
2076
2077   while (n_left_from > 0)
2078     {
2079       u32 n_left_to_next;
2080
2081       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2082
2083       while (n_left_from > 0 && n_left_to_next > 0)
2084         {
2085           u32 bi0;
2086           vlib_buffer_t *b0;
2087           u32 next0;
2088           u32 sw_if_index0;
2089           ip4_header_t *ip0;
2090           ip_csum_t sum0;
2091           u32 new_addr0, old_addr0;
2092           u16 old_port0, new_port0;
2093           udp_header_t *udp0;
2094           tcp_header_t *tcp0;
2095           icmp46_header_t *icmp0;
2096           snat_session_key_t key0, sm0;
2097           u32 proto0;
2098           u32 rx_fib_index0;
2099
2100           /* speculatively enqueue b0 to the current next frame */
2101           bi0 = from[0];
2102           to_next[0] = bi0;
2103           from += 1;
2104           to_next += 1;
2105           n_left_from -= 1;
2106           n_left_to_next -= 1;
2107
2108           b0 = vlib_get_buffer (vm, bi0);
2109           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2110
2111           ip0 = vlib_buffer_get_current (b0);
2112           udp0 = ip4_next_header (ip0);
2113           tcp0 = (tcp_header_t *) udp0;
2114           icmp0 = (icmp46_header_t *) udp0;
2115
2116           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2117           rx_fib_index0 =
2118             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2119
2120           if (PREDICT_FALSE (ip0->ttl == 1))
2121             {
2122               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2123               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2124                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2125                                            0);
2126               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2127               goto trace0;
2128             }
2129
2130           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2131
2132           if (PREDICT_FALSE (proto0 == ~0))
2133             goto trace0;
2134
2135           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2136             {
2137               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
2138                                    rx_fib_index0, node, next0, ~0, 0, 0);
2139               goto trace0;
2140             }
2141
2142           key0.addr = ip0->src_address;
2143           key0.protocol = proto0;
2144           key0.port = udp0->src_port;
2145           key0.fib_index = rx_fib_index0;
2146
2147           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0, 0))
2148             {
2149               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2150               next0 = SNAT_IN2OUT_NEXT_DROP;
2151               goto trace0;
2152             }
2153
2154           new_addr0 = sm0.addr.as_u32;
2155           new_port0 = sm0.port;
2156           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2157           old_addr0 = ip0->src_address.as_u32;
2158           ip0->src_address.as_u32 = new_addr0;
2159
2160           sum0 = ip0->checksum;
2161           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2162                                  ip4_header_t,
2163                                  src_address /* changed member */ );
2164           ip0->checksum = ip_csum_fold (sum0);
2165
2166           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
2167             {
2168               old_port0 = udp0->src_port;
2169               udp0->src_port = new_port0;
2170
2171               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2172                 {
2173                   sum0 = tcp0->checksum;
2174                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2175                                          ip4_header_t,
2176                                          dst_address /* changed member */ );
2177                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2178                                          ip4_header_t /* cheat */ ,
2179                                          length /* changed member */ );
2180                   mss_clamping (sm, tcp0, &sum0);
2181                   tcp0->checksum = ip_csum_fold (sum0);
2182                 }
2183               else if (udp0->checksum)
2184                 {
2185                   sum0 = udp0->checksum;
2186                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2187                                          ip4_header_t,
2188                                          dst_address /* changed member */ );
2189                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2190                                          ip4_header_t /* cheat */ ,
2191                                          length /* changed member */ );
2192                   udp0->checksum = ip_csum_fold (sum0);
2193                 }
2194             }
2195           else
2196             {
2197               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2198                 {
2199                   sum0 = tcp0->checksum;
2200                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2201                                          ip4_header_t,
2202                                          dst_address /* changed member */ );
2203                   mss_clamping (sm, tcp0, &sum0);
2204                   tcp0->checksum = ip_csum_fold (sum0);
2205                 }
2206               else if (udp0->checksum)
2207                 {
2208                   sum0 = udp0->checksum;
2209                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2210                                          ip4_header_t,
2211                                          dst_address /* changed member */ );
2212                   udp0->checksum = ip_csum_fold (sum0);
2213                 }
2214             }
2215
2216           /* Hairpinning */
2217           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2218
2219         trace0:
2220           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2221                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2222             {
2223               snat_in2out_trace_t *t =
2224                 vlib_add_trace (vm, node, b0, sizeof (*t));
2225               t->sw_if_index = sw_if_index0;
2226               t->next_index = next0;
2227             }
2228
2229           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2230
2231           /* verify speculative enqueue, maybe switch current next frame */
2232           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2233                                            to_next, n_left_to_next,
2234                                            bi0, next0);
2235         }
2236
2237       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2238     }
2239
2240   vlib_node_increment_counter (vm, stats_node_index,
2241                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2242                                pkts_processed);
2243   return frame->n_vectors;
2244 }
2245
2246
2247 /* *INDENT-OFF* */
2248 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2249   .name = "nat44-in2out-fast",
2250   .vector_size = sizeof (u32),
2251   .format_trace = format_snat_in2out_fast_trace,
2252   .type = VLIB_NODE_TYPE_INTERNAL,
2253
2254   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2255   .error_strings = snat_in2out_error_strings,
2256
2257   .runtime_data_bytes = sizeof (snat_runtime_t),
2258
2259   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2260
2261   /* edit / add dispositions here */
2262   .next_nodes = {
2263     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2264     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2265     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2266     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2267     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2268   },
2269 };
2270 /* *INDENT-ON* */
2271
2272 /*
2273  * fd.io coding-style-patch-verification: ON
2274  *
2275  * Local Variables:
2276  * eval: (c-set-style "gnu")
2277  * End:
2278  */