src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33
  34 #include <vppinfra/hash.h>
  35 #include <vppinfra/error.h>
  36 #include <vppinfra/elog.h>
  37
  38 typedef struct
  39 {
  40   u32 sw_if_index;
  41   u32 next_index;
  42   u32 session_index;
  43   u32 is_slow_path;
  44 } snat_in2out_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 *
  48 format_snat_in2out_trace (u8 * s, va_list * args)
  49 {
  50   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  51   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  52   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  53   char *tag;
  54
  55   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  56
  57   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  58               t->sw_if_index, t->next_index, t->session_index);
  59
  60   return s;
  61 }
  62
  63 static u8 *
  64 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  65 {
  66   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  67   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  68   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  69
  70   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  71               t->sw_if_index, t->next_index);
  72
  73   return s;
  74 }
  75
  76 #define foreach_snat_in2out_error                       \
  77 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  78 _(IN2OUT_PACKETS, "good in2out packets processed")      \
  79 _(OUT_OF_PORTS, "out of ports")                         \
  80 _(BAD_OUTSIDE_FIB, "outside VRF ID not found")          \
  81 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  82 _(NO_TRANSLATION, "no translation")                     \
  83 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  84 _(DROP_FRAGMENT, "drop fragment")                       \
  85 _(MAX_REASS, "maximum reassemblies exceeded")           \
  86 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  87 _(TCP_PACKETS, "TCP packets")                           \
  88 _(UDP_PACKETS, "UDP packets")                           \
  89 _(ICMP_PACKETS, "ICMP packets")                         \
  90 _(OTHER_PACKETS, "other protocol packets")              \
  91 _(FRAGMENTS, "fragments")                               \
  92 _(CACHED_FRAGMENTS, "cached fragments")                 \
  93 _(PROCESSED_FRAGMENTS, "processed fragments")
  94
  95 typedef enum
  96 {
  97 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  98   foreach_snat_in2out_error
  99 #undef _
 100     SNAT_IN2OUT_N_ERROR,
 101 } snat_in2out_error_t;
 102
 103 static char *snat_in2out_error_strings[] = {
 104 #define _(sym,string) string,
 105   foreach_snat_in2out_error
 106 #undef _
 107 };
 108
 109 typedef enum
 110 {
 111   SNAT_IN2OUT_NEXT_LOOKUP,
 112   SNAT_IN2OUT_NEXT_DROP,
 113   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 114   SNAT_IN2OUT_NEXT_SLOW_PATH,
 115   SNAT_IN2OUT_NEXT_REASS,
 116   SNAT_IN2OUT_N_NEXT,
 117 } snat_in2out_next_t;
 118
 119 static inline int
 120 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 121                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 122                     u32 rx_fib_index0, u32 thread_index)
 123 {
 124   udp_header_t *udp0 = ip4_next_header (ip0);
 125   snat_session_key_t key0, sm0;
 126   clib_bihash_kv_8_8_t kv0, value0;
 127
 128   key0.addr = ip0->dst_address;
 129   key0.port = udp0->dst_port;
 130   key0.protocol = proto0;
 131   key0.fib_index = sm->outside_fib_index;
 132   kv0.key = key0.as_u64;
 133
 134   /* NAT packet aimed at external address if */
 135   /* has active sessions */
 136   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 137                               &value0))
 138     {
 139       /* or is static mappings */
 140       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 141         return 0;
 142     }
 143   else
 144     return 0;
 145
 146   if (sm->forwarding_enabled)
 147     return 1;
 148
 149   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 150                                   rx_fib_index0);
 151 }
 152
 153 static inline int
 154 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 155                                   u32 proto0, u16 src_port, u16 dst_port,
 156                                   u32 thread_index, u32 sw_if_index)
 157 {
 158   snat_session_key_t key0;
 159   clib_bihash_kv_8_8_t kv0, value0;
 160   snat_interface_t *i;
 161
 162   /* src NAT check */
 163   key0.addr = ip0->src_address;
 164   key0.port = src_port;
 165   key0.protocol = proto0;
 166   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 167   kv0.key = key0.as_u64;
 168
 169   if (!clib_bihash_search_8_8
 170       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 171     return 1;
 172
 173   /* dst NAT check */
 174   key0.addr = ip0->dst_address;
 175   key0.port = dst_port;
 176   key0.protocol = proto0;
 177   kv0.key = key0.as_u64;
 178   if (!clib_bihash_search_8_8
 179       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 180     {
 181       /* hairpinning */
 182     /* *INDENT-OFF* */
 183     pool_foreach (i, sm->output_feature_interfaces,
 184     ({
 185       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 186         return 0;
 187     }));
 188     /* *INDENT-ON* */
 189       return 1;
 190     }
 191
 192   return 0;
 193 }
 194
 195 #ifndef CLIB_MARCH_VARIANT
 196 int
 197 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 198 {
 199   snat_main_t *sm = &snat_main;
 200   nat44_is_idle_session_ctx_t *ctx = arg;
 201   snat_session_t *s;
 202   u64 sess_timeout_time;
 203   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 204                                                        ctx->thread_index);
 205   clib_bihash_kv_8_8_t s_kv;
 206
 207   s = pool_elt_at_index (tsm->sessions, kv->value);
 208   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 209   if (ctx->now >= sess_timeout_time)
 210     {
 211       s_kv.key = s->out2in.as_u64;
 212       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 213         nat_log_warn ("out2in key del failed");
 214
 215       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 216                                            s->in2out.addr.as_u32,
 217                                            s->out2in.addr.as_u32,
 218                                            s->in2out.protocol,
 219                                            s->in2out.port,
 220                                            s->out2in.port,
 221                                            s->in2out.fib_index);
 222
 223       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 224                                &s->in2out.addr, s->in2out.port,
 225                                &s->out2in.addr, s->out2in.port,
 226                                s->in2out.protocol);
 227
 228       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 229                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 230                    ctx->thread_index);
 231
 232       if (!snat_is_session_static (s))
 233         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 234                                             &s->out2in);
 235
 236       nat44_delete_session (sm, s, ctx->thread_index);
 237       return 1;
 238     }
 239
 240   return 0;
 241 }
 242 #endif
 243
 244 static u32
 245 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 246            ip4_header_t * ip0,
 247            u32 rx_fib_index0,
 248            snat_session_key_t * key0,
 249            snat_session_t ** sessionp,
 250            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 251 {
 252   snat_user_t *u;
 253   snat_session_t *s = 0;
 254   clib_bihash_kv_8_8_t kv0;
 255   snat_session_key_t key1;
 256   udp_header_t *udp0 = ip4_next_header (ip0);
 257   u8 is_sm = 0;
 258   nat_outside_fib_t *outside_fib;
 259   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 260   u8 identity_nat;
 261   fib_prefix_t pfx = {
 262     .fp_proto = FIB_PROTOCOL_IP4,
 263     .fp_len = 32,
 264     .fp_addr = {
 265                 .ip4.as_u32 = ip0->dst_address.as_u32,
 266                 },
 267   };
 268   nat44_is_idle_session_ctx_t ctx0;
 269
 270   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 271     {
 272       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 273       nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 274       nat_log_notice ("maximum sessions exceeded");
 275       return SNAT_IN2OUT_NEXT_DROP;
 276     }
 277
 278   key1.protocol = key0->protocol;
 279
 280   /* First try to match static mapping by local address and port */
 281   if (snat_static_mapping_match
 282       (sm, *key0, &key1, 0, 0, 0, 0, 0, &identity_nat))
 283     {
 284       /* Try to create dynamic translation */
 285       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 286                                                thread_index, &key1,
 287                                                sm->port_per_thread,
 288                                                sm->per_thread_data
 289                                                [thread_index].snat_thread_index))
 290         {
 291           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 292           return SNAT_IN2OUT_NEXT_DROP;
 293         }
 294     }
 295   else
 296     {
 297       if (PREDICT_FALSE (identity_nat))
 298         {
 299           *sessionp = s;
 300           return next0;
 301         }
 302
 303       is_sm = 1;
 304     }
 305
 306   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 307                               thread_index);
 308   if (!u)
 309     {
 310       nat_log_warn ("create NAT user failed");
 311       return SNAT_IN2OUT_NEXT_DROP;
 312     }
 313
 314   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 315   if (!s)
 316     {
 317       nat44_delete_user_with_no_session (sm, u, thread_index);
 318       nat_log_warn ("create NAT session failed");
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   if (is_sm)
 323     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 324   user_session_increment (sm, u, is_sm);
 325   s->in2out = *key0;
 326   s->out2in = key1;
 327   s->out2in.protocol = key0->protocol;
 328   s->out2in.fib_index = sm->outside_fib_index;
 329   switch (vec_len (sm->outside_fibs))
 330     {
 331     case 0:
 332       s->out2in.fib_index = sm->outside_fib_index;
 333       break;
 334     case 1:
 335       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 336       break;
 337     default:
 338       /* *INDENT-OFF* */
 339       vec_foreach (outside_fib, sm->outside_fibs)
 340         {
 341           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 342           if (FIB_NODE_INDEX_INVALID != fei)
 343             {
 344               if (fib_entry_get_resolving_interface (fei) != ~0)
 345                 {
 346                   s->out2in.fib_index = outside_fib->fib_index;
 347                   break;
 348                 }
 349             }
 350         }
 351       /* *INDENT-ON* */
 352       break;
 353     }
 354   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 355   s->ext_host_port = udp0->dst_port;
 356   *sessionp = s;
 357
 358   /* Add to translation hashes */
 359   ctx0.now = now;
 360   ctx0.thread_index = thread_index;
 361   kv0.key = s->in2out.as_u64;
 362   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 363   if (clib_bihash_add_or_overwrite_stale_8_8
 364       (&sm->per_thread_data[thread_index].in2out, &kv0,
 365        nat44_i2o_is_idle_session_cb, &ctx0))
 366     nat_log_notice ("in2out key add failed");
 367
 368   kv0.key = s->out2in.as_u64;
 369   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 370
 371   if (clib_bihash_add_or_overwrite_stale_8_8
 372       (&sm->per_thread_data[thread_index].out2in, &kv0,
 373        nat44_o2i_is_idle_session_cb, &ctx0))
 374     nat_log_notice ("out2in key add failed");
 375
 376   /* log NAT event */
 377   snat_ipfix_logging_nat44_ses_create (thread_index,
 378                                        s->in2out.addr.as_u32,
 379                                        s->out2in.addr.as_u32,
 380                                        s->in2out.protocol,
 381                                        s->in2out.port,
 382                                        s->out2in.port, s->in2out.fib_index);
 383
 384   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 385                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 386                            s->out2in.port, s->in2out.protocol);
 387
 388   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 389                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 390                &s->ext_host_nat_addr, s->ext_host_nat_port,
 391                s->in2out.protocol, s->in2out.fib_index, s->flags,
 392                thread_index, 0);
 393
 394   return next0;
 395 }
 396
 397 #ifndef CLIB_MARCH_VARIANT
 398 static_always_inline
 399   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 400                                     snat_session_key_t * p_key0)
 401 {
 402   icmp46_header_t *icmp0;
 403   snat_session_key_t key0;
 404   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 405   ip4_header_t *inner_ip0 = 0;
 406   void *l4_header = 0;
 407   icmp46_header_t *inner_icmp0;
 408
 409   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 410   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 411
 412   if (!icmp_is_error_message (icmp0))
 413     {
 414       key0.protocol = SNAT_PROTOCOL_ICMP;
 415       key0.addr = ip0->src_address;
 416       key0.port = echo0->identifier;
 417     }
 418   else
 419     {
 420       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 421       l4_header = ip4_next_header (inner_ip0);
 422       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 423       key0.addr = inner_ip0->dst_address;
 424       switch (key0.protocol)
 425         {
 426         case SNAT_PROTOCOL_ICMP:
 427           inner_icmp0 = (icmp46_header_t *) l4_header;
 428           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 429           key0.port = inner_echo0->identifier;
 430           break;
 431         case SNAT_PROTOCOL_UDP:
 432         case SNAT_PROTOCOL_TCP:
 433           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 434           break;
 435         default:
 436           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 437         }
 438     }
 439   *p_key0 = key0;
 440   return -1;                    /* success */
 441 }
 442
 443 /**
 444  * Get address and port values to be used for ICMP packet translation
 445  * and create session if needed
 446  *
 447  * @param[in,out] sm             NAT main
 448  * @param[in,out] node           NAT node runtime
 449  * @param[in] thread_index       thread index
 450  * @param[in,out] b0             buffer containing packet to be translated
 451  * @param[out] p_proto           protocol used for matching
 452  * @param[out] p_value           address and port after NAT translation
 453  * @param[out] p_dont_translate  if packet should not be translated
 454  * @param d                      optional parameter
 455  * @param e                      optional parameter
 456  */
 457 u32
 458 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 459                         u32 thread_index, vlib_buffer_t * b0,
 460                         ip4_header_t * ip0, u8 * p_proto,
 461                         snat_session_key_t * p_value,
 462                         u8 * p_dont_translate, void *d, void *e)
 463 {
 464   icmp46_header_t *icmp0;
 465   u32 sw_if_index0;
 466   u32 rx_fib_index0;
 467   snat_session_key_t key0;
 468   snat_session_t *s0 = 0;
 469   u8 dont_translate = 0;
 470   clib_bihash_kv_8_8_t kv0, value0;
 471   u32 next0 = ~0;
 472   int err;
 473
 474   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 475   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 476   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 477
 478   err = icmp_get_key (ip0, &key0);
 479   if (err != -1)
 480     {
 481       b0->error = node->errors[err];
 482       next0 = SNAT_IN2OUT_NEXT_DROP;
 483       goto out;
 484     }
 485   key0.fib_index = rx_fib_index0;
 486
 487   kv0.key = key0.as_u64;
 488
 489   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 490                               &value0))
 491     {
 492       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 493         {
 494           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 495                                                                key0.protocol,
 496                                                                key0.port,
 497                                                                key0.port,
 498                                                                thread_index,
 499                                                                sw_if_index0)))
 500             {
 501               dont_translate = 1;
 502               goto out;
 503             }
 504         }
 505       else
 506         {
 507           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 508                                                  ip0, SNAT_PROTOCOL_ICMP,
 509                                                  rx_fib_index0,
 510                                                  thread_index)))
 511             {
 512               dont_translate = 1;
 513               goto out;
 514             }
 515         }
 516
 517       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 518         {
 519           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 520           next0 = SNAT_IN2OUT_NEXT_DROP;
 521           goto out;
 522         }
 523
 524       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 525                          thread_index, vlib_time_now (sm->vlib_main));
 526
 527       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 528         goto out;
 529
 530       if (!s0)
 531         {
 532           dont_translate = 1;
 533           goto out;
 534         }
 535     }
 536   else
 537     {
 538       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 539                          icmp0->type != ICMP4_echo_reply &&
 540                          !icmp_is_error_message (icmp0)))
 541         {
 542           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 543           next0 = SNAT_IN2OUT_NEXT_DROP;
 544           goto out;
 545         }
 546
 547       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 548                               value0.value);
 549     }
 550
 551 out:
 552   *p_proto = key0.protocol;
 553   if (s0)
 554     *p_value = s0->out2in;
 555   *p_dont_translate = dont_translate;
 556   if (d)
 557     *(snat_session_t **) d = s0;
 558   return next0;
 559 }
 560 #endif
 561
 562 #ifndef CLIB_MARCH_VARIANT
 563 /**
 564  * Get address and port values to be used for ICMP packet translation
 565  *
 566  * @param[in] sm                 NAT main
 567  * @param[in,out] node           NAT node runtime
 568  * @param[in] thread_index       thread index
 569  * @param[in,out] b0             buffer containing packet to be translated
 570  * @param[out] p_proto           protocol used for matching
 571  * @param[out] p_value           address and port after NAT translation
 572  * @param[out] p_dont_translate  if packet should not be translated
 573  * @param d                      optional parameter
 574  * @param e                      optional parameter
 575  */
 576 u32
 577 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 578                         u32 thread_index, vlib_buffer_t * b0,
 579                         ip4_header_t * ip0, u8 * p_proto,
 580                         snat_session_key_t * p_value,
 581                         u8 * p_dont_translate, void *d, void *e)
 582 {
 583   icmp46_header_t *icmp0;
 584   u32 sw_if_index0;
 585   u32 rx_fib_index0;
 586   snat_session_key_t key0;
 587   snat_session_key_t sm0;
 588   u8 dont_translate = 0;
 589   u8 is_addr_only;
 590   u32 next0 = ~0;
 591   int err;
 592
 593   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 594   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 595   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 596
 597   err = icmp_get_key (ip0, &key0);
 598   if (err != -1)
 599     {
 600       b0->error = node->errors[err];
 601       next0 = SNAT_IN2OUT_NEXT_DROP;
 602       goto out2;
 603     }
 604   key0.fib_index = rx_fib_index0;
 605
 606   if (snat_static_mapping_match
 607       (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0, 0))
 608     {
 609       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 610                                                   IP_PROTOCOL_ICMP,
 611                                                   rx_fib_index0)))
 612         {
 613           dont_translate = 1;
 614           goto out;
 615         }
 616
 617       if (icmp_is_error_message (icmp0))
 618         {
 619           next0 = SNAT_IN2OUT_NEXT_DROP;
 620           goto out;
 621         }
 622
 623       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 624       next0 = SNAT_IN2OUT_NEXT_DROP;
 625       goto out;
 626     }
 627
 628   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 629                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 630                      !icmp_is_error_message (icmp0)))
 631     {
 632       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 633       next0 = SNAT_IN2OUT_NEXT_DROP;
 634       goto out;
 635     }
 636
 637 out:
 638   *p_value = sm0;
 639 out2:
 640   *p_proto = key0.protocol;
 641   *p_dont_translate = dont_translate;
 642   return next0;
 643 }
 644 #endif
 645
 646 #ifndef CLIB_MARCH_VARIANT
 647 u32
 648 icmp_in2out (snat_main_t * sm,
 649              vlib_buffer_t * b0,
 650              ip4_header_t * ip0,
 651              icmp46_header_t * icmp0,
 652              u32 sw_if_index0,
 653              u32 rx_fib_index0,
 654              vlib_node_runtime_t * node,
 655              u32 next0, u32 thread_index, void *d, void *e)
 656 {
 657   snat_session_key_t sm0;
 658   u8 protocol;
 659   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 660   ip4_header_t *inner_ip0;
 661   void *l4_header = 0;
 662   icmp46_header_t *inner_icmp0;
 663   u8 dont_translate;
 664   u32 new_addr0, old_addr0;
 665   u16 old_id0, new_id0;
 666   u16 old_checksum0, new_checksum0;
 667   ip_csum_t sum0;
 668   u16 checksum0;
 669   u32 next0_tmp;
 670
 671   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 672
 673   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 674                                         &protocol, &sm0, &dont_translate, d,
 675                                         e);
 676   if (next0_tmp != ~0)
 677     next0 = next0_tmp;
 678   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 679     goto out;
 680
 681   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 682     {
 683       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 684                                              (u8 *)
 685                                              vlib_buffer_get_current (b0),
 686                                              ntohs (ip0->length) -
 687                                              ip4_header_bytes (ip0), 0);
 688       checksum0 = ~ip_csum_fold (sum0);
 689       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 690         {
 691           next0 = SNAT_IN2OUT_NEXT_DROP;
 692           goto out;
 693         }
 694     }
 695
 696   old_addr0 = ip0->src_address.as_u32;
 697   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 698
 699   sum0 = ip0->checksum;
 700   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 701                          src_address /* changed member */ );
 702   ip0->checksum = ip_csum_fold (sum0);
 703
 704   if (icmp0->checksum == 0)
 705     icmp0->checksum = 0xffff;
 706
 707   if (!icmp_is_error_message (icmp0))
 708     {
 709       new_id0 = sm0.port;
 710       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 711         {
 712           old_id0 = echo0->identifier;
 713           new_id0 = sm0.port;
 714           echo0->identifier = new_id0;
 715
 716           sum0 = icmp0->checksum;
 717           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 718                                  identifier);
 719           icmp0->checksum = ip_csum_fold (sum0);
 720         }
 721     }
 722   else
 723     {
 724       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 725       l4_header = ip4_next_header (inner_ip0);
 726
 727       if (!ip4_header_checksum_is_valid (inner_ip0))
 728         {
 729           next0 = SNAT_IN2OUT_NEXT_DROP;
 730           goto out;
 731         }
 732
 733       /* update inner destination IP address */
 734       old_addr0 = inner_ip0->dst_address.as_u32;
 735       inner_ip0->dst_address = sm0.addr;
 736       new_addr0 = inner_ip0->dst_address.as_u32;
 737       sum0 = icmp0->checksum;
 738       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 739                              dst_address /* changed member */ );
 740       icmp0->checksum = ip_csum_fold (sum0);
 741
 742       /* update inner IP header checksum */
 743       old_checksum0 = inner_ip0->checksum;
 744       sum0 = inner_ip0->checksum;
 745       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 746                              dst_address /* changed member */ );
 747       inner_ip0->checksum = ip_csum_fold (sum0);
 748       new_checksum0 = inner_ip0->checksum;
 749       sum0 = icmp0->checksum;
 750       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 751                              checksum);
 752       icmp0->checksum = ip_csum_fold (sum0);
 753
 754       switch (protocol)
 755         {
 756         case SNAT_PROTOCOL_ICMP:
 757           inner_icmp0 = (icmp46_header_t *) l4_header;
 758           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 759
 760           old_id0 = inner_echo0->identifier;
 761           new_id0 = sm0.port;
 762           inner_echo0->identifier = new_id0;
 763
 764           sum0 = icmp0->checksum;
 765           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 766                                  identifier);
 767           icmp0->checksum = ip_csum_fold (sum0);
 768           break;
 769         case SNAT_PROTOCOL_UDP:
 770         case SNAT_PROTOCOL_TCP:
 771           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 772           new_id0 = sm0.port;
 773           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 774
 775           sum0 = icmp0->checksum;
 776           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 777                                  dst_port);
 778           icmp0->checksum = ip_csum_fold (sum0);
 779           break;
 780         default:
 781           ASSERT (0);
 782         }
 783     }
 784
 785   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 786     {
 787       if (sm->deterministic ||
 788           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 789                                       sm->endpoint_dependent))
 790         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 791     }
 792
 793 out:
 794   return next0;
 795 }
 796 #endif
 797
 798 static inline u32
 799 icmp_in2out_slow_path (snat_main_t * sm,
 800                        vlib_buffer_t * b0,
 801                        ip4_header_t * ip0,
 802                        icmp46_header_t * icmp0,
 803                        u32 sw_if_index0,
 804                        u32 rx_fib_index0,
 805                        vlib_node_runtime_t * node,
 806                        u32 next0,
 807                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 808 {
 809   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 810                        next0, thread_index, p_s0, 0);
 811   snat_session_t *s0 = *p_s0;
 812   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 813     {
 814       /* Accounting */
 815       nat44_session_update_counters (s0, now,
 816                                      vlib_buffer_length_in_chain
 817                                      (sm->vlib_main, b0), thread_index);
 818       /* Per-user LRU list maintenance */
 819       nat44_session_update_lru (sm, s0, thread_index);
 820     }
 821   return next0;
 822 }
 823
 824 static int
 825 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 826                              vlib_buffer_t * b,
 827                              ip4_header_t * ip, u32 rx_fib_index)
 828 {
 829   clib_bihash_kv_8_8_t kv, value;
 830   snat_static_mapping_t *m;
 831   snat_session_key_t m_key;
 832   u32 old_addr, new_addr;
 833   ip_csum_t sum;
 834
 835   m_key.addr = ip->src_address;
 836   m_key.port = 0;
 837   m_key.protocol = 0;
 838   m_key.fib_index = rx_fib_index;
 839   kv.key = m_key.as_u64;
 840   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 841     return 1;
 842
 843   m = pool_elt_at_index (sm->static_mappings, value.value);
 844
 845   old_addr = ip->src_address.as_u32;
 846   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 847   sum = ip->checksum;
 848   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 849   ip->checksum = ip_csum_fold (sum);
 850
 851
 852   /* Hairpinning */
 853   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 854     {
 855       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 856       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 857     }
 858
 859   return 0;
 860 }
 861
 862 static inline uword
 863 snat_in2out_node_fn_inline (vlib_main_t * vm,
 864                             vlib_node_runtime_t * node,
 865                             vlib_frame_t * frame, int is_slow_path,
 866                             int is_output_feature)
 867 {
 868   u32 n_left_from, *from, *to_next;
 869   snat_in2out_next_t next_index;
 870   u32 pkts_processed = 0;
 871   snat_main_t *sm = &snat_main;
 872   f64 now = vlib_time_now (vm);
 873   u32 stats_node_index;
 874   u32 thread_index = vm->thread_index;
 875   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 876     0, fragments = 0;
 877
 878   stats_node_index = is_slow_path ? sm->in2out_slowpath_node_index :
 879     sm->in2out_node_index;
 880
 881   from = vlib_frame_vector_args (frame);
 882   n_left_from = frame->n_vectors;
 883   next_index = node->cached_next_index;
 884
 885   while (n_left_from > 0)
 886     {
 887       u32 n_left_to_next;
 888
 889       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 890
 891       while (n_left_from >= 4 && n_left_to_next >= 2)
 892         {
 893           u32 bi0, bi1;
 894           vlib_buffer_t *b0, *b1;
 895           u32 next0, next1;
 896           u32 sw_if_index0, sw_if_index1;
 897           ip4_header_t *ip0, *ip1;
 898           ip_csum_t sum0, sum1;
 899           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 900           u16 old_port0, new_port0, old_port1, new_port1;
 901           udp_header_t *udp0, *udp1;
 902           tcp_header_t *tcp0, *tcp1;
 903           icmp46_header_t *icmp0, *icmp1;
 904           snat_session_key_t key0, key1;
 905           u32 rx_fib_index0, rx_fib_index1;
 906           u32 proto0, proto1;
 907           snat_session_t *s0 = 0, *s1 = 0;
 908           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 909           u32 iph_offset0 = 0, iph_offset1 = 0;
 910
 911           /* Prefetch next iteration. */
 912           {
 913             vlib_buffer_t *p2, *p3;
 914
 915             p2 = vlib_get_buffer (vm, from[2]);
 916             p3 = vlib_get_buffer (vm, from[3]);
 917
 918             vlib_prefetch_buffer_header (p2, LOAD);
 919             vlib_prefetch_buffer_header (p3, LOAD);
 920
 921             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 922             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 923           }
 924
 925           /* speculatively enqueue b0 and b1 to the current next frame */
 926           to_next[0] = bi0 = from[0];
 927           to_next[1] = bi1 = from[1];
 928           from += 2;
 929           to_next += 2;
 930           n_left_from -= 2;
 931           n_left_to_next -= 2;
 932
 933           b0 = vlib_get_buffer (vm, bi0);
 934           b1 = vlib_get_buffer (vm, bi1);
 935
 936           if (is_output_feature)
 937             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 938
 939           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 940                                   iph_offset0);
 941
 942           udp0 = ip4_next_header (ip0);
 943           tcp0 = (tcp_header_t *) udp0;
 944           icmp0 = (icmp46_header_t *) udp0;
 945
 946           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 947           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 948                                    sw_if_index0);
 949
 950           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 951
 952           if (PREDICT_FALSE (ip0->ttl == 1))
 953             {
 954               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 955               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 956                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 957                                            0);
 958               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 959               goto trace00;
 960             }
 961
 962           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 963
 964           /* Next configured feature, probably ip4-lookup */
 965           if (is_slow_path)
 966             {
 967               if (PREDICT_FALSE (proto0 == ~0))
 968                 {
 969                   if (nat_in2out_sm_unknown_proto
 970                       (sm, b0, ip0, rx_fib_index0))
 971                     {
 972                       next0 = SNAT_IN2OUT_NEXT_DROP;
 973                       b0->error =
 974                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 975                     }
 976                   other_packets++;
 977                   goto trace00;
 978                 }
 979
 980               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 981                 {
 982                   next0 = icmp_in2out_slow_path
 983                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 984                      node, next0, now, thread_index, &s0);
 985                   icmp_packets++;
 986                   goto trace00;
 987                 }
 988             }
 989           else
 990             {
 991               if (PREDICT_FALSE (proto0 == ~0))
 992                 {
 993                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 994                   goto trace00;
 995                 }
 996
 997               if (ip4_is_fragment (ip0))
 998                 {
 999                   next0 = SNAT_IN2OUT_NEXT_REASS;
1000                   fragments++;
1001                   goto trace00;
1002                 }
1003
1004               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1005                 {
1006                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1007                   goto trace00;
1008                 }
1009             }
1010
1011           key0.addr = ip0->src_address;
1012           key0.port = udp0->src_port;
1013           key0.protocol = proto0;
1014           key0.fib_index = rx_fib_index0;
1015
1016           kv0.key = key0.as_u64;
1017
1018           if (PREDICT_FALSE
1019               (clib_bihash_search_8_8
1020                (&sm->per_thread_data[thread_index].in2out, &kv0,
1021                 &value0) != 0))
1022             {
1023               if (is_slow_path)
1024                 {
1025                   if (is_output_feature)
1026                     {
1027                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1028                                                                            ip0,
1029                                                                            proto0,
1030                                                                            udp0->src_port,
1031                                                                            udp0->dst_port,
1032                                                                            thread_index,
1033                                                                            sw_if_index0)))
1034                         goto trace00;
1035                     }
1036                   else
1037                     {
1038                       if (PREDICT_FALSE
1039                           (snat_not_translate
1040                            (sm, node, sw_if_index0, ip0, proto0,
1041                             rx_fib_index0, thread_index)))
1042                         goto trace00;
1043                     }
1044
1045                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1046                                      &s0, node, next0, thread_index, now);
1047                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1048                     goto trace00;
1049
1050                   if (PREDICT_FALSE (!s0))
1051                     goto trace00;
1052                 }
1053               else
1054                 {
1055                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1056                   goto trace00;
1057                 }
1058             }
1059           else
1060             s0 =
1061               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1062                                  value0.value);
1063
1064           b0->flags |= VNET_BUFFER_F_IS_NATED;
1065
1066           old_addr0 = ip0->src_address.as_u32;
1067           ip0->src_address = s0->out2in.addr;
1068           new_addr0 = ip0->src_address.as_u32;
1069           if (!is_output_feature)
1070             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1071
1072           sum0 = ip0->checksum;
1073           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1074                                  ip4_header_t,
1075                                  src_address /* changed member */ );
1076           ip0->checksum = ip_csum_fold (sum0);
1077
1078           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1079             {
1080               old_port0 = tcp0->src_port;
1081               tcp0->src_port = s0->out2in.port;
1082               new_port0 = tcp0->src_port;
1083
1084               sum0 = tcp0->checksum;
1085               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1086                                      ip4_header_t,
1087                                      dst_address /* changed member */ );
1088               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1089                                      ip4_header_t /* cheat */ ,
1090                                      length /* changed member */ );
1091               mss_clamping (sm, tcp0, &sum0);
1092               tcp0->checksum = ip_csum_fold (sum0);
1093               tcp_packets++;
1094             }
1095           else
1096             {
1097               old_port0 = udp0->src_port;
1098               udp0->src_port = s0->out2in.port;
1099               udp0->checksum = 0;
1100               udp_packets++;
1101             }
1102
1103           /* Accounting */
1104           nat44_session_update_counters (s0, now,
1105                                          vlib_buffer_length_in_chain (vm, b0),
1106                                          thread_index);
1107           /* Per-user LRU list maintenance */
1108           nat44_session_update_lru (sm, s0, thread_index);
1109         trace00:
1110
1111           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1112                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1113             {
1114               snat_in2out_trace_t *t =
1115                 vlib_add_trace (vm, node, b0, sizeof (*t));
1116               t->is_slow_path = is_slow_path;
1117               t->sw_if_index = sw_if_index0;
1118               t->next_index = next0;
1119               t->session_index = ~0;
1120               if (s0)
1121                 t->session_index =
1122                   s0 - sm->per_thread_data[thread_index].sessions;
1123             }
1124
1125           pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1126
1127           if (is_output_feature)
1128             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1129
1130           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1131                                   iph_offset1);
1132
1133           udp1 = ip4_next_header (ip1);
1134           tcp1 = (tcp_header_t *) udp1;
1135           icmp1 = (icmp46_header_t *) udp1;
1136
1137           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1138           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1139                                    sw_if_index1);
1140
1141           if (PREDICT_FALSE (ip1->ttl == 1))
1142             {
1143               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1144               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1145                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1146                                            0);
1147               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1148               goto trace01;
1149             }
1150
1151           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1152
1153           /* Next configured feature, probably ip4-lookup */
1154           if (is_slow_path)
1155             {
1156               if (PREDICT_FALSE (proto1 == ~0))
1157                 {
1158                   if (nat_in2out_sm_unknown_proto
1159                       (sm, b1, ip1, rx_fib_index1))
1160                     {
1161                       next1 = SNAT_IN2OUT_NEXT_DROP;
1162                       b1->error =
1163                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1164                     }
1165                   other_packets++;
1166                   goto trace01;
1167                 }
1168
1169               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1170                 {
1171                   next1 = icmp_in2out_slow_path
1172                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1173                      next1, now, thread_index, &s1);
1174                   icmp_packets++;
1175                   goto trace01;
1176                 }
1177             }
1178           else
1179             {
1180               if (PREDICT_FALSE (proto1 == ~0))
1181                 {
1182                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1183                   goto trace01;
1184                 }
1185
1186               if (ip4_is_fragment (ip1))
1187                 {
1188                   next1 = SNAT_IN2OUT_NEXT_REASS;
1189                   fragments++;
1190                   goto trace01;
1191                 }
1192
1193               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1194                 {
1195                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1196                   goto trace01;
1197                 }
1198             }
1199
1200           key1.addr = ip1->src_address;
1201           key1.port = udp1->src_port;
1202           key1.protocol = proto1;
1203           key1.fib_index = rx_fib_index1;
1204
1205           kv1.key = key1.as_u64;
1206
1207           if (PREDICT_FALSE
1208               (clib_bihash_search_8_8
1209                (&sm->per_thread_data[thread_index].in2out, &kv1,
1210                 &value1) != 0))
1211             {
1212               if (is_slow_path)
1213                 {
1214                   if (is_output_feature)
1215                     {
1216                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1217                                                                            ip1,
1218                                                                            proto1,
1219                                                                            udp1->src_port,
1220                                                                            udp1->dst_port,
1221                                                                            thread_index,
1222                                                                            sw_if_index1)))
1223                         goto trace01;
1224                     }
1225                   else
1226                     {
1227                       if (PREDICT_FALSE
1228                           (snat_not_translate
1229                            (sm, node, sw_if_index1, ip1, proto1,
1230                             rx_fib_index1, thread_index)))
1231                         goto trace01;
1232                     }
1233
1234                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1235                                      &s1, node, next1, thread_index, now);
1236                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1237                     goto trace01;
1238
1239                   if (PREDICT_FALSE (!s1))
1240                     goto trace01;
1241                 }
1242               else
1243                 {
1244                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1245                   goto trace01;
1246                 }
1247             }
1248           else
1249             s1 =
1250               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1251                                  value1.value);
1252
1253           b1->flags |= VNET_BUFFER_F_IS_NATED;
1254
1255           old_addr1 = ip1->src_address.as_u32;
1256           ip1->src_address = s1->out2in.addr;
1257           new_addr1 = ip1->src_address.as_u32;
1258           if (!is_output_feature)
1259             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1260
1261           sum1 = ip1->checksum;
1262           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1263                                  ip4_header_t,
1264                                  src_address /* changed member */ );
1265           ip1->checksum = ip_csum_fold (sum1);
1266
1267           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1268             {
1269               old_port1 = tcp1->src_port;
1270               tcp1->src_port = s1->out2in.port;
1271               new_port1 = tcp1->src_port;
1272
1273               sum1 = tcp1->checksum;
1274               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1275                                      ip4_header_t,
1276                                      dst_address /* changed member */ );
1277               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1278                                      ip4_header_t /* cheat */ ,
1279                                      length /* changed member */ );
1280               mss_clamping (sm, tcp1, &sum1);
1281               tcp1->checksum = ip_csum_fold (sum1);
1282               tcp_packets++;
1283             }
1284           else
1285             {
1286               old_port1 = udp1->src_port;
1287               udp1->src_port = s1->out2in.port;
1288               udp1->checksum = 0;
1289               udp_packets++;
1290             }
1291
1292           /* Accounting */
1293           nat44_session_update_counters (s1, now,
1294                                          vlib_buffer_length_in_chain (vm, b1),
1295                                          thread_index);
1296           /* Per-user LRU list maintenance */
1297           nat44_session_update_lru (sm, s1, thread_index);
1298         trace01:
1299
1300           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1301                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1302             {
1303               snat_in2out_trace_t *t =
1304                 vlib_add_trace (vm, node, b1, sizeof (*t));
1305               t->sw_if_index = sw_if_index1;
1306               t->next_index = next1;
1307               t->session_index = ~0;
1308               if (s1)
1309                 t->session_index =
1310                   s1 - sm->per_thread_data[thread_index].sessions;
1311             }
1312
1313           pkts_processed += next1 == SNAT_IN2OUT_NEXT_LOOKUP;
1314
1315           /* verify speculative enqueues, maybe switch current next frame */
1316           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1317                                            to_next, n_left_to_next,
1318                                            bi0, bi1, next0, next1);
1319         }
1320
1321       while (n_left_from > 0 && n_left_to_next > 0)
1322         {
1323           u32 bi0;
1324           vlib_buffer_t *b0;
1325           u32 next0;
1326           u32 sw_if_index0;
1327           ip4_header_t *ip0;
1328           ip_csum_t sum0;
1329           u32 new_addr0, old_addr0;
1330           u16 old_port0, new_port0;
1331           udp_header_t *udp0;
1332           tcp_header_t *tcp0;
1333           icmp46_header_t *icmp0;
1334           snat_session_key_t key0;
1335           u32 rx_fib_index0;
1336           u32 proto0;
1337           snat_session_t *s0 = 0;
1338           clib_bihash_kv_8_8_t kv0, value0;
1339           u32 iph_offset0 = 0;
1340
1341           /* speculatively enqueue b0 to the current next frame */
1342           bi0 = from[0];
1343           to_next[0] = bi0;
1344           from += 1;
1345           to_next += 1;
1346           n_left_from -= 1;
1347           n_left_to_next -= 1;
1348
1349           b0 = vlib_get_buffer (vm, bi0);
1350           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1351
1352           if (is_output_feature)
1353             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1354
1355           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1356                                   iph_offset0);
1357
1358           udp0 = ip4_next_header (ip0);
1359           tcp0 = (tcp_header_t *) udp0;
1360           icmp0 = (icmp46_header_t *) udp0;
1361
1362           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1363           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1364                                    sw_if_index0);
1365
1366           if (PREDICT_FALSE (ip0->ttl == 1))
1367             {
1368               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1369               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1370                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1371                                            0);
1372               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1373               goto trace0;
1374             }
1375
1376           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1377
1378           /* Next configured feature, probably ip4-lookup */
1379           if (is_slow_path)
1380             {
1381               if (PREDICT_FALSE (proto0 == ~0))
1382                 {
1383                   if (nat_in2out_sm_unknown_proto
1384                       (sm, b0, ip0, rx_fib_index0))
1385                     {
1386                       next0 = SNAT_IN2OUT_NEXT_DROP;
1387                       b0->error =
1388                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1389                     }
1390                   other_packets++;
1391                   goto trace0;
1392                 }
1393
1394               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1395                 {
1396                   next0 = icmp_in2out_slow_path
1397                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1398                      next0, now, thread_index, &s0);
1399                   icmp_packets++;
1400                   goto trace0;
1401                 }
1402             }
1403           else
1404             {
1405               if (PREDICT_FALSE (proto0 == ~0))
1406                 {
1407                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1408                   goto trace0;
1409                 }
1410
1411               if (ip4_is_fragment (ip0))
1412                 {
1413                   next0 = SNAT_IN2OUT_NEXT_REASS;
1414                   fragments++;
1415                   goto trace0;
1416                 }
1417
1418               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1419                 {
1420                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1421                   goto trace0;
1422                 }
1423             }
1424
1425           key0.addr = ip0->src_address;
1426           key0.port = udp0->src_port;
1427           key0.protocol = proto0;
1428           key0.fib_index = rx_fib_index0;
1429
1430           kv0.key = key0.as_u64;
1431
1432           if (clib_bihash_search_8_8
1433               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1434             {
1435               if (is_slow_path)
1436                 {
1437                   if (is_output_feature)
1438                     {
1439                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1440                                                                            ip0,
1441                                                                            proto0,
1442                                                                            udp0->src_port,
1443                                                                            udp0->dst_port,
1444                                                                            thread_index,
1445                                                                            sw_if_index0)))
1446                         goto trace0;
1447                     }
1448                   else
1449                     {
1450                       if (PREDICT_FALSE
1451                           (snat_not_translate
1452                            (sm, node, sw_if_index0, ip0, proto0,
1453                             rx_fib_index0, thread_index)))
1454                         goto trace0;
1455                     }
1456
1457                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1458                                      &s0, node, next0, thread_index, now);
1459
1460                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1461                     goto trace0;
1462
1463                   if (PREDICT_FALSE (!s0))
1464                     goto trace0;
1465                 }
1466               else
1467                 {
1468                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1469                   goto trace0;
1470                 }
1471             }
1472           else
1473             s0 =
1474               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1475                                  value0.value);
1476
1477           b0->flags |= VNET_BUFFER_F_IS_NATED;
1478
1479           old_addr0 = ip0->src_address.as_u32;
1480           ip0->src_address = s0->out2in.addr;
1481           new_addr0 = ip0->src_address.as_u32;
1482           if (!is_output_feature)
1483             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1484
1485           sum0 = ip0->checksum;
1486           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1487                                  ip4_header_t,
1488                                  src_address /* changed member */ );
1489           ip0->checksum = ip_csum_fold (sum0);
1490
1491           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1492             {
1493               old_port0 = tcp0->src_port;
1494               tcp0->src_port = s0->out2in.port;
1495               new_port0 = tcp0->src_port;
1496
1497               sum0 = tcp0->checksum;
1498               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1499                                      ip4_header_t,
1500                                      dst_address /* changed member */ );
1501               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1502                                      ip4_header_t /* cheat */ ,
1503                                      length /* changed member */ );
1504               mss_clamping (sm, tcp0, &sum0);
1505               tcp0->checksum = ip_csum_fold (sum0);
1506               tcp_packets++;
1507             }
1508           else
1509             {
1510               old_port0 = udp0->src_port;
1511               udp0->src_port = s0->out2in.port;
1512               udp0->checksum = 0;
1513               udp_packets++;
1514             }
1515
1516           /* Accounting */
1517           nat44_session_update_counters (s0, now,
1518                                          vlib_buffer_length_in_chain (vm, b0),
1519                                          thread_index);
1520           /* Per-user LRU list maintenance */
1521           nat44_session_update_lru (sm, s0, thread_index);
1522
1523         trace0:
1524           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1525                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1526             {
1527               snat_in2out_trace_t *t =
1528                 vlib_add_trace (vm, node, b0, sizeof (*t));
1529               t->is_slow_path = is_slow_path;
1530               t->sw_if_index = sw_if_index0;
1531               t->next_index = next0;
1532               t->session_index = ~0;
1533               if (s0)
1534                 t->session_index =
1535                   s0 - sm->per_thread_data[thread_index].sessions;
1536             }
1537
1538           pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1539
1540           /* verify speculative enqueue, maybe switch current next frame */
1541           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1542                                            to_next, n_left_to_next,
1543                                            bi0, next0);
1544         }
1545
1546       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1547     }
1548
1549   vlib_node_increment_counter (vm, stats_node_index,
1550                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1551                                pkts_processed);
1552   vlib_node_increment_counter (vm, stats_node_index,
1553                                SNAT_IN2OUT_ERROR_TCP_PACKETS, tcp_packets);
1554   vlib_node_increment_counter (vm, stats_node_index,
1555                                SNAT_IN2OUT_ERROR_UDP_PACKETS, tcp_packets);
1556   vlib_node_increment_counter (vm, stats_node_index,
1557                                SNAT_IN2OUT_ERROR_ICMP_PACKETS, icmp_packets);
1558   vlib_node_increment_counter (vm, stats_node_index,
1559                                SNAT_IN2OUT_ERROR_OTHER_PACKETS,
1560                                other_packets);
1561   vlib_node_increment_counter (vm, stats_node_index,
1562                                SNAT_IN2OUT_ERROR_FRAGMENTS, fragments);
1563
1564   return frame->n_vectors;
1565 }
1566
1567 VLIB_NODE_FN (snat_in2out_node) (vlib_main_t * vm,
1568                                  vlib_node_runtime_t * node,
1569                                  vlib_frame_t * frame)
1570 {
1571   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1572                                      0);
1573 }
1574
1575 /* *INDENT-OFF* */
1576 VLIB_REGISTER_NODE (snat_in2out_node) = {
1577   .name = "nat44-in2out",
1578   .vector_size = sizeof (u32),
1579   .format_trace = format_snat_in2out_trace,
1580   .type = VLIB_NODE_TYPE_INTERNAL,
1581
1582   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1583   .error_strings = snat_in2out_error_strings,
1584
1585   .runtime_data_bytes = sizeof (snat_runtime_t),
1586
1587   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1588
1589   /* edit / add dispositions here */
1590   .next_nodes = {
1591     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1592     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1593     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1594     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1595     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1596   },
1597 };
1598 /* *INDENT-ON* */
1599
1600 VLIB_NODE_FN (snat_in2out_output_node) (vlib_main_t * vm,
1601                                         vlib_node_runtime_t * node,
1602                                         vlib_frame_t * frame)
1603 {
1604   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1605                                      1);
1606 }
1607
1608 /* *INDENT-OFF* */
1609 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1610   .name = "nat44-in2out-output",
1611   .vector_size = sizeof (u32),
1612   .format_trace = format_snat_in2out_trace,
1613   .type = VLIB_NODE_TYPE_INTERNAL,
1614
1615   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1616   .error_strings = snat_in2out_error_strings,
1617
1618   .runtime_data_bytes = sizeof (snat_runtime_t),
1619
1620   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1621
1622   /* edit / add dispositions here */
1623   .next_nodes = {
1624     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1625     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1626     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1627     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1628     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1629   },
1630 };
1631 /* *INDENT-ON* */
1632
1633 VLIB_NODE_FN (snat_in2out_slowpath_node) (vlib_main_t * vm,
1634                                           vlib_node_runtime_t * node,
1635                                           vlib_frame_t * frame)
1636 {
1637   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1638                                      0);
1639 }
1640
1641 /* *INDENT-OFF* */
1642 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1643   .name = "nat44-in2out-slowpath",
1644   .vector_size = sizeof (u32),
1645   .format_trace = format_snat_in2out_trace,
1646   .type = VLIB_NODE_TYPE_INTERNAL,
1647
1648   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1649   .error_strings = snat_in2out_error_strings,
1650
1651   .runtime_data_bytes = sizeof (snat_runtime_t),
1652
1653   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1654
1655   /* edit / add dispositions here */
1656   .next_nodes = {
1657     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1658     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1659     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1660     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1661     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1662   },
1663 };
1664 /* *INDENT-ON* */
1665
1666 VLIB_NODE_FN (snat_in2out_output_slowpath_node) (vlib_main_t * vm,
1667                                                  vlib_node_runtime_t * node,
1668                                                  vlib_frame_t * frame)
1669 {
1670   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1671                                      1);
1672 }
1673
1674 /* *INDENT-OFF* */
1675 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1676   .name = "nat44-in2out-output-slowpath",
1677   .vector_size = sizeof (u32),
1678   .format_trace = format_snat_in2out_trace,
1679   .type = VLIB_NODE_TYPE_INTERNAL,
1680
1681   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1682   .error_strings = snat_in2out_error_strings,
1683
1684   .runtime_data_bytes = sizeof (snat_runtime_t),
1685
1686   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1687
1688   /* edit / add dispositions here */
1689   .next_nodes = {
1690     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1691     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1692     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1693     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1694     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1695   },
1696 };
1697 /* *INDENT-ON* */
1698
1699 VLIB_NODE_FN (nat44_in2out_reass_node) (vlib_main_t * vm,
1700                                         vlib_node_runtime_t * node,
1701                                         vlib_frame_t * frame)
1702 {
1703   u32 n_left_from, *from, *to_next;
1704   snat_in2out_next_t next_index;
1705   u32 pkts_processed = 0, cached_fragments = 0;
1706   snat_main_t *sm = &snat_main;
1707   f64 now = vlib_time_now (vm);
1708   u32 thread_index = vm->thread_index;
1709   snat_main_per_thread_data_t *per_thread_data =
1710     &sm->per_thread_data[thread_index];
1711   u32 *fragments_to_drop = 0;
1712   u32 *fragments_to_loopback = 0;
1713
1714   from = vlib_frame_vector_args (frame);
1715   n_left_from = frame->n_vectors;
1716   next_index = node->cached_next_index;
1717
1718   while (n_left_from > 0)
1719     {
1720       u32 n_left_to_next;
1721
1722       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1723
1724       while (n_left_from > 0 && n_left_to_next > 0)
1725         {
1726           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1727           vlib_buffer_t *b0;
1728           u32 next0;
1729           u8 cached0 = 0;
1730           ip4_header_t *ip0;
1731           nat_reass_ip4_t *reass0;
1732           udp_header_t *udp0;
1733           tcp_header_t *tcp0;
1734           icmp46_header_t *icmp0;
1735           snat_session_key_t key0;
1736           clib_bihash_kv_8_8_t kv0, value0;
1737           snat_session_t *s0 = 0;
1738           u16 old_port0, new_port0;
1739           ip_csum_t sum0;
1740
1741           /* speculatively enqueue b0 to the current next frame */
1742           bi0 = from[0];
1743           to_next[0] = bi0;
1744           from += 1;
1745           to_next += 1;
1746           n_left_from -= 1;
1747           n_left_to_next -= 1;
1748
1749           b0 = vlib_get_buffer (vm, bi0);
1750           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1751
1752           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1753           rx_fib_index0 =
1754             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1755                                                  sw_if_index0);
1756
1757           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1758             {
1759               next0 = SNAT_IN2OUT_NEXT_DROP;
1760               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1761               goto trace0;
1762             }
1763
1764           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1765           udp0 = ip4_next_header (ip0);
1766           tcp0 = (tcp_header_t *) udp0;
1767           icmp0 = (icmp46_header_t *) udp0;
1768           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1769
1770           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1771                                                  ip0->dst_address,
1772                                                  ip0->fragment_id,
1773                                                  ip0->protocol,
1774                                                  1, &fragments_to_drop);
1775
1776           if (PREDICT_FALSE (!reass0))
1777             {
1778               next0 = SNAT_IN2OUT_NEXT_DROP;
1779               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1780               nat_log_notice ("maximum reassemblies exceeded");
1781               goto trace0;
1782             }
1783
1784           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1785             {
1786               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1787                 {
1788                   next0 = icmp_in2out_slow_path
1789                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1790                      next0, now, thread_index, &s0);
1791
1792                   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP))
1793                     {
1794                       if (s0)
1795                         reass0->sess_index = s0 - per_thread_data->sessions;
1796                       else
1797                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1798                       nat_ip4_reass_get_frags (reass0,
1799                                                &fragments_to_loopback);
1800                     }
1801
1802                   goto trace0;
1803                 }
1804
1805               key0.addr = ip0->src_address;
1806               key0.port = udp0->src_port;
1807               key0.protocol = proto0;
1808               key0.fib_index = rx_fib_index0;
1809               kv0.key = key0.as_u64;
1810
1811               if (clib_bihash_search_8_8
1812                   (&per_thread_data->in2out, &kv0, &value0))
1813                 {
1814                   if (PREDICT_FALSE
1815                       (snat_not_translate
1816                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1817                         thread_index)))
1818                     goto trace0;
1819
1820                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1821                                      &s0, node, next0, thread_index, now);
1822
1823                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1824                     goto trace0;
1825
1826                   if (PREDICT_FALSE (!s0))
1827                     goto trace0;
1828
1829                   reass0->sess_index = s0 - per_thread_data->sessions;
1830                 }
1831               else
1832                 {
1833                   s0 = pool_elt_at_index (per_thread_data->sessions,
1834                                           value0.value);
1835                   reass0->sess_index = value0.value;
1836                 }
1837               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1838             }
1839           else
1840             {
1841               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1842                 {
1843                   if (nat_ip4_reass_add_fragment
1844                       (thread_index, reass0, bi0, &fragments_to_drop))
1845                     {
1846                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1847                       nat_log_notice
1848                         ("maximum fragments per reassembly exceeded");
1849                       next0 = SNAT_IN2OUT_NEXT_DROP;
1850                       goto trace0;
1851                     }
1852                   cached0 = 1;
1853                   goto trace0;
1854                 }
1855               s0 = pool_elt_at_index (per_thread_data->sessions,
1856                                       reass0->sess_index);
1857             }
1858
1859           old_addr0 = ip0->src_address.as_u32;
1860           ip0->src_address = s0->out2in.addr;
1861           new_addr0 = ip0->src_address.as_u32;
1862           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1863
1864           sum0 = ip0->checksum;
1865           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1866                                  ip4_header_t,
1867                                  src_address /* changed member */ );
1868           ip0->checksum = ip_csum_fold (sum0);
1869
1870           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1871             {
1872               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1873                 {
1874                   old_port0 = tcp0->src_port;
1875                   tcp0->src_port = s0->out2in.port;
1876                   new_port0 = tcp0->src_port;
1877
1878                   sum0 = tcp0->checksum;
1879                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1880                                          ip4_header_t,
1881                                          dst_address /* changed member */ );
1882                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1883                                          ip4_header_t /* cheat */ ,
1884                                          length /* changed member */ );
1885                   tcp0->checksum = ip_csum_fold (sum0);
1886                 }
1887               else
1888                 {
1889                   old_port0 = udp0->src_port;
1890                   udp0->src_port = s0->out2in.port;
1891                   udp0->checksum = 0;
1892                 }
1893             }
1894
1895           /* Hairpinning */
1896           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1897                                    s0->ext_host_port, proto0, 0);
1898
1899           /* Accounting */
1900           nat44_session_update_counters (s0, now,
1901                                          vlib_buffer_length_in_chain (vm, b0),
1902                                          thread_index);
1903           /* Per-user LRU list maintenance */
1904           nat44_session_update_lru (sm, s0, thread_index);
1905
1906         trace0:
1907           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1908                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1909             {
1910               nat44_reass_trace_t *t =
1911                 vlib_add_trace (vm, node, b0, sizeof (*t));
1912               t->cached = cached0;
1913               t->sw_if_index = sw_if_index0;
1914               t->next_index = next0;
1915             }
1916
1917           if (cached0)
1918             {
1919               n_left_to_next++;
1920               to_next--;
1921               cached_fragments++;
1922             }
1923           else
1924             {
1925               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1926
1927               /* verify speculative enqueue, maybe switch current next frame */
1928               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1929                                                to_next, n_left_to_next,
1930                                                bi0, next0);
1931             }
1932
1933           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1934             {
1935               from = vlib_frame_vector_args (frame);
1936               u32 len = vec_len (fragments_to_loopback);
1937               if (len <= VLIB_FRAME_SIZE)
1938                 {
1939                   clib_memcpy_fast (from, fragments_to_loopback,
1940                                     sizeof (u32) * len);
1941                   n_left_from = len;
1942                   vec_reset_length (fragments_to_loopback);
1943                 }
1944               else
1945                 {
1946                   clib_memcpy_fast (from, fragments_to_loopback +
1947                                     (len - VLIB_FRAME_SIZE),
1948                                     sizeof (u32) * VLIB_FRAME_SIZE);
1949                   n_left_from = VLIB_FRAME_SIZE;
1950                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1951                 }
1952             }
1953         }
1954
1955       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1956     }
1957
1958   vlib_node_increment_counter (vm, sm->in2out_reass_node_index,
1959                                SNAT_IN2OUT_ERROR_PROCESSED_FRAGMENTS,
1960                                pkts_processed);
1961   vlib_node_increment_counter (vm, sm->in2out_reass_node_index,
1962                                SNAT_IN2OUT_ERROR_CACHED_FRAGMENTS,
1963                                cached_fragments);
1964
1965   nat_send_all_to_node (vm, fragments_to_drop, node,
1966                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1967                         SNAT_IN2OUT_NEXT_DROP);
1968
1969   vec_free (fragments_to_drop);
1970   vec_free (fragments_to_loopback);
1971   return frame->n_vectors;
1972 }
1973
1974 /* *INDENT-OFF* */
1975 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
1976   .name = "nat44-in2out-reass",
1977   .vector_size = sizeof (u32),
1978   .format_trace = format_nat44_reass_trace,
1979   .type = VLIB_NODE_TYPE_INTERNAL,
1980
1981   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1982   .error_strings = snat_in2out_error_strings,
1983
1984   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1985   .next_nodes = {
1986     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1987     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1988     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1989     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1990     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1991   },
1992 };
1993 /* *INDENT-ON* */
1994
1995 VLIB_NODE_FN (snat_in2out_fast_node) (vlib_main_t * vm,
1996                                       vlib_node_runtime_t * node,
1997                                       vlib_frame_t * frame)
1998 {
1999   u32 n_left_from, *from, *to_next;
2000   snat_in2out_next_t next_index;
2001   u32 pkts_processed = 0;
2002   snat_main_t *sm = &snat_main;
2003   u32 stats_node_index;
2004
2005   stats_node_index = sm->in2out_fast_node_index;
2006
2007   from = vlib_frame_vector_args (frame);
2008   n_left_from = frame->n_vectors;
2009   next_index = node->cached_next_index;
2010
2011   while (n_left_from > 0)
2012     {
2013       u32 n_left_to_next;
2014
2015       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2016
2017       while (n_left_from > 0 && n_left_to_next > 0)
2018         {
2019           u32 bi0;
2020           vlib_buffer_t *b0;
2021           u32 next0;
2022           u32 sw_if_index0;
2023           ip4_header_t *ip0;
2024           ip_csum_t sum0;
2025           u32 new_addr0, old_addr0;
2026           u16 old_port0, new_port0;
2027           udp_header_t *udp0;
2028           tcp_header_t *tcp0;
2029           icmp46_header_t *icmp0;
2030           snat_session_key_t key0, sm0;
2031           u32 proto0;
2032           u32 rx_fib_index0;
2033
2034           /* speculatively enqueue b0 to the current next frame */
2035           bi0 = from[0];
2036           to_next[0] = bi0;
2037           from += 1;
2038           to_next += 1;
2039           n_left_from -= 1;
2040           n_left_to_next -= 1;
2041
2042           b0 = vlib_get_buffer (vm, bi0);
2043           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2044
2045           ip0 = vlib_buffer_get_current (b0);
2046           udp0 = ip4_next_header (ip0);
2047           tcp0 = (tcp_header_t *) udp0;
2048           icmp0 = (icmp46_header_t *) udp0;
2049
2050           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2051           rx_fib_index0 =
2052             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2053
2054           if (PREDICT_FALSE (ip0->ttl == 1))
2055             {
2056               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2057               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2058                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2059                                            0);
2060               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2061               goto trace0;
2062             }
2063
2064           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2065
2066           if (PREDICT_FALSE (proto0 == ~0))
2067             goto trace0;
2068
2069           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2070             {
2071               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
2072                                    rx_fib_index0, node, next0, ~0, 0, 0);
2073               goto trace0;
2074             }
2075
2076           key0.addr = ip0->src_address;
2077           key0.protocol = proto0;
2078           key0.port = udp0->src_port;
2079           key0.fib_index = rx_fib_index0;
2080
2081           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0, 0))
2082             {
2083               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2084               next0 = SNAT_IN2OUT_NEXT_DROP;
2085               goto trace0;
2086             }
2087
2088           new_addr0 = sm0.addr.as_u32;
2089           new_port0 = sm0.port;
2090           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2091           old_addr0 = ip0->src_address.as_u32;
2092           ip0->src_address.as_u32 = new_addr0;
2093
2094           sum0 = ip0->checksum;
2095           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2096                                  ip4_header_t,
2097                                  src_address /* changed member */ );
2098           ip0->checksum = ip_csum_fold (sum0);
2099
2100           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
2101             {
2102               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2103                 {
2104                   old_port0 = tcp0->src_port;
2105                   tcp0->src_port = new_port0;
2106
2107                   sum0 = tcp0->checksum;
2108                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2109                                          ip4_header_t,
2110                                          dst_address /* changed member */ );
2111                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2112                                          ip4_header_t /* cheat */ ,
2113                                          length /* changed member */ );
2114                   mss_clamping (sm, tcp0, &sum0);
2115                   tcp0->checksum = ip_csum_fold (sum0);
2116                 }
2117               else
2118                 {
2119                   old_port0 = udp0->src_port;
2120                   udp0->src_port = new_port0;
2121                   udp0->checksum = 0;
2122                 }
2123             }
2124           else
2125             {
2126               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2127                 {
2128                   sum0 = tcp0->checksum;
2129                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2130                                          ip4_header_t,
2131                                          dst_address /* changed member */ );
2132                   mss_clamping (sm, tcp0, &sum0);
2133                   tcp0->checksum = ip_csum_fold (sum0);
2134                 }
2135             }
2136
2137           /* Hairpinning */
2138           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2139
2140         trace0:
2141           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2142                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2143             {
2144               snat_in2out_trace_t *t =
2145                 vlib_add_trace (vm, node, b0, sizeof (*t));
2146               t->sw_if_index = sw_if_index0;
2147               t->next_index = next0;
2148             }
2149
2150           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2151
2152           /* verify speculative enqueue, maybe switch current next frame */
2153           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2154                                            to_next, n_left_to_next,
2155                                            bi0, next0);
2156         }
2157
2158       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2159     }
2160
2161   vlib_node_increment_counter (vm, stats_node_index,
2162                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2163                                pkts_processed);
2164   return frame->n_vectors;
2165 }
2166
2167
2168 /* *INDENT-OFF* */
2169 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2170   .name = "nat44-in2out-fast",
2171   .vector_size = sizeof (u32),
2172   .format_trace = format_snat_in2out_fast_trace,
2173   .type = VLIB_NODE_TYPE_INTERNAL,
2174
2175   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2176   .error_strings = snat_in2out_error_strings,
2177
2178   .runtime_data_bytes = sizeof (snat_runtime_t),
2179
2180   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2181
2182   /* edit / add dispositions here */
2183   .next_nodes = {
2184     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2185     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2186     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2187     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2188     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2189   },
2190 };
2191 /* *INDENT-ON* */
2192
2193 /*
2194  * fd.io coding-style-patch-verification: ON
2195  *
2196  * Local Variables:
2197  * eval: (c-set-style "gnu")
2198  * End:
2199  */