src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/ethernet/ethernet.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <nat/nat.h>
  28 #include <nat/nat_ipfix_logging.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_syslog.h>
  32 #include <nat/nat_ha.h>
  33
  34 #include <vppinfra/hash.h>
  35 #include <vppinfra/error.h>
  36 #include <vppinfra/elog.h>
  37
  38 typedef struct
  39 {
  40   u32 sw_if_index;
  41   u32 next_index;
  42   u32 session_index;
  43   u32 is_slow_path;
  44 } snat_in2out_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 *
  48 format_snat_in2out_trace (u8 * s, va_list * args)
  49 {
  50   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  51   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  52   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  53   char *tag;
  54
  55   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  56
  57   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  58               t->sw_if_index, t->next_index, t->session_index);
  59
  60   return s;
  61 }
  62
  63 static u8 *
  64 format_snat_in2out_fast_trace (u8 * s, va_list * args)
  65 {
  66   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  67   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  68   snat_in2out_trace_t *t = va_arg (*args, snat_in2out_trace_t *);
  69
  70   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  71               t->sw_if_index, t->next_index);
  72
  73   return s;
  74 }
  75
  76 #define foreach_snat_in2out_error                       \
  77 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  78 _(IN2OUT_PACKETS, "good in2out packets processed")      \
  79 _(OUT_OF_PORTS, "out of ports")                         \
  80 _(BAD_OUTSIDE_FIB, "outside VRF ID not found")          \
  81 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  82 _(NO_TRANSLATION, "no translation")                     \
  83 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  84 _(DROP_FRAGMENT, "drop fragment")                       \
  85 _(MAX_REASS, "maximum reassemblies exceeded")           \
  86 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  87 _(TCP_PACKETS, "TCP packets")                           \
  88 _(UDP_PACKETS, "UDP packets")                           \
  89 _(ICMP_PACKETS, "ICMP packets")                         \
  90 _(OTHER_PACKETS, "other protocol packets")              \
  91 _(FRAGMENTS, "fragments")                               \
  92 _(CACHED_FRAGMENTS, "cached fragments")                 \
  93 _(PROCESSED_FRAGMENTS, "processed fragments")
  94
  95 typedef enum
  96 {
  97 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
  98   foreach_snat_in2out_error
  99 #undef _
 100     SNAT_IN2OUT_N_ERROR,
 101 } snat_in2out_error_t;
 102
 103 static char *snat_in2out_error_strings[] = {
 104 #define _(sym,string) string,
 105   foreach_snat_in2out_error
 106 #undef _
 107 };
 108
 109 typedef enum
 110 {
 111   SNAT_IN2OUT_NEXT_LOOKUP,
 112   SNAT_IN2OUT_NEXT_DROP,
 113   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 114   SNAT_IN2OUT_NEXT_SLOW_PATH,
 115   SNAT_IN2OUT_NEXT_REASS,
 116   SNAT_IN2OUT_N_NEXT,
 117 } snat_in2out_next_t;
 118
 119 static inline int
 120 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 121                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 122                     u32 rx_fib_index0, u32 thread_index)
 123 {
 124   udp_header_t *udp0 = ip4_next_header (ip0);
 125   snat_session_key_t key0, sm0;
 126   clib_bihash_kv_8_8_t kv0, value0;
 127
 128   key0.addr = ip0->dst_address;
 129   key0.port = udp0->dst_port;
 130   key0.protocol = proto0;
 131   key0.fib_index = sm->outside_fib_index;
 132   kv0.key = key0.as_u64;
 133
 134   /* NAT packet aimed at external address if */
 135   /* has active sessions */
 136   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 137                               &value0))
 138     {
 139       /* or is static mappings */
 140       if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 141         return 0;
 142     }
 143   else
 144     return 0;
 145
 146   if (sm->forwarding_enabled)
 147     return 1;
 148
 149   return snat_not_translate_fast (sm, node, sw_if_index0, ip0, proto0,
 150                                   rx_fib_index0);
 151 }
 152
 153 static inline int
 154 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 155                                   u32 proto0, u16 src_port, u16 dst_port,
 156                                   u32 thread_index, u32 sw_if_index)
 157 {
 158   snat_session_key_t key0;
 159   clib_bihash_kv_8_8_t kv0, value0;
 160   snat_interface_t *i;
 161
 162   /* src NAT check */
 163   key0.addr = ip0->src_address;
 164   key0.port = src_port;
 165   key0.protocol = proto0;
 166   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 167   kv0.key = key0.as_u64;
 168
 169   if (!clib_bihash_search_8_8
 170       (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 171     return 1;
 172
 173   /* dst NAT check */
 174   key0.addr = ip0->dst_address;
 175   key0.port = dst_port;
 176   key0.protocol = proto0;
 177   kv0.key = key0.as_u64;
 178   if (!clib_bihash_search_8_8
 179       (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
 180     {
 181       /* hairpinning */
 182     /* *INDENT-OFF* */
 183     pool_foreach (i, sm->output_feature_interfaces,
 184     ({
 185       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 186         return 0;
 187     }));
 188     /* *INDENT-ON* */
 189       return 1;
 190     }
 191
 192   return 0;
 193 }
 194
 195 #ifndef CLIB_MARCH_VARIANT
 196 int
 197 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 198 {
 199   snat_main_t *sm = &snat_main;
 200   nat44_is_idle_session_ctx_t *ctx = arg;
 201   snat_session_t *s;
 202   u64 sess_timeout_time;
 203   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 204                                                        ctx->thread_index);
 205   clib_bihash_kv_8_8_t s_kv;
 206
 207   s = pool_elt_at_index (tsm->sessions, kv->value);
 208   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 209   if (ctx->now >= sess_timeout_time)
 210     {
 211       s_kv.key = s->out2in.as_u64;
 212       if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
 213         nat_elog_warn ("out2in key del failed");
 214
 215       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 216                                            s->in2out.addr.as_u32,
 217                                            s->out2in.addr.as_u32,
 218                                            s->in2out.protocol,
 219                                            s->in2out.port,
 220                                            s->out2in.port,
 221                                            s->in2out.fib_index);
 222
 223       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 224                                &s->in2out.addr, s->in2out.port,
 225                                &s->out2in.addr, s->out2in.port,
 226                                s->in2out.protocol);
 227
 228       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 229                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 230                    ctx->thread_index);
 231
 232       if (!snat_is_session_static (s))
 233         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 234                                             &s->out2in);
 235
 236       nat44_delete_session (sm, s, ctx->thread_index);
 237       return 1;
 238     }
 239
 240   return 0;
 241 }
 242 #endif
 243
 244 static u32
 245 slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 246            ip4_header_t * ip0,
 247            u32 rx_fib_index0,
 248            snat_session_key_t * key0,
 249            snat_session_t ** sessionp,
 250            vlib_node_runtime_t * node, u32 next0, u32 thread_index, f64 now)
 251 {
 252   snat_user_t *u;
 253   snat_session_t *s = 0;
 254   clib_bihash_kv_8_8_t kv0;
 255   snat_session_key_t key1;
 256   udp_header_t *udp0 = ip4_next_header (ip0);
 257   u8 is_sm = 0;
 258   nat_outside_fib_t *outside_fib;
 259   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 260   u8 identity_nat;
 261   fib_prefix_t pfx = {
 262     .fp_proto = FIB_PROTOCOL_IP4,
 263     .fp_len = 32,
 264     .fp_addr = {
 265                 .ip4.as_u32 = ip0->dst_address.as_u32,
 266                 },
 267   };
 268   nat44_is_idle_session_ctx_t ctx0;
 269
 270   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
 271     {
 272       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 273       nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 274       nat_elog_notice ("maximum sessions exceeded");
 275       return SNAT_IN2OUT_NEXT_DROP;
 276     }
 277
 278   key1.protocol = key0->protocol;
 279
 280   /* First try to match static mapping by local address and port */
 281   if (snat_static_mapping_match
 282       (sm, *key0, &key1, 0, 0, 0, 0, 0, &identity_nat))
 283     {
 284       /* Try to create dynamic translation */
 285       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 286                                                thread_index, &key1,
 287                                                sm->port_per_thread,
 288                                                sm->per_thread_data
 289                                                [thread_index].snat_thread_index))
 290         {
 291           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 292           return SNAT_IN2OUT_NEXT_DROP;
 293         }
 294     }
 295   else
 296     {
 297       if (PREDICT_FALSE (identity_nat))
 298         {
 299           *sessionp = s;
 300           return next0;
 301         }
 302
 303       is_sm = 1;
 304     }
 305
 306   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 307                               thread_index);
 308   if (!u)
 309     {
 310       nat_elog_warn ("create NAT user failed");
 311       return SNAT_IN2OUT_NEXT_DROP;
 312     }
 313
 314   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 315   if (!s)
 316     {
 317       nat44_delete_user_with_no_session (sm, u, thread_index);
 318       nat_elog_warn ("create NAT session failed");
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   if (is_sm)
 323     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 324   user_session_increment (sm, u, is_sm);
 325   s->in2out = *key0;
 326   s->out2in = key1;
 327   s->out2in.protocol = key0->protocol;
 328   s->out2in.fib_index = sm->outside_fib_index;
 329   switch (vec_len (sm->outside_fibs))
 330     {
 331     case 0:
 332       s->out2in.fib_index = sm->outside_fib_index;
 333       break;
 334     case 1:
 335       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 336       break;
 337     default:
 338       /* *INDENT-OFF* */
 339       vec_foreach (outside_fib, sm->outside_fibs)
 340         {
 341           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 342           if (FIB_NODE_INDEX_INVALID != fei)
 343             {
 344               if (fib_entry_get_resolving_interface (fei) != ~0)
 345                 {
 346                   s->out2in.fib_index = outside_fib->fib_index;
 347                   break;
 348                 }
 349             }
 350         }
 351       /* *INDENT-ON* */
 352       break;
 353     }
 354   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 355   s->ext_host_port = udp0->dst_port;
 356   *sessionp = s;
 357
 358   /* Add to translation hashes */
 359   ctx0.now = now;
 360   ctx0.thread_index = thread_index;
 361   kv0.key = s->in2out.as_u64;
 362   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 363   if (clib_bihash_add_or_overwrite_stale_8_8
 364       (&sm->per_thread_data[thread_index].in2out, &kv0,
 365        nat44_i2o_is_idle_session_cb, &ctx0))
 366     nat_elog_notice ("in2out key add failed");
 367
 368   kv0.key = s->out2in.as_u64;
 369   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 370
 371   if (clib_bihash_add_or_overwrite_stale_8_8
 372       (&sm->per_thread_data[thread_index].out2in, &kv0,
 373        nat44_o2i_is_idle_session_cb, &ctx0))
 374     nat_elog_notice ("out2in key add failed");
 375
 376   /* log NAT event */
 377   snat_ipfix_logging_nat44_ses_create (thread_index,
 378                                        s->in2out.addr.as_u32,
 379                                        s->out2in.addr.as_u32,
 380                                        s->in2out.protocol,
 381                                        s->in2out.port,
 382                                        s->out2in.port, s->in2out.fib_index);
 383
 384   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 385                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 386                            s->out2in.port, s->in2out.protocol);
 387
 388   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 389                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 390                &s->ext_host_nat_addr, s->ext_host_nat_port,
 391                s->in2out.protocol, s->in2out.fib_index, s->flags,
 392                thread_index, 0);
 393
 394   return next0;
 395 }
 396
 397 #ifndef CLIB_MARCH_VARIANT
 398 static_always_inline
 399   snat_in2out_error_t icmp_get_key (ip4_header_t * ip0,
 400                                     snat_session_key_t * p_key0)
 401 {
 402   icmp46_header_t *icmp0;
 403   snat_session_key_t key0;
 404   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 405   ip4_header_t *inner_ip0 = 0;
 406   void *l4_header = 0;
 407   icmp46_header_t *inner_icmp0;
 408
 409   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 410   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 411
 412   if (!icmp_is_error_message (icmp0))
 413     {
 414       key0.protocol = SNAT_PROTOCOL_ICMP;
 415       key0.addr = ip0->src_address;
 416       key0.port = echo0->identifier;
 417     }
 418   else
 419     {
 420       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 421       l4_header = ip4_next_header (inner_ip0);
 422       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 423       key0.addr = inner_ip0->dst_address;
 424       switch (key0.protocol)
 425         {
 426         case SNAT_PROTOCOL_ICMP:
 427           inner_icmp0 = (icmp46_header_t *) l4_header;
 428           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 429           key0.port = inner_echo0->identifier;
 430           break;
 431         case SNAT_PROTOCOL_UDP:
 432         case SNAT_PROTOCOL_TCP:
 433           key0.port = ((tcp_udp_header_t *) l4_header)->dst_port;
 434           break;
 435         default:
 436           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 437         }
 438     }
 439   *p_key0 = key0;
 440   return -1;                    /* success */
 441 }
 442
 443 /**
 444  * Get address and port values to be used for ICMP packet translation
 445  * and create session if needed
 446  *
 447  * @param[in,out] sm             NAT main
 448  * @param[in,out] node           NAT node runtime
 449  * @param[in] thread_index       thread index
 450  * @param[in,out] b0             buffer containing packet to be translated
 451  * @param[out] p_proto           protocol used for matching
 452  * @param[out] p_value           address and port after NAT translation
 453  * @param[out] p_dont_translate  if packet should not be translated
 454  * @param d                      optional parameter
 455  * @param e                      optional parameter
 456  */
 457 u32
 458 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 459                         u32 thread_index, vlib_buffer_t * b0,
 460                         ip4_header_t * ip0, u8 * p_proto,
 461                         snat_session_key_t * p_value,
 462                         u8 * p_dont_translate, void *d, void *e)
 463 {
 464   icmp46_header_t *icmp0;
 465   u32 sw_if_index0;
 466   u32 rx_fib_index0;
 467   snat_session_key_t key0;
 468   snat_session_t *s0 = 0;
 469   u8 dont_translate = 0;
 470   clib_bihash_kv_8_8_t kv0, value0;
 471   u32 next0 = ~0;
 472   int err;
 473
 474   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 475   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 476   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 477
 478   err = icmp_get_key (ip0, &key0);
 479   if (err != -1)
 480     {
 481       b0->error = node->errors[err];
 482       next0 = SNAT_IN2OUT_NEXT_DROP;
 483       goto out;
 484     }
 485   key0.fib_index = rx_fib_index0;
 486
 487   kv0.key = key0.as_u64;
 488
 489   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 490                               &value0))
 491     {
 492       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 493         {
 494           if (PREDICT_FALSE (nat_not_translate_output_feature (sm, ip0,
 495                                                                key0.protocol,
 496                                                                key0.port,
 497                                                                key0.port,
 498                                                                thread_index,
 499                                                                sw_if_index0)))
 500             {
 501               dont_translate = 1;
 502               goto out;
 503             }
 504         }
 505       else
 506         {
 507           if (PREDICT_FALSE (snat_not_translate (sm, node, sw_if_index0,
 508                                                  ip0, SNAT_PROTOCOL_ICMP,
 509                                                  rx_fib_index0,
 510                                                  thread_index)))
 511             {
 512               dont_translate = 1;
 513               goto out;
 514             }
 515         }
 516
 517       if (PREDICT_FALSE (icmp_is_error_message (icmp0)))
 518         {
 519           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 520           next0 = SNAT_IN2OUT_NEXT_DROP;
 521           goto out;
 522         }
 523
 524       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
 525                          thread_index, vlib_time_now (sm->vlib_main));
 526
 527       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 528         goto out;
 529
 530       if (!s0)
 531         {
 532           dont_translate = 1;
 533           goto out;
 534         }
 535     }
 536   else
 537     {
 538       if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 539                          icmp0->type != ICMP4_echo_reply &&
 540                          !icmp_is_error_message (icmp0)))
 541         {
 542           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 543           next0 = SNAT_IN2OUT_NEXT_DROP;
 544           goto out;
 545         }
 546
 547       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 548                               value0.value);
 549     }
 550
 551 out:
 552   *p_proto = key0.protocol;
 553   if (s0)
 554     *p_value = s0->out2in;
 555   *p_dont_translate = dont_translate;
 556   if (d)
 557     *(snat_session_t **) d = s0;
 558   return next0;
 559 }
 560 #endif
 561
 562 #ifndef CLIB_MARCH_VARIANT
 563 /**
 564  * Get address and port values to be used for ICMP packet translation
 565  *
 566  * @param[in] sm                 NAT main
 567  * @param[in,out] node           NAT node runtime
 568  * @param[in] thread_index       thread index
 569  * @param[in,out] b0             buffer containing packet to be translated
 570  * @param[out] p_proto           protocol used for matching
 571  * @param[out] p_value           address and port after NAT translation
 572  * @param[out] p_dont_translate  if packet should not be translated
 573  * @param d                      optional parameter
 574  * @param e                      optional parameter
 575  */
 576 u32
 577 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 578                         u32 thread_index, vlib_buffer_t * b0,
 579                         ip4_header_t * ip0, u8 * p_proto,
 580                         snat_session_key_t * p_value,
 581                         u8 * p_dont_translate, void *d, void *e)
 582 {
 583   icmp46_header_t *icmp0;
 584   u32 sw_if_index0;
 585   u32 rx_fib_index0;
 586   snat_session_key_t key0;
 587   snat_session_key_t sm0;
 588   u8 dont_translate = 0;
 589   u8 is_addr_only;
 590   u32 next0 = ~0;
 591   int err;
 592
 593   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 594   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 595   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 596
 597   err = icmp_get_key (ip0, &key0);
 598   if (err != -1)
 599     {
 600       b0->error = node->errors[err];
 601       next0 = SNAT_IN2OUT_NEXT_DROP;
 602       goto out2;
 603     }
 604   key0.fib_index = rx_fib_index0;
 605
 606   if (snat_static_mapping_match
 607       (sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0, 0))
 608     {
 609       if (PREDICT_FALSE (snat_not_translate_fast (sm, node, sw_if_index0, ip0,
 610                                                   IP_PROTOCOL_ICMP,
 611                                                   rx_fib_index0)))
 612         {
 613           dont_translate = 1;
 614           goto out;
 615         }
 616
 617       if (icmp_is_error_message (icmp0))
 618         {
 619           next0 = SNAT_IN2OUT_NEXT_DROP;
 620           goto out;
 621         }
 622
 623       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 624       next0 = SNAT_IN2OUT_NEXT_DROP;
 625       goto out;
 626     }
 627
 628   if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
 629                      (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 630                      !icmp_is_error_message (icmp0)))
 631     {
 632       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 633       next0 = SNAT_IN2OUT_NEXT_DROP;
 634       goto out;
 635     }
 636
 637 out:
 638   *p_value = sm0;
 639 out2:
 640   *p_proto = key0.protocol;
 641   *p_dont_translate = dont_translate;
 642   return next0;
 643 }
 644 #endif
 645
 646 #ifndef CLIB_MARCH_VARIANT
 647 u32
 648 icmp_in2out (snat_main_t * sm,
 649              vlib_buffer_t * b0,
 650              ip4_header_t * ip0,
 651              icmp46_header_t * icmp0,
 652              u32 sw_if_index0,
 653              u32 rx_fib_index0,
 654              vlib_node_runtime_t * node,
 655              u32 next0, u32 thread_index, void *d, void *e)
 656 {
 657   snat_session_key_t sm0;
 658   u8 protocol;
 659   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 660   ip4_header_t *inner_ip0;
 661   void *l4_header = 0;
 662   icmp46_header_t *inner_icmp0;
 663   u8 dont_translate;
 664   u32 new_addr0, old_addr0;
 665   u16 old_id0, new_id0;
 666   u16 old_checksum0, new_checksum0;
 667   ip_csum_t sum0;
 668   u16 checksum0;
 669   u32 next0_tmp;
 670
 671   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 672
 673   next0_tmp = sm->icmp_match_in2out_cb (sm, node, thread_index, b0, ip0,
 674                                         &protocol, &sm0, &dont_translate, d,
 675                                         e);
 676   if (next0_tmp != ~0)
 677     next0 = next0_tmp;
 678   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 679     goto out;
 680
 681   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 682     {
 683       sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
 684                                              (u8 *)
 685                                              vlib_buffer_get_current (b0),
 686                                              ntohs (ip0->length) -
 687                                              ip4_header_bytes (ip0), 0);
 688       checksum0 = ~ip_csum_fold (sum0);
 689       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 690         {
 691           next0 = SNAT_IN2OUT_NEXT_DROP;
 692           goto out;
 693         }
 694     }
 695
 696   old_addr0 = ip0->src_address.as_u32;
 697   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 698
 699   sum0 = ip0->checksum;
 700   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 701                          src_address /* changed member */ );
 702   ip0->checksum = ip_csum_fold (sum0);
 703
 704   if (icmp0->checksum == 0)
 705     icmp0->checksum = 0xffff;
 706
 707   if (!icmp_is_error_message (icmp0))
 708     {
 709       new_id0 = sm0.port;
 710       if (PREDICT_FALSE (new_id0 != echo0->identifier))
 711         {
 712           old_id0 = echo0->identifier;
 713           new_id0 = sm0.port;
 714           echo0->identifier = new_id0;
 715
 716           sum0 = icmp0->checksum;
 717           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 718                                  identifier);
 719           icmp0->checksum = ip_csum_fold (sum0);
 720         }
 721     }
 722   else
 723     {
 724       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 725       l4_header = ip4_next_header (inner_ip0);
 726
 727       if (!ip4_header_checksum_is_valid (inner_ip0))
 728         {
 729           next0 = SNAT_IN2OUT_NEXT_DROP;
 730           goto out;
 731         }
 732
 733       /* update inner destination IP address */
 734       old_addr0 = inner_ip0->dst_address.as_u32;
 735       inner_ip0->dst_address = sm0.addr;
 736       new_addr0 = inner_ip0->dst_address.as_u32;
 737       sum0 = icmp0->checksum;
 738       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 739                              dst_address /* changed member */ );
 740       icmp0->checksum = ip_csum_fold (sum0);
 741
 742       /* update inner IP header checksum */
 743       old_checksum0 = inner_ip0->checksum;
 744       sum0 = inner_ip0->checksum;
 745       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 746                              dst_address /* changed member */ );
 747       inner_ip0->checksum = ip_csum_fold (sum0);
 748       new_checksum0 = inner_ip0->checksum;
 749       sum0 = icmp0->checksum;
 750       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 751                              checksum);
 752       icmp0->checksum = ip_csum_fold (sum0);
 753
 754       switch (protocol)
 755         {
 756         case SNAT_PROTOCOL_ICMP:
 757           inner_icmp0 = (icmp46_header_t *) l4_header;
 758           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 759
 760           old_id0 = inner_echo0->identifier;
 761           new_id0 = sm0.port;
 762           inner_echo0->identifier = new_id0;
 763
 764           sum0 = icmp0->checksum;
 765           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 766                                  identifier);
 767           icmp0->checksum = ip_csum_fold (sum0);
 768           break;
 769         case SNAT_PROTOCOL_UDP:
 770         case SNAT_PROTOCOL_TCP:
 771           old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 772           new_id0 = sm0.port;
 773           ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 774
 775           sum0 = icmp0->checksum;
 776           sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 777                                  dst_port);
 778           icmp0->checksum = ip_csum_fold (sum0);
 779           break;
 780         default:
 781           ASSERT (0);
 782         }
 783     }
 784
 785   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 786     {
 787       if (sm->deterministic ||
 788           0 != snat_icmp_hairpinning (sm, b0, ip0, icmp0,
 789                                       sm->endpoint_dependent))
 790         vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 791     }
 792
 793 out:
 794   return next0;
 795 }
 796 #endif
 797
 798 static inline u32
 799 icmp_in2out_slow_path (snat_main_t * sm,
 800                        vlib_buffer_t * b0,
 801                        ip4_header_t * ip0,
 802                        icmp46_header_t * icmp0,
 803                        u32 sw_if_index0,
 804                        u32 rx_fib_index0,
 805                        vlib_node_runtime_t * node,
 806                        u32 next0,
 807                        f64 now, u32 thread_index, snat_session_t ** p_s0)
 808 {
 809   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 810                        next0, thread_index, p_s0, 0);
 811   snat_session_t *s0 = *p_s0;
 812   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP && s0))
 813     {
 814       /* Accounting */
 815       nat44_session_update_counters (s0, now,
 816                                      vlib_buffer_length_in_chain
 817                                      (sm->vlib_main, b0), thread_index);
 818       /* Per-user LRU list maintenance */
 819       nat44_session_update_lru (sm, s0, thread_index);
 820     }
 821   return next0;
 822 }
 823
 824 static int
 825 nat_in2out_sm_unknown_proto (snat_main_t * sm,
 826                              vlib_buffer_t * b,
 827                              ip4_header_t * ip, u32 rx_fib_index)
 828 {
 829   clib_bihash_kv_8_8_t kv, value;
 830   snat_static_mapping_t *m;
 831   snat_session_key_t m_key;
 832   u32 old_addr, new_addr;
 833   ip_csum_t sum;
 834
 835   m_key.addr = ip->src_address;
 836   m_key.port = 0;
 837   m_key.protocol = 0;
 838   m_key.fib_index = rx_fib_index;
 839   kv.key = m_key.as_u64;
 840   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 841     return 1;
 842
 843   m = pool_elt_at_index (sm->static_mappings, value.value);
 844
 845   old_addr = ip->src_address.as_u32;
 846   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 847   sum = ip->checksum;
 848   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 849   ip->checksum = ip_csum_fold (sum);
 850
 851
 852   /* Hairpinning */
 853   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 854     {
 855       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 856       nat_hairpinning_sm_unknown_proto (sm, b, ip);
 857     }
 858
 859   return 0;
 860 }
 861
 862 static inline uword
 863 snat_in2out_node_fn_inline (vlib_main_t * vm,
 864                             vlib_node_runtime_t * node,
 865                             vlib_frame_t * frame, int is_slow_path,
 866                             int is_output_feature)
 867 {
 868   u32 n_left_from, *from, *to_next;
 869   snat_in2out_next_t next_index;
 870   u32 pkts_processed = 0;
 871   snat_main_t *sm = &snat_main;
 872   f64 now = vlib_time_now (vm);
 873   u32 stats_node_index;
 874   u32 thread_index = vm->thread_index;
 875   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 876     0, fragments = 0;
 877
 878   stats_node_index = is_slow_path ? sm->in2out_slowpath_node_index :
 879     sm->in2out_node_index;
 880
 881   from = vlib_frame_vector_args (frame);
 882   n_left_from = frame->n_vectors;
 883   next_index = node->cached_next_index;
 884
 885   while (n_left_from > 0)
 886     {
 887       u32 n_left_to_next;
 888
 889       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 890
 891       while (n_left_from >= 4 && n_left_to_next >= 2)
 892         {
 893           u32 bi0, bi1;
 894           vlib_buffer_t *b0, *b1;
 895           u32 next0, next1;
 896           u32 sw_if_index0, sw_if_index1;
 897           ip4_header_t *ip0, *ip1;
 898           ip_csum_t sum0, sum1;
 899           u32 new_addr0, old_addr0, new_addr1, old_addr1;
 900           u16 old_port0, new_port0, old_port1, new_port1;
 901           udp_header_t *udp0, *udp1;
 902           tcp_header_t *tcp0, *tcp1;
 903           icmp46_header_t *icmp0, *icmp1;
 904           snat_session_key_t key0, key1;
 905           u32 rx_fib_index0, rx_fib_index1;
 906           u32 proto0, proto1;
 907           snat_session_t *s0 = 0, *s1 = 0;
 908           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 909           u32 iph_offset0 = 0, iph_offset1 = 0;
 910
 911           /* Prefetch next iteration. */
 912           {
 913             vlib_buffer_t *p2, *p3;
 914
 915             p2 = vlib_get_buffer (vm, from[2]);
 916             p3 = vlib_get_buffer (vm, from[3]);
 917
 918             vlib_prefetch_buffer_header (p2, LOAD);
 919             vlib_prefetch_buffer_header (p3, LOAD);
 920
 921             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 922             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 923           }
 924
 925           /* speculatively enqueue b0 and b1 to the current next frame */
 926           to_next[0] = bi0 = from[0];
 927           to_next[1] = bi1 = from[1];
 928           from += 2;
 929           to_next += 2;
 930           n_left_from -= 2;
 931           n_left_to_next -= 2;
 932
 933           b0 = vlib_get_buffer (vm, bi0);
 934           b1 = vlib_get_buffer (vm, bi1);
 935
 936           if (is_output_feature)
 937             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
 938
 939           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 940                                   iph_offset0);
 941
 942           udp0 = ip4_next_header (ip0);
 943           tcp0 = (tcp_header_t *) udp0;
 944           icmp0 = (icmp46_header_t *) udp0;
 945
 946           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 947           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 948                                    sw_if_index0);
 949
 950           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
 951
 952           if (PREDICT_FALSE (ip0->ttl == 1))
 953             {
 954               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 955               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 956                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 957                                            0);
 958               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
 959               goto trace00;
 960             }
 961
 962           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 963
 964           /* Next configured feature, probably ip4-lookup */
 965           if (is_slow_path)
 966             {
 967               if (PREDICT_FALSE (proto0 == ~0))
 968                 {
 969                   if (nat_in2out_sm_unknown_proto
 970                       (sm, b0, ip0, rx_fib_index0))
 971                     {
 972                       next0 = SNAT_IN2OUT_NEXT_DROP;
 973                       b0->error =
 974                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 975                     }
 976                   other_packets++;
 977                   goto trace00;
 978                 }
 979
 980               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 981                 {
 982                   next0 = icmp_in2out_slow_path
 983                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 984                      node, next0, now, thread_index, &s0);
 985                   icmp_packets++;
 986                   goto trace00;
 987                 }
 988             }
 989           else
 990             {
 991               if (PREDICT_FALSE (proto0 == ~0))
 992                 {
 993                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
 994                   goto trace00;
 995                 }
 996
 997               if (ip4_is_fragment (ip0))
 998                 {
 999                   next0 = SNAT_IN2OUT_NEXT_REASS;
1000                   fragments++;
1001                   goto trace00;
1002                 }
1003
1004               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1005                 {
1006                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1007                   goto trace00;
1008                 }
1009             }
1010
1011           key0.addr = ip0->src_address;
1012           key0.port = udp0->src_port;
1013           key0.protocol = proto0;
1014           key0.fib_index = rx_fib_index0;
1015
1016           kv0.key = key0.as_u64;
1017
1018           if (PREDICT_FALSE
1019               (clib_bihash_search_8_8
1020                (&sm->per_thread_data[thread_index].in2out, &kv0,
1021                 &value0) != 0))
1022             {
1023               if (is_slow_path)
1024                 {
1025                   if (is_output_feature)
1026                     {
1027                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1028                                                                            ip0,
1029                                                                            proto0,
1030                                                                            udp0->src_port,
1031                                                                            udp0->dst_port,
1032                                                                            thread_index,
1033                                                                            sw_if_index0)))
1034                         goto trace00;
1035                     }
1036                   else
1037                     {
1038                       if (PREDICT_FALSE
1039                           (snat_not_translate
1040                            (sm, node, sw_if_index0, ip0, proto0,
1041                             rx_fib_index0, thread_index)))
1042                         goto trace00;
1043                     }
1044
1045                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1046                                      &s0, node, next0, thread_index, now);
1047                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1048                     goto trace00;
1049
1050                   if (PREDICT_FALSE (!s0))
1051                     goto trace00;
1052                 }
1053               else
1054                 {
1055                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1056                   goto trace00;
1057                 }
1058             }
1059           else
1060             s0 =
1061               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1062                                  value0.value);
1063
1064           b0->flags |= VNET_BUFFER_F_IS_NATED;
1065
1066           old_addr0 = ip0->src_address.as_u32;
1067           ip0->src_address = s0->out2in.addr;
1068           new_addr0 = ip0->src_address.as_u32;
1069           if (!is_output_feature)
1070             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1071
1072           sum0 = ip0->checksum;
1073           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1074                                  ip4_header_t,
1075                                  src_address /* changed member */ );
1076           ip0->checksum = ip_csum_fold (sum0);
1077
1078           old_port0 = udp0->src_port;
1079           new_port0 = udp0->src_port = s0->out2in.port;
1080
1081           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1082             {
1083               sum0 = tcp0->checksum;
1084               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1085                                      ip4_header_t,
1086                                      dst_address /* changed member */ );
1087               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1088                                      ip4_header_t /* cheat */ ,
1089                                      length /* changed member */ );
1090               mss_clamping (sm, tcp0, &sum0);
1091               tcp0->checksum = ip_csum_fold (sum0);
1092               tcp_packets++;
1093             }
1094           else
1095             {
1096               if (PREDICT_FALSE (udp0->checksum))
1097                 {
1098                   sum0 = udp0->checksum;
1099                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1100                                          ip4_header_t,
1101                                          dst_address /* changed member */ );
1102                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1103                                          ip4_header_t /* cheat */ ,
1104                                          length /* changed member */ );
1105                   udp0->checksum = ip_csum_fold (sum0);
1106                 }
1107               udp_packets++;
1108             }
1109
1110           /* Accounting */
1111           nat44_session_update_counters (s0, now,
1112                                          vlib_buffer_length_in_chain (vm, b0),
1113                                          thread_index);
1114           /* Per-user LRU list maintenance */
1115           nat44_session_update_lru (sm, s0, thread_index);
1116         trace00:
1117
1118           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1119                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1120             {
1121               snat_in2out_trace_t *t =
1122                 vlib_add_trace (vm, node, b0, sizeof (*t));
1123               t->is_slow_path = is_slow_path;
1124               t->sw_if_index = sw_if_index0;
1125               t->next_index = next0;
1126               t->session_index = ~0;
1127               if (s0)
1128                 t->session_index =
1129                   s0 - sm->per_thread_data[thread_index].sessions;
1130             }
1131
1132           pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1133
1134           if (is_output_feature)
1135             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1136
1137           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1138                                   iph_offset1);
1139
1140           udp1 = ip4_next_header (ip1);
1141           tcp1 = (tcp_header_t *) udp1;
1142           icmp1 = (icmp46_header_t *) udp1;
1143
1144           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1145           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1146                                    sw_if_index1);
1147
1148           if (PREDICT_FALSE (ip1->ttl == 1))
1149             {
1150               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1151               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1152                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1153                                            0);
1154               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1155               goto trace01;
1156             }
1157
1158           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1159
1160           /* Next configured feature, probably ip4-lookup */
1161           if (is_slow_path)
1162             {
1163               if (PREDICT_FALSE (proto1 == ~0))
1164                 {
1165                   if (nat_in2out_sm_unknown_proto
1166                       (sm, b1, ip1, rx_fib_index1))
1167                     {
1168                       next1 = SNAT_IN2OUT_NEXT_DROP;
1169                       b1->error =
1170                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1171                     }
1172                   other_packets++;
1173                   goto trace01;
1174                 }
1175
1176               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1177                 {
1178                   next1 = icmp_in2out_slow_path
1179                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1180                      next1, now, thread_index, &s1);
1181                   icmp_packets++;
1182                   goto trace01;
1183                 }
1184             }
1185           else
1186             {
1187               if (PREDICT_FALSE (proto1 == ~0))
1188                 {
1189                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1190                   goto trace01;
1191                 }
1192
1193               if (ip4_is_fragment (ip1))
1194                 {
1195                   next1 = SNAT_IN2OUT_NEXT_REASS;
1196                   fragments++;
1197                   goto trace01;
1198                 }
1199
1200               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1201                 {
1202                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1203                   goto trace01;
1204                 }
1205             }
1206
1207           key1.addr = ip1->src_address;
1208           key1.port = udp1->src_port;
1209           key1.protocol = proto1;
1210           key1.fib_index = rx_fib_index1;
1211
1212           kv1.key = key1.as_u64;
1213
1214           if (PREDICT_FALSE
1215               (clib_bihash_search_8_8
1216                (&sm->per_thread_data[thread_index].in2out, &kv1,
1217                 &value1) != 0))
1218             {
1219               if (is_slow_path)
1220                 {
1221                   if (is_output_feature)
1222                     {
1223                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1224                                                                            ip1,
1225                                                                            proto1,
1226                                                                            udp1->src_port,
1227                                                                            udp1->dst_port,
1228                                                                            thread_index,
1229                                                                            sw_if_index1)))
1230                         goto trace01;
1231                     }
1232                   else
1233                     {
1234                       if (PREDICT_FALSE
1235                           (snat_not_translate
1236                            (sm, node, sw_if_index1, ip1, proto1,
1237                             rx_fib_index1, thread_index)))
1238                         goto trace01;
1239                     }
1240
1241                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1242                                      &s1, node, next1, thread_index, now);
1243                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1244                     goto trace01;
1245
1246                   if (PREDICT_FALSE (!s1))
1247                     goto trace01;
1248                 }
1249               else
1250                 {
1251                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1252                   goto trace01;
1253                 }
1254             }
1255           else
1256             s1 =
1257               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1258                                  value1.value);
1259
1260           b1->flags |= VNET_BUFFER_F_IS_NATED;
1261
1262           old_addr1 = ip1->src_address.as_u32;
1263           ip1->src_address = s1->out2in.addr;
1264           new_addr1 = ip1->src_address.as_u32;
1265           if (!is_output_feature)
1266             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1267
1268           sum1 = ip1->checksum;
1269           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1270                                  ip4_header_t,
1271                                  src_address /* changed member */ );
1272           ip1->checksum = ip_csum_fold (sum1);
1273
1274           old_port1 = udp1->src_port;
1275           new_port1 = udp1->src_port = s1->out2in.port;
1276
1277           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1278             {
1279               sum1 = tcp1->checksum;
1280               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1281                                      ip4_header_t,
1282                                      dst_address /* changed member */ );
1283               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1284                                      ip4_header_t /* cheat */ ,
1285                                      length /* changed member */ );
1286               mss_clamping (sm, tcp1, &sum1);
1287               tcp1->checksum = ip_csum_fold (sum1);
1288               tcp_packets++;
1289             }
1290           else
1291             {
1292               if (PREDICT_FALSE (udp1->checksum))
1293                 {
1294                   sum1 = udp1->checksum;
1295                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1296                                          ip4_header_t,
1297                                          dst_address /* changed member */ );
1298                   sum1 = ip_csum_update (sum1, old_port1, new_port1,
1299                                          ip4_header_t /* cheat */ ,
1300                                          length /* changed member */ );
1301                   udp1->checksum = ip_csum_fold (sum1);
1302                 }
1303               udp_packets++;
1304             }
1305
1306           /* Accounting */
1307           nat44_session_update_counters (s1, now,
1308                                          vlib_buffer_length_in_chain (vm, b1),
1309                                          thread_index);
1310           /* Per-user LRU list maintenance */
1311           nat44_session_update_lru (sm, s1, thread_index);
1312         trace01:
1313
1314           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1315                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1316             {
1317               snat_in2out_trace_t *t =
1318                 vlib_add_trace (vm, node, b1, sizeof (*t));
1319               t->sw_if_index = sw_if_index1;
1320               t->next_index = next1;
1321               t->session_index = ~0;
1322               if (s1)
1323                 t->session_index =
1324                   s1 - sm->per_thread_data[thread_index].sessions;
1325             }
1326
1327           pkts_processed += next1 == SNAT_IN2OUT_NEXT_LOOKUP;
1328
1329           /* verify speculative enqueues, maybe switch current next frame */
1330           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1331                                            to_next, n_left_to_next,
1332                                            bi0, bi1, next0, next1);
1333         }
1334
1335       while (n_left_from > 0 && n_left_to_next > 0)
1336         {
1337           u32 bi0;
1338           vlib_buffer_t *b0;
1339           u32 next0;
1340           u32 sw_if_index0;
1341           ip4_header_t *ip0;
1342           ip_csum_t sum0;
1343           u32 new_addr0, old_addr0;
1344           u16 old_port0, new_port0;
1345           udp_header_t *udp0;
1346           tcp_header_t *tcp0;
1347           icmp46_header_t *icmp0;
1348           snat_session_key_t key0;
1349           u32 rx_fib_index0;
1350           u32 proto0;
1351           snat_session_t *s0 = 0;
1352           clib_bihash_kv_8_8_t kv0, value0;
1353           u32 iph_offset0 = 0;
1354
1355           /* speculatively enqueue b0 to the current next frame */
1356           bi0 = from[0];
1357           to_next[0] = bi0;
1358           from += 1;
1359           to_next += 1;
1360           n_left_from -= 1;
1361           n_left_to_next -= 1;
1362
1363           b0 = vlib_get_buffer (vm, bi0);
1364           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1365
1366           if (is_output_feature)
1367             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1368
1369           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1370                                   iph_offset0);
1371
1372           udp0 = ip4_next_header (ip0);
1373           tcp0 = (tcp_header_t *) udp0;
1374           icmp0 = (icmp46_header_t *) udp0;
1375
1376           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1377           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1378                                    sw_if_index0);
1379
1380           if (PREDICT_FALSE (ip0->ttl == 1))
1381             {
1382               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1383               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1384                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1385                                            0);
1386               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1387               goto trace0;
1388             }
1389
1390           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1391
1392           /* Next configured feature, probably ip4-lookup */
1393           if (is_slow_path)
1394             {
1395               if (PREDICT_FALSE (proto0 == ~0))
1396                 {
1397                   if (nat_in2out_sm_unknown_proto
1398                       (sm, b0, ip0, rx_fib_index0))
1399                     {
1400                       next0 = SNAT_IN2OUT_NEXT_DROP;
1401                       b0->error =
1402                         node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1403                     }
1404                   other_packets++;
1405                   goto trace0;
1406                 }
1407
1408               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1409                 {
1410                   next0 = icmp_in2out_slow_path
1411                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1412                      next0, now, thread_index, &s0);
1413                   icmp_packets++;
1414                   goto trace0;
1415                 }
1416             }
1417           else
1418             {
1419               if (PREDICT_FALSE (proto0 == ~0))
1420                 {
1421                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1422                   goto trace0;
1423                 }
1424
1425               if (ip4_is_fragment (ip0))
1426                 {
1427                   next0 = SNAT_IN2OUT_NEXT_REASS;
1428                   fragments++;
1429                   goto trace0;
1430                 }
1431
1432               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1433                 {
1434                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1435                   goto trace0;
1436                 }
1437             }
1438
1439           key0.addr = ip0->src_address;
1440           key0.port = udp0->src_port;
1441           key0.protocol = proto0;
1442           key0.fib_index = rx_fib_index0;
1443
1444           kv0.key = key0.as_u64;
1445
1446           if (clib_bihash_search_8_8
1447               (&sm->per_thread_data[thread_index].in2out, &kv0, &value0))
1448             {
1449               if (is_slow_path)
1450                 {
1451                   if (is_output_feature)
1452                     {
1453                       if (PREDICT_FALSE (nat_not_translate_output_feature (sm,
1454                                                                            ip0,
1455                                                                            proto0,
1456                                                                            udp0->src_port,
1457                                                                            udp0->dst_port,
1458                                                                            thread_index,
1459                                                                            sw_if_index0)))
1460                         goto trace0;
1461                     }
1462                   else
1463                     {
1464                       if (PREDICT_FALSE
1465                           (snat_not_translate
1466                            (sm, node, sw_if_index0, ip0, proto0,
1467                             rx_fib_index0, thread_index)))
1468                         goto trace0;
1469                     }
1470
1471                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1472                                      &s0, node, next0, thread_index, now);
1473
1474                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1475                     goto trace0;
1476
1477                   if (PREDICT_FALSE (!s0))
1478                     goto trace0;
1479                 }
1480               else
1481                 {
1482                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1483                   goto trace0;
1484                 }
1485             }
1486           else
1487             s0 =
1488               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1489                                  value0.value);
1490
1491           b0->flags |= VNET_BUFFER_F_IS_NATED;
1492
1493           old_addr0 = ip0->src_address.as_u32;
1494           ip0->src_address = s0->out2in.addr;
1495           new_addr0 = ip0->src_address.as_u32;
1496           if (!is_output_feature)
1497             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1498
1499           sum0 = ip0->checksum;
1500           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1501                                  ip4_header_t,
1502                                  src_address /* changed member */ );
1503           ip0->checksum = ip_csum_fold (sum0);
1504
1505           old_port0 = udp0->src_port;
1506           new_port0 = udp0->src_port = s0->out2in.port;
1507
1508           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1509             {
1510               sum0 = tcp0->checksum;
1511               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1512                                      ip4_header_t,
1513                                      dst_address /* changed member */ );
1514               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1515                                      ip4_header_t /* cheat */ ,
1516                                      length /* changed member */ );
1517               mss_clamping (sm, tcp0, &sum0);
1518               tcp0->checksum = ip_csum_fold (sum0);
1519               tcp_packets++;
1520             }
1521           else
1522             {
1523               if (PREDICT_FALSE (udp0->checksum))
1524                 {
1525                   sum0 = udp0->checksum;
1526                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1527                                          ip4_header_t,
1528                                          dst_address /* changed member */ );
1529                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1530                                          ip4_header_t /* cheat */ ,
1531                                          length /* changed member */ );
1532                   udp0->checksum = ip_csum_fold (sum0);
1533                 }
1534               udp_packets++;
1535             }
1536
1537           /* Accounting */
1538           nat44_session_update_counters (s0, now,
1539                                          vlib_buffer_length_in_chain (vm, b0),
1540                                          thread_index);
1541           /* Per-user LRU list maintenance */
1542           nat44_session_update_lru (sm, s0, thread_index);
1543
1544         trace0:
1545           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1546                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1547             {
1548               snat_in2out_trace_t *t =
1549                 vlib_add_trace (vm, node, b0, sizeof (*t));
1550               t->is_slow_path = is_slow_path;
1551               t->sw_if_index = sw_if_index0;
1552               t->next_index = next0;
1553               t->session_index = ~0;
1554               if (s0)
1555                 t->session_index =
1556                   s0 - sm->per_thread_data[thread_index].sessions;
1557             }
1558
1559           pkts_processed += next0 == SNAT_IN2OUT_NEXT_LOOKUP;
1560
1561           /* verify speculative enqueue, maybe switch current next frame */
1562           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1563                                            to_next, n_left_to_next,
1564                                            bi0, next0);
1565         }
1566
1567       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1568     }
1569
1570   vlib_node_increment_counter (vm, stats_node_index,
1571                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1572                                pkts_processed);
1573   vlib_node_increment_counter (vm, stats_node_index,
1574                                SNAT_IN2OUT_ERROR_TCP_PACKETS, tcp_packets);
1575   vlib_node_increment_counter (vm, stats_node_index,
1576                                SNAT_IN2OUT_ERROR_UDP_PACKETS, tcp_packets);
1577   vlib_node_increment_counter (vm, stats_node_index,
1578                                SNAT_IN2OUT_ERROR_ICMP_PACKETS, icmp_packets);
1579   vlib_node_increment_counter (vm, stats_node_index,
1580                                SNAT_IN2OUT_ERROR_OTHER_PACKETS,
1581                                other_packets);
1582   vlib_node_increment_counter (vm, stats_node_index,
1583                                SNAT_IN2OUT_ERROR_FRAGMENTS, fragments);
1584
1585   return frame->n_vectors;
1586 }
1587
1588 VLIB_NODE_FN (snat_in2out_node) (vlib_main_t * vm,
1589                                  vlib_node_runtime_t * node,
1590                                  vlib_frame_t * frame)
1591 {
1592   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1593                                      0);
1594 }
1595
1596 /* *INDENT-OFF* */
1597 VLIB_REGISTER_NODE (snat_in2out_node) = {
1598   .name = "nat44-in2out",
1599   .vector_size = sizeof (u32),
1600   .format_trace = format_snat_in2out_trace,
1601   .type = VLIB_NODE_TYPE_INTERNAL,
1602
1603   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1604   .error_strings = snat_in2out_error_strings,
1605
1606   .runtime_data_bytes = sizeof (snat_runtime_t),
1607
1608   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1609
1610   /* edit / add dispositions here */
1611   .next_nodes = {
1612     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1613     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1614     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1615     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1616     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1617   },
1618 };
1619 /* *INDENT-ON* */
1620
1621 VLIB_NODE_FN (snat_in2out_output_node) (vlib_main_t * vm,
1622                                         vlib_node_runtime_t * node,
1623                                         vlib_frame_t * frame)
1624 {
1625   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */ ,
1626                                      1);
1627 }
1628
1629 /* *INDENT-OFF* */
1630 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1631   .name = "nat44-in2out-output",
1632   .vector_size = sizeof (u32),
1633   .format_trace = format_snat_in2out_trace,
1634   .type = VLIB_NODE_TYPE_INTERNAL,
1635
1636   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1637   .error_strings = snat_in2out_error_strings,
1638
1639   .runtime_data_bytes = sizeof (snat_runtime_t),
1640
1641   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1642
1643   /* edit / add dispositions here */
1644   .next_nodes = {
1645     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1646     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1647     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1648     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1649     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1650   },
1651 };
1652 /* *INDENT-ON* */
1653
1654 VLIB_NODE_FN (snat_in2out_slowpath_node) (vlib_main_t * vm,
1655                                           vlib_node_runtime_t * node,
1656                                           vlib_frame_t * frame)
1657 {
1658   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1659                                      0);
1660 }
1661
1662 /* *INDENT-OFF* */
1663 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1664   .name = "nat44-in2out-slowpath",
1665   .vector_size = sizeof (u32),
1666   .format_trace = format_snat_in2out_trace,
1667   .type = VLIB_NODE_TYPE_INTERNAL,
1668
1669   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1670   .error_strings = snat_in2out_error_strings,
1671
1672   .runtime_data_bytes = sizeof (snat_runtime_t),
1673
1674   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1675
1676   /* edit / add dispositions here */
1677   .next_nodes = {
1678     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1679     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1680     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1681     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1682     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1683   },
1684 };
1685 /* *INDENT-ON* */
1686
1687 VLIB_NODE_FN (snat_in2out_output_slowpath_node) (vlib_main_t * vm,
1688                                                  vlib_node_runtime_t * node,
1689                                                  vlib_frame_t * frame)
1690 {
1691   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */ ,
1692                                      1);
1693 }
1694
1695 /* *INDENT-OFF* */
1696 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1697   .name = "nat44-in2out-output-slowpath",
1698   .vector_size = sizeof (u32),
1699   .format_trace = format_snat_in2out_trace,
1700   .type = VLIB_NODE_TYPE_INTERNAL,
1701
1702   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1703   .error_strings = snat_in2out_error_strings,
1704
1705   .runtime_data_bytes = sizeof (snat_runtime_t),
1706
1707   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1708
1709   /* edit / add dispositions here */
1710   .next_nodes = {
1711     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1712     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1713     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1714     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1715     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1716   },
1717 };
1718 /* *INDENT-ON* */
1719
1720 VLIB_NODE_FN (nat44_in2out_reass_node) (vlib_main_t * vm,
1721                                         vlib_node_runtime_t * node,
1722                                         vlib_frame_t * frame)
1723 {
1724   u32 n_left_from, *from, *to_next;
1725   snat_in2out_next_t next_index;
1726   u32 pkts_processed = 0, cached_fragments = 0;
1727   snat_main_t *sm = &snat_main;
1728   f64 now = vlib_time_now (vm);
1729   u32 thread_index = vm->thread_index;
1730   snat_main_per_thread_data_t *per_thread_data =
1731     &sm->per_thread_data[thread_index];
1732   u32 *fragments_to_drop = 0;
1733   u32 *fragments_to_loopback = 0;
1734
1735   from = vlib_frame_vector_args (frame);
1736   n_left_from = frame->n_vectors;
1737   next_index = node->cached_next_index;
1738
1739   while (n_left_from > 0)
1740     {
1741       u32 n_left_to_next;
1742
1743       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1744
1745       while (n_left_from > 0 && n_left_to_next > 0)
1746         {
1747           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1748           vlib_buffer_t *b0;
1749           u32 next0;
1750           u8 cached0 = 0;
1751           ip4_header_t *ip0;
1752           nat_reass_ip4_t *reass0;
1753           udp_header_t *udp0;
1754           tcp_header_t *tcp0;
1755           icmp46_header_t *icmp0;
1756           snat_session_key_t key0;
1757           clib_bihash_kv_8_8_t kv0, value0;
1758           snat_session_t *s0 = 0;
1759           u16 old_port0, new_port0;
1760           ip_csum_t sum0;
1761
1762           /* speculatively enqueue b0 to the current next frame */
1763           bi0 = from[0];
1764           to_next[0] = bi0;
1765           from += 1;
1766           to_next += 1;
1767           n_left_from -= 1;
1768           n_left_to_next -= 1;
1769
1770           b0 = vlib_get_buffer (vm, bi0);
1771           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1772
1773           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1774           rx_fib_index0 =
1775             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1776                                                  sw_if_index0);
1777
1778           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1779             {
1780               next0 = SNAT_IN2OUT_NEXT_DROP;
1781               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
1782               goto trace0;
1783             }
1784
1785           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1786           udp0 = ip4_next_header (ip0);
1787           tcp0 = (tcp_header_t *) udp0;
1788           icmp0 = (icmp46_header_t *) udp0;
1789           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1790
1791           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1792                                                  ip0->dst_address,
1793                                                  ip0->fragment_id,
1794                                                  ip0->protocol,
1795                                                  1, &fragments_to_drop);
1796
1797           if (PREDICT_FALSE (!reass0))
1798             {
1799               next0 = SNAT_IN2OUT_NEXT_DROP;
1800               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
1801               nat_elog_notice ("maximum reassemblies exceeded");
1802               goto trace0;
1803             }
1804
1805           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1806             {
1807               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1808                 {
1809                   next0 = icmp_in2out_slow_path
1810                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1811                      next0, now, thread_index, &s0);
1812
1813                   if (PREDICT_TRUE (next0 != SNAT_IN2OUT_NEXT_DROP))
1814                     {
1815                       if (s0)
1816                         reass0->sess_index = s0 - per_thread_data->sessions;
1817                       else
1818                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1819                       nat_ip4_reass_get_frags (reass0,
1820                                                &fragments_to_loopback);
1821                     }
1822
1823                   goto trace0;
1824                 }
1825
1826               key0.addr = ip0->src_address;
1827               key0.port = udp0->src_port;
1828               key0.protocol = proto0;
1829               key0.fib_index = rx_fib_index0;
1830               kv0.key = key0.as_u64;
1831
1832               if (clib_bihash_search_8_8
1833                   (&per_thread_data->in2out, &kv0, &value0))
1834                 {
1835                   if (PREDICT_FALSE
1836                       (snat_not_translate
1837                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1838                         thread_index)))
1839                     goto trace0;
1840
1841                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1842                                      &s0, node, next0, thread_index, now);
1843
1844                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1845                     goto trace0;
1846
1847                   if (PREDICT_FALSE (!s0))
1848                     goto trace0;
1849
1850                   reass0->sess_index = s0 - per_thread_data->sessions;
1851                 }
1852               else
1853                 {
1854                   s0 = pool_elt_at_index (per_thread_data->sessions,
1855                                           value0.value);
1856                   reass0->sess_index = value0.value;
1857                 }
1858               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1859             }
1860           else
1861             {
1862               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1863                 {
1864                   if (nat_ip4_reass_add_fragment
1865                       (thread_index, reass0, bi0, &fragments_to_drop))
1866                     {
1867                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
1868                       nat_elog_notice
1869                         ("maximum fragments per reassembly exceeded");
1870                       next0 = SNAT_IN2OUT_NEXT_DROP;
1871                       goto trace0;
1872                     }
1873                   cached0 = 1;
1874                   goto trace0;
1875                 }
1876               s0 = pool_elt_at_index (per_thread_data->sessions,
1877                                       reass0->sess_index);
1878             }
1879
1880           old_addr0 = ip0->src_address.as_u32;
1881           ip0->src_address = s0->out2in.addr;
1882           new_addr0 = ip0->src_address.as_u32;
1883           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1884
1885           sum0 = ip0->checksum;
1886           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1887                                  ip4_header_t,
1888                                  src_address /* changed member */ );
1889           ip0->checksum = ip_csum_fold (sum0);
1890
1891           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1892             {
1893               old_port0 = udp0->src_port;
1894               new_port0 = udp0->src_port = s0->out2in.port;
1895
1896               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1897                 {
1898                   sum0 = tcp0->checksum;
1899                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1900                                          ip4_header_t,
1901                                          dst_address /* changed member */ );
1902                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1903                                          ip4_header_t /* cheat */ ,
1904                                          length /* changed member */ );
1905                   tcp0->checksum = ip_csum_fold (sum0);
1906                 }
1907               else if (PREDICT_FALSE (udp0->checksum))
1908                 {
1909                   sum0 = udp0->checksum;
1910                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1911                                          ip4_header_t,
1912                                          dst_address /* changed member */ );
1913                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1914                                          ip4_header_t /* cheat */ ,
1915                                          length /* changed member */ );
1916                   udp0->checksum = ip_csum_fold (sum0);
1917                 }
1918             }
1919
1920           /* Hairpinning */
1921           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1922                                    s0->ext_host_port, proto0, 0);
1923
1924           /* Accounting */
1925           nat44_session_update_counters (s0, now,
1926                                          vlib_buffer_length_in_chain (vm, b0),
1927                                          thread_index);
1928           /* Per-user LRU list maintenance */
1929           nat44_session_update_lru (sm, s0, thread_index);
1930
1931         trace0:
1932           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1933                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1934             {
1935               nat44_reass_trace_t *t =
1936                 vlib_add_trace (vm, node, b0, sizeof (*t));
1937               t->cached = cached0;
1938               t->sw_if_index = sw_if_index0;
1939               t->next_index = next0;
1940             }
1941
1942           if (cached0)
1943             {
1944               n_left_to_next++;
1945               to_next--;
1946               cached_fragments++;
1947             }
1948           else
1949             {
1950               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1951
1952               /* verify speculative enqueue, maybe switch current next frame */
1953               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1954                                                to_next, n_left_to_next,
1955                                                bi0, next0);
1956             }
1957
1958           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1959             {
1960               from = vlib_frame_vector_args (frame);
1961               u32 len = vec_len (fragments_to_loopback);
1962               if (len <= VLIB_FRAME_SIZE)
1963                 {
1964                   clib_memcpy_fast (from, fragments_to_loopback,
1965                                     sizeof (u32) * len);
1966                   n_left_from = len;
1967                   vec_reset_length (fragments_to_loopback);
1968                 }
1969               else
1970                 {
1971                   clib_memcpy_fast (from, fragments_to_loopback +
1972                                     (len - VLIB_FRAME_SIZE),
1973                                     sizeof (u32) * VLIB_FRAME_SIZE);
1974                   n_left_from = VLIB_FRAME_SIZE;
1975                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1976                 }
1977             }
1978         }
1979
1980       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1981     }
1982
1983   vlib_node_increment_counter (vm, sm->in2out_reass_node_index,
1984                                SNAT_IN2OUT_ERROR_PROCESSED_FRAGMENTS,
1985                                pkts_processed);
1986   vlib_node_increment_counter (vm, sm->in2out_reass_node_index,
1987                                SNAT_IN2OUT_ERROR_CACHED_FRAGMENTS,
1988                                cached_fragments);
1989
1990   nat_send_all_to_node (vm, fragments_to_drop, node,
1991                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
1992                         SNAT_IN2OUT_NEXT_DROP);
1993
1994   vec_free (fragments_to_drop);
1995   vec_free (fragments_to_loopback);
1996   return frame->n_vectors;
1997 }
1998
1999 /* *INDENT-OFF* */
2000 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2001   .name = "nat44-in2out-reass",
2002   .vector_size = sizeof (u32),
2003   .format_trace = format_nat44_reass_trace,
2004   .type = VLIB_NODE_TYPE_INTERNAL,
2005
2006   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2007   .error_strings = snat_in2out_error_strings,
2008
2009   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2010   .next_nodes = {
2011     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2012     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2013     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2014     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2015     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2016   },
2017 };
2018 /* *INDENT-ON* */
2019
2020 VLIB_NODE_FN (snat_in2out_fast_node) (vlib_main_t * vm,
2021                                       vlib_node_runtime_t * node,
2022                                       vlib_frame_t * frame)
2023 {
2024   u32 n_left_from, *from, *to_next;
2025   snat_in2out_next_t next_index;
2026   u32 pkts_processed = 0;
2027   snat_main_t *sm = &snat_main;
2028   u32 stats_node_index;
2029
2030   stats_node_index = sm->in2out_fast_node_index;
2031
2032   from = vlib_frame_vector_args (frame);
2033   n_left_from = frame->n_vectors;
2034   next_index = node->cached_next_index;
2035
2036   while (n_left_from > 0)
2037     {
2038       u32 n_left_to_next;
2039
2040       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2041
2042       while (n_left_from > 0 && n_left_to_next > 0)
2043         {
2044           u32 bi0;
2045           vlib_buffer_t *b0;
2046           u32 next0;
2047           u32 sw_if_index0;
2048           ip4_header_t *ip0;
2049           ip_csum_t sum0;
2050           u32 new_addr0, old_addr0;
2051           u16 old_port0, new_port0;
2052           udp_header_t *udp0;
2053           tcp_header_t *tcp0;
2054           icmp46_header_t *icmp0;
2055           snat_session_key_t key0, sm0;
2056           u32 proto0;
2057           u32 rx_fib_index0;
2058
2059           /* speculatively enqueue b0 to the current next frame */
2060           bi0 = from[0];
2061           to_next[0] = bi0;
2062           from += 1;
2063           to_next += 1;
2064           n_left_from -= 1;
2065           n_left_to_next -= 1;
2066
2067           b0 = vlib_get_buffer (vm, bi0);
2068           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2069
2070           ip0 = vlib_buffer_get_current (b0);
2071           udp0 = ip4_next_header (ip0);
2072           tcp0 = (tcp_header_t *) udp0;
2073           icmp0 = (icmp46_header_t *) udp0;
2074
2075           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2076           rx_fib_index0 =
2077             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2078
2079           if (PREDICT_FALSE (ip0->ttl == 1))
2080             {
2081               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2082               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2083                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2084                                            0);
2085               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2086               goto trace0;
2087             }
2088
2089           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2090
2091           if (PREDICT_FALSE (proto0 == ~0))
2092             goto trace0;
2093
2094           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2095             {
2096               next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0,
2097                                    rx_fib_index0, node, next0, ~0, 0, 0);
2098               goto trace0;
2099             }
2100
2101           key0.addr = ip0->src_address;
2102           key0.protocol = proto0;
2103           key0.port = udp0->src_port;
2104           key0.fib_index = rx_fib_index0;
2105
2106           if (snat_static_mapping_match (sm, key0, &sm0, 0, 0, 0, 0, 0, 0))
2107             {
2108               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2109               next0 = SNAT_IN2OUT_NEXT_DROP;
2110               goto trace0;
2111             }
2112
2113           new_addr0 = sm0.addr.as_u32;
2114           new_port0 = sm0.port;
2115           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2116           old_addr0 = ip0->src_address.as_u32;
2117           ip0->src_address.as_u32 = new_addr0;
2118
2119           sum0 = ip0->checksum;
2120           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2121                                  ip4_header_t,
2122                                  src_address /* changed member */ );
2123           ip0->checksum = ip_csum_fold (sum0);
2124
2125           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
2126             {
2127               old_port0 = udp0->src_port;
2128               udp0->src_port = new_port0;
2129
2130               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2131                 {
2132                   sum0 = tcp0->checksum;
2133                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2134                                          ip4_header_t,
2135                                          dst_address /* changed member */ );
2136                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2137                                          ip4_header_t /* cheat */ ,
2138                                          length /* changed member */ );
2139                   mss_clamping (sm, tcp0, &sum0);
2140                   tcp0->checksum = ip_csum_fold (sum0);
2141                 }
2142               else if (udp0->checksum)
2143                 {
2144                   sum0 = udp0->checksum;
2145                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2146                                          ip4_header_t,
2147                                          dst_address /* changed member */ );
2148                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2149                                          ip4_header_t /* cheat */ ,
2150                                          length /* changed member */ );
2151                   udp0->checksum = ip_csum_fold (sum0);
2152                 }
2153             }
2154           else
2155             {
2156               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2157                 {
2158                   sum0 = tcp0->checksum;
2159                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2160                                          ip4_header_t,
2161                                          dst_address /* changed member */ );
2162                   mss_clamping (sm, tcp0, &sum0);
2163                   tcp0->checksum = ip_csum_fold (sum0);
2164                 }
2165               else if (udp0->checksum)
2166                 {
2167                   sum0 = udp0->checksum;
2168                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2169                                          ip4_header_t,
2170                                          dst_address /* changed member */ );
2171                   udp0->checksum = ip_csum_fold (sum0);
2172                 }
2173             }
2174
2175           /* Hairpinning */
2176           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
2177
2178         trace0:
2179           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2180                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2181             {
2182               snat_in2out_trace_t *t =
2183                 vlib_add_trace (vm, node, b0, sizeof (*t));
2184               t->sw_if_index = sw_if_index0;
2185               t->next_index = next0;
2186             }
2187
2188           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2189
2190           /* verify speculative enqueue, maybe switch current next frame */
2191           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2192                                            to_next, n_left_to_next,
2193                                            bi0, next0);
2194         }
2195
2196       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2197     }
2198
2199   vlib_node_increment_counter (vm, stats_node_index,
2200                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2201                                pkts_processed);
2202   return frame->n_vectors;
2203 }
2204
2205
2206 /* *INDENT-OFF* */
2207 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
2208   .name = "nat44-in2out-fast",
2209   .vector_size = sizeof (u32),
2210   .format_trace = format_snat_in2out_fast_trace,
2211   .type = VLIB_NODE_TYPE_INTERNAL,
2212
2213   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2214   .error_strings = snat_in2out_error_strings,
2215
2216   .runtime_data_bytes = sizeof (snat_runtime_t),
2217
2218   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2219
2220   /* edit / add dispositions here */
2221   .next_nodes = {
2222     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2223     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2224     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2225     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2226     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2227   },
2228 };
2229 /* *INDENT-ON* */
2230
2231 /*
2232  * fd.io coding-style-patch-verification: ON
2233  *
2234  * Local Variables:
2235  * eval: (c-set-style "gnu")
2236  * End:
2237  */