src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/udp/udp.h>
  26 #include <vnet/ethernet/ethernet.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 #include <vppinfra/hash.h>
  36 #include <vppinfra/error.h>
  37 #include <vppinfra/elog.h>
  38
  39 typedef struct
  40 {
  41   u32 sw_if_index;
  42   u32 next_index;
  43   u32 session_index;
  44 } snat_out2in_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 *
  48 format_snat_out2in_trace (u8 * s, va_list * args)
  49 {
  50   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  51   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  52   snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
  53
  54   s =
  55     format (s,
  56             "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  57             t->sw_if_index, t->next_index, t->session_index);
  58   return s;
  59 }
  60
  61 static u8 *
  62 format_snat_out2in_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
  67
  68   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70   return s;
  71 }
  72
  73 #define foreach_snat_out2in_error                       \
  74 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  75 _(OUT2IN_PACKETS, "good out2in packets processed")      \
  76 _(OUT_OF_PORTS, "out of ports")                         \
  77 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  78 _(NO_TRANSLATION, "no translation")                     \
  79 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  80 _(DROP_FRAGMENT, "drop fragment")                       \
  81 _(MAX_REASS, "maximum reassemblies exceeded")           \
  82 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  83 _(TCP_PACKETS, "TCP packets")                           \
  84 _(UDP_PACKETS, "UDP packets")                           \
  85 _(ICMP_PACKETS, "ICMP packets")                         \
  86 _(OTHER_PACKETS, "other protocol packets")              \
  87 _(FRAGMENTS, "fragments")                               \
  88 _(CANNOT_CREATE_USER, "cannot create NAT user")
  89
  90 typedef enum
  91 {
  92 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  93   foreach_snat_out2in_error
  94 #undef _
  95     SNAT_OUT2IN_N_ERROR,
  96 } snat_out2in_error_t;
  97
  98 static char *snat_out2in_error_strings[] = {
  99 #define _(sym,string) string,
 100   foreach_snat_out2in_error
 101 #undef _
 102 };
 103
 104 typedef enum
 105 {
 106   SNAT_OUT2IN_NEXT_DROP,
 107   SNAT_OUT2IN_NEXT_LOOKUP,
 108   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 109   SNAT_OUT2IN_N_NEXT,
 110 } snat_out2in_next_t;
 111
 112 #ifndef CLIB_MARCH_VARIANT
 113 int
 114 nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 115 {
 116   snat_main_t *sm = &snat_main;
 117   nat44_is_idle_session_ctx_t *ctx = arg;
 118   snat_session_t *s;
 119   u64 sess_timeout_time;
 120   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 121                                                        ctx->thread_index);
 122   clib_bihash_kv_8_8_t s_kv;
 123
 124   s = pool_elt_at_index (tsm->sessions, kv->value);
 125   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 126   if (ctx->now >= sess_timeout_time)
 127     {
 128       init_nat_i2o_k (&s_kv, s);
 129       if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
 130         nat_elog_warn ("out2in key del failed");
 131
 132       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 133                                            s->in2out.addr.as_u32,
 134                                            s->out2in.addr.as_u32,
 135                                            s->nat_proto,
 136                                            s->in2out.port,
 137                                            s->out2in.port,
 138                                            s->in2out.fib_index);
 139
 140       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 141                                &s->in2out.addr, s->in2out.port,
 142                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 143
 144       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 145                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 146                    ctx->thread_index);
 147
 148       if (!snat_is_session_static (s))
 149         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 150                                             &s->out2in.addr, s->out2in.port,
 151                                             s->nat_proto);
 152
 153       nat44_delete_session (sm, s, ctx->thread_index);
 154       return 1;
 155     }
 156
 157   return 0;
 158 }
 159 #endif
 160
 161 /**
 162  * @brief Create session for static mapping.
 163  *
 164  * Create NAT session initiated by host from external network with static
 165  * mapping.
 166  *
 167  * @param sm     NAT main.
 168  * @param b0     Vlib buffer.
 169  * @param in2out In2out NAT44 session key.
 170  * @param out2in Out2in NAT44 session key.
 171  * @param node   Vlib node.
 172  *
 173  * @returns SNAT session if successfully created otherwise 0.
 174  */
 175 static inline snat_session_t *
 176 create_session_for_static_mapping (snat_main_t * sm,
 177                                    vlib_buffer_t * b0,
 178                                    ip4_address_t i2o_addr,
 179                                    u16 i2o_port,
 180                                    u32 i2o_fib_index,
 181                                    ip4_address_t o2i_addr,
 182                                    u16 o2i_port,
 183                                    u32 o2i_fib_index,
 184                                    nat_protocol_t proto,
 185                                    vlib_node_runtime_t * node,
 186                                    u32 thread_index, f64 now)
 187 {
 188   snat_user_t *u;
 189   snat_session_t *s;
 190   clib_bihash_kv_8_8_t kv0;
 191   ip4_header_t *ip0;
 192   udp_header_t *udp0;
 193   nat44_is_idle_session_ctx_t ctx0;
 194
 195   if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 196     {
 197       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 198       nat_elog_notice ("maximum sessions exceeded");
 199       return 0;
 200     }
 201
 202   ip0 = vlib_buffer_get_current (b0);
 203   udp0 = ip4_next_header (ip0);
 204
 205   u = nat_user_get_or_create (sm, &i2o_addr, i2o_fib_index, thread_index);
 206   if (!u)
 207     {
 208       b0->error = node->errors[SNAT_OUT2IN_ERROR_CANNOT_CREATE_USER];
 209       return 0;
 210     }
 211
 212   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 213   if (!s)
 214     {
 215       nat44_delete_user_with_no_session (sm, u, thread_index);
 216       nat_elog_warn ("create NAT session failed");
 217       return 0;
 218     }
 219
 220   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 221   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 222   s->ext_host_port = udp0->src_port;
 223   user_session_increment (sm, u, 1 /* static */ );
 224   s->in2out.addr = i2o_addr;
 225   s->in2out.port = i2o_port;
 226   s->in2out.fib_index = i2o_fib_index;
 227   s->out2in.addr = o2i_addr;
 228   s->out2in.port = o2i_port;
 229   s->out2in.fib_index = o2i_fib_index;
 230   s->nat_proto = proto;
 231
 232   /* Add to translation hashes */
 233   ctx0.now = now;
 234   ctx0.thread_index = thread_index;
 235   init_nat_i2o_kv (&kv0, s, s - sm->per_thread_data[thread_index].sessions);
 236   if (clib_bihash_add_or_overwrite_stale_8_8
 237       (&sm->per_thread_data[thread_index].in2out, &kv0,
 238        nat44_i2o_is_idle_session_cb, &ctx0))
 239     nat_elog_notice ("in2out key add failed");
 240
 241   init_nat_o2i_kv (&kv0, s, s - sm->per_thread_data[thread_index].sessions);
 242   if (clib_bihash_add_or_overwrite_stale_8_8
 243       (&sm->per_thread_data[thread_index].out2in, &kv0,
 244        nat44_o2i_is_idle_session_cb, &ctx0))
 245     nat_elog_notice ("out2in key add failed");
 246
 247   /* log NAT event */
 248   snat_ipfix_logging_nat44_ses_create (thread_index,
 249                                        s->in2out.addr.as_u32,
 250                                        s->out2in.addr.as_u32,
 251                                        s->nat_proto,
 252                                        s->in2out.port,
 253                                        s->out2in.port, s->in2out.fib_index);
 254
 255   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 256                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 257                            s->out2in.port, s->nat_proto);
 258
 259   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 260                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 261                &s->ext_host_nat_addr, s->ext_host_nat_port,
 262                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 263
 264   return s;
 265 }
 266
 267 #ifndef CLIB_MARCH_VARIANT
 268 static_always_inline snat_out2in_error_t
 269 icmp_get_key (vlib_buffer_t * b, ip4_header_t * ip0,
 270               ip4_address_t * addr, u16 * port, nat_protocol_t * nat_proto)
 271 {
 272   icmp46_header_t *icmp0;
 273   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 274   ip4_header_t *inner_ip0;
 275   void *l4_header = 0;
 276   icmp46_header_t *inner_icmp0;
 277
 278   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 279   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 280
 281   if (!icmp_type_is_error_message
 282       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 283     {
 284       *nat_proto = NAT_PROTOCOL_ICMP;
 285       *addr = ip0->dst_address;
 286       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 287     }
 288   else
 289     {
 290       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 291       l4_header = ip4_next_header (inner_ip0);
 292       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 293       *addr = inner_ip0->src_address;
 294       switch (*nat_proto)
 295         {
 296         case NAT_PROTOCOL_ICMP:
 297           inner_icmp0 = (icmp46_header_t *) l4_header;
 298           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 299           *port = inner_echo0->identifier;
 300           break;
 301         case NAT_PROTOCOL_UDP:
 302         case NAT_PROTOCOL_TCP:
 303           *port = ((tcp_udp_header_t *) l4_header)->src_port;
 304           break;
 305         default:
 306           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 307         }
 308     }
 309   return -1;                    /* success */
 310 }
 311
 312 /**
 313  * Get address and port values to be used for ICMP packet translation
 314  * and create session if needed
 315  *
 316  * @param[in,out] sm             NAT main
 317  * @param[in,out] node           NAT node runtime
 318  * @param[in] thread_index       thread index
 319  * @param[in,out] b0             buffer containing packet to be translated
 320  * @param[in,out] ip0            ip header
 321  * @param[out] p_proto           protocol used for matching
 322  * @param[out] p_value           address and port after NAT translation
 323  * @param[out] p_dont_translate  if packet should not be translated
 324  * @param d                      optional parameter
 325  * @param e                      optional parameter
 326  */
 327 u32
 328 icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 329                         u32 thread_index, vlib_buffer_t * b0,
 330                         ip4_header_t * ip0, ip4_address_t * addr,
 331                         u16 * port, u32 * fib_index,
 332                         nat_protocol_t * proto, void *d, void *e,
 333                         u8 * dont_translate)
 334 {
 335   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 336   u32 sw_if_index0;
 337   snat_session_t *s0 = 0;
 338   clib_bihash_kv_8_8_t kv0, value0;
 339   u8 is_addr_only;
 340   u32 next0 = ~0;
 341   int err;
 342   u8 identity_nat;
 343   vlib_main_t *vm = vlib_get_main ();
 344   *dont_translate = 0;
 345
 346   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 347   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 348
 349   *proto = 0;
 350
 351   err = icmp_get_key (b0, ip0, addr, port, proto);
 352   if (err != -1)
 353     {
 354       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 355       next0 = SNAT_OUT2IN_NEXT_DROP;
 356       goto out;
 357     }
 358
 359   ip4_address_t mapping_addr;
 360   u16 mapping_port;
 361   u32 mapping_fib_index;
 362
 363   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 364   if (clib_bihash_search_8_8 (&tsm->out2in, &kv0, &value0))
 365     {
 366       /* Try to match static mapping by external address and port,
 367          destination address and port in packet */
 368       if (snat_static_mapping_match
 369           (sm, *addr, *port, *fib_index, *proto,
 370            &mapping_addr, &mapping_port, &mapping_fib_index, 1, &is_addr_only,
 371            0, 0, 0, &identity_nat))
 372         {
 373           if (!sm->forwarding_enabled)
 374             {
 375               /* Don't NAT packet aimed at the intfc address */
 376               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index0,
 377                                                     ip0->dst_address.as_u32)))
 378                 {
 379                   *dont_translate = 1;
 380                   goto out;
 381                 }
 382               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 383               next0 = SNAT_OUT2IN_NEXT_DROP;
 384               goto out;
 385             }
 386           else
 387             {
 388               *dont_translate = 1;
 389               goto out;
 390             }
 391         }
 392
 393       if (PREDICT_FALSE
 394           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 395            ICMP4_echo_reply
 396            && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 397                ICMP4_echo_request || !is_addr_only)))
 398         {
 399           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 400           next0 = SNAT_OUT2IN_NEXT_DROP;
 401           goto out;
 402         }
 403
 404       if (PREDICT_FALSE (identity_nat))
 405         {
 406           *dont_translate = 1;
 407           goto out;
 408         }
 409       /* Create session initiated by host from external network */
 410       s0 =
 411         create_session_for_static_mapping (sm, b0, mapping_addr, mapping_port,
 412                                            mapping_fib_index, *addr, *port,
 413                                            *fib_index, *proto, node,
 414                                            thread_index, vlib_time_now (vm));
 415
 416       if (!s0)
 417         {
 418           next0 = SNAT_OUT2IN_NEXT_DROP;
 419           goto out;
 420         }
 421     }
 422   else
 423     {
 424       if (PREDICT_FALSE
 425           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 426            ICMP4_echo_reply
 427            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 428            ICMP4_echo_request
 429            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 430                                            reass.icmp_type_or_tcp_flags)))
 431         {
 432           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 433           next0 = SNAT_OUT2IN_NEXT_DROP;
 434           goto out;
 435         }
 436
 437       s0 = pool_elt_at_index (tsm->sessions, value0.value);
 438     }
 439
 440 out:
 441   if (s0)
 442     {
 443       *addr = s0->in2out.addr;
 444       *port = s0->in2out.port;
 445       *fib_index = s0->in2out.fib_index;
 446     }
 447   if (d)
 448     *(snat_session_t **) d = s0;
 449   return next0;
 450 }
 451 #endif
 452
 453 #ifndef CLIB_MARCH_VARIANT
 454 /**
 455  * Get address and port values to be used for ICMP packet translation
 456  *
 457  * @param[in] sm                 NAT main
 458  * @param[in,out] node           NAT node runtime
 459  * @param[in] thread_index       thread index
 460  * @param[in,out] b0             buffer containing packet to be translated
 461  * @param[in,out] ip0            ip header
 462  * @param[out] p_proto           protocol used for matching
 463  * @param[out] p_value           address and port after NAT translation
 464  * @param[out] p_dont_translate  if packet should not be translated
 465  * @param d                      optional parameter
 466  * @param e                      optional parameter
 467  */
 468 u32
 469 icmp_match_out2in_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 470                         u32 thread_index, vlib_buffer_t * b0,
 471                         ip4_header_t * ip0, ip4_address_t * mapping_addr,
 472                         u16 * mapping_port, u32 * mapping_fib_index,
 473                         nat_protocol_t * proto, void *d, void *e,
 474                         u8 * dont_translate)
 475 {
 476   u32 sw_if_index0;
 477   u32 rx_fib_index0;
 478   u8 is_addr_only;
 479   u32 next0 = ~0;
 480   int err;
 481   ip4_address_t addr;
 482   u16 port;
 483   *dont_translate = 0;
 484
 485   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 486   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 487
 488   err = icmp_get_key (b0, ip0, &addr, &port, proto);
 489   if (err != -1)
 490     {
 491       b0->error = node->errors[err];
 492       next0 = SNAT_OUT2IN_NEXT_DROP;
 493       goto out;
 494     }
 495   if (snat_static_mapping_match
 496       (sm, addr, port, rx_fib_index0, *proto, mapping_addr, mapping_port,
 497        mapping_fib_index, 1, &is_addr_only, 0, 0, 0, 0))
 498     {
 499       /* Don't NAT packet aimed at the intfc address */
 500       if (is_interface_addr (sm, node, sw_if_index0, ip0->dst_address.as_u32))
 501         {
 502           *dont_translate = 1;
 503           goto out;
 504         }
 505       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 506       next0 = SNAT_OUT2IN_NEXT_DROP;
 507       goto out;
 508     }
 509
 510   if (PREDICT_FALSE
 511       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_reply
 512        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 513            ICMP4_echo_request || !is_addr_only)
 514        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 515                                        reass.icmp_type_or_tcp_flags)))
 516     {
 517       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 518       next0 = SNAT_OUT2IN_NEXT_DROP;
 519       goto out;
 520     }
 521
 522 out:
 523   return next0;
 524 }
 525 #endif
 526
 527 #ifndef CLIB_MARCH_VARIANT
 528 u32
 529 icmp_out2in (snat_main_t * sm,
 530              vlib_buffer_t * b0,
 531              ip4_header_t * ip0,
 532              icmp46_header_t * icmp0,
 533              u32 sw_if_index0,
 534              u32 rx_fib_index0,
 535              vlib_node_runtime_t * node,
 536              u32 next0, u32 thread_index, void *d, void *e)
 537 {
 538   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 539   ip4_header_t *inner_ip0 = 0;
 540   void *l4_header = 0;
 541   icmp46_header_t *inner_icmp0;
 542   u8 dont_translate;
 543   u32 new_addr0, old_addr0;
 544   u16 old_id0, new_id0;
 545   ip_csum_t sum0;
 546   u16 checksum0;
 547   u32 next0_tmp;
 548   vlib_main_t *vm = vlib_get_main ();
 549   ip4_address_t addr;
 550   u16 port;
 551   u32 fib_index;
 552   nat_protocol_t proto;
 553
 554   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 555
 556   next0_tmp = sm->icmp_match_out2in_cb (sm, node, thread_index, b0, ip0,
 557                                         &addr, &port, &fib_index, &proto,
 558                                         d, e, &dont_translate);
 559   if (next0_tmp != ~0)
 560     next0 = next0_tmp;
 561   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 562     goto out;
 563
 564   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 565     {
 566       sum0 =
 567         ip_incremental_checksum_buffer (vm, b0,
 568                                         (u8 *) icmp0 -
 569                                         (u8 *) vlib_buffer_get_current (b0),
 570                                         ntohs (ip0->length) -
 571                                         ip4_header_bytes (ip0), 0);
 572       checksum0 = ~ip_csum_fold (sum0);
 573       if (checksum0 != 0 && checksum0 != 0xffff)
 574         {
 575           next0 = SNAT_OUT2IN_NEXT_DROP;
 576           goto out;
 577         }
 578     }
 579
 580   old_addr0 = ip0->dst_address.as_u32;
 581   new_addr0 = ip0->dst_address.as_u32 = addr.as_u32;
 582   vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
 583
 584   sum0 = ip0->checksum;
 585   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 586                          dst_address /* changed member */ );
 587   ip0->checksum = ip_csum_fold (sum0);
 588
 589
 590   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 591     {
 592       if (icmp0->checksum == 0)
 593         icmp0->checksum = 0xffff;
 594
 595       if (!icmp_type_is_error_message (icmp0->type))
 596         {
 597           new_id0 = port;
 598           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 599             {
 600               old_id0 = echo0->identifier;
 601               new_id0 = port;
 602               echo0->identifier = new_id0;
 603
 604               sum0 = icmp0->checksum;
 605               sum0 =
 606                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 607                                 identifier /* changed member */ );
 608               icmp0->checksum = ip_csum_fold (sum0);
 609             }
 610         }
 611       else
 612         {
 613           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 614           l4_header = ip4_next_header (inner_ip0);
 615
 616           if (!ip4_header_checksum_is_valid (inner_ip0))
 617             {
 618               next0 = SNAT_OUT2IN_NEXT_DROP;
 619               goto out;
 620             }
 621
 622           old_addr0 = inner_ip0->src_address.as_u32;
 623           inner_ip0->src_address = addr;
 624           new_addr0 = inner_ip0->src_address.as_u32;
 625
 626           sum0 = icmp0->checksum;
 627           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 628                                  src_address /* changed member */ );
 629           icmp0->checksum = ip_csum_fold (sum0);
 630
 631           switch (proto)
 632             {
 633             case NAT_PROTOCOL_ICMP:
 634               inner_icmp0 = (icmp46_header_t *) l4_header;
 635               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 636
 637               old_id0 = inner_echo0->identifier;
 638               new_id0 = port;
 639               inner_echo0->identifier = new_id0;
 640
 641               sum0 = icmp0->checksum;
 642               sum0 =
 643                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 644                                 identifier);
 645               icmp0->checksum = ip_csum_fold (sum0);
 646               break;
 647             case NAT_PROTOCOL_UDP:
 648             case NAT_PROTOCOL_TCP:
 649               old_id0 = ((tcp_udp_header_t *) l4_header)->src_port;
 650               new_id0 = port;
 651               ((tcp_udp_header_t *) l4_header)->src_port = new_id0;
 652
 653               sum0 = icmp0->checksum;
 654               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 655                                      src_port);
 656               icmp0->checksum = ip_csum_fold (sum0);
 657               break;
 658             default:
 659               ASSERT (0);
 660             }
 661         }
 662     }
 663
 664 out:
 665   return next0;
 666 }
 667 #endif
 668
 669 static inline u32
 670 icmp_out2in_slow_path (snat_main_t * sm,
 671                        vlib_buffer_t * b0,
 672                        ip4_header_t * ip0,
 673                        icmp46_header_t * icmp0,
 674                        u32 sw_if_index0,
 675                        u32 rx_fib_index0,
 676                        vlib_node_runtime_t * node,
 677                        u32 next0, f64 now,
 678                        u32 thread_index, snat_session_t ** p_s0)
 679 {
 680   vlib_main_t *vm = vlib_get_main ();
 681
 682   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 683                        next0, thread_index, p_s0, 0);
 684   snat_session_t *s0 = *p_s0;
 685   if (PREDICT_TRUE (next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 686     {
 687       /* Accounting */
 688       nat44_session_update_counters (s0, now,
 689                                      vlib_buffer_length_in_chain
 690                                      (vm, b0), thread_index);
 691       /* Per-user LRU list maintenance */
 692       nat44_session_update_lru (sm, s0, thread_index);
 693     }
 694   return next0;
 695 }
 696
 697 static int
 698 nat_out2in_sm_unknown_proto (snat_main_t * sm,
 699                              vlib_buffer_t * b,
 700                              ip4_header_t * ip, u32 rx_fib_index)
 701 {
 702   clib_bihash_kv_8_8_t kv, value;
 703   snat_static_mapping_t *m;
 704   u32 old_addr, new_addr;
 705   ip_csum_t sum;
 706
 707   init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 708   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 709     return 1;
 710
 711   m = pool_elt_at_index (sm->static_mappings, value.value);
 712
 713   old_addr = ip->dst_address.as_u32;
 714   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 715   sum = ip->checksum;
 716   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 717   ip->checksum = ip_csum_fold (sum);
 718
 719   vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 720   return 0;
 721 }
 722
 723 VLIB_NODE_FN (snat_out2in_node) (vlib_main_t * vm,
 724                                  vlib_node_runtime_t * node,
 725                                  vlib_frame_t * frame)
 726 {
 727   u32 n_left_from, *from, *to_next;
 728   snat_out2in_next_t next_index;
 729   u32 pkts_processed = 0;
 730   snat_main_t *sm = &snat_main;
 731   f64 now = vlib_time_now (vm);
 732   u32 thread_index = vm->thread_index;
 733   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 734   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 735     0, fragments = 0;
 736
 737   from = vlib_frame_vector_args (frame);
 738   n_left_from = frame->n_vectors;
 739   next_index = node->cached_next_index;
 740
 741   while (n_left_from > 0)
 742     {
 743       u32 n_left_to_next;
 744
 745       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 746
 747       while (n_left_from >= 4 && n_left_to_next >= 2)
 748         {
 749           u32 bi0, bi1;
 750           vlib_buffer_t *b0, *b1;
 751           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 752           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 753           u32 sw_if_index0, sw_if_index1;
 754           ip4_header_t *ip0, *ip1;
 755           ip_csum_t sum0, sum1;
 756           u32 new_addr0, old_addr0;
 757           u16 new_port0, old_port0;
 758           u32 new_addr1, old_addr1;
 759           u16 new_port1, old_port1;
 760           udp_header_t *udp0, *udp1;
 761           tcp_header_t *tcp0, *tcp1;
 762           icmp46_header_t *icmp0, *icmp1;
 763           u32 rx_fib_index0, rx_fib_index1;
 764           u32 proto0, proto1;
 765           snat_session_t *s0 = 0, *s1 = 0;
 766           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 767           u8 identity_nat0, identity_nat1;
 768           ip4_address_t sm_addr0, sm_addr1;
 769           u16 sm_port0, sm_port1;
 770           u32 sm_fib_index0, sm_fib_index1;
 771
 772           /* Prefetch next iteration. */
 773           {
 774             vlib_buffer_t *p2, *p3;
 775
 776             p2 = vlib_get_buffer (vm, from[2]);
 777             p3 = vlib_get_buffer (vm, from[3]);
 778
 779             vlib_prefetch_buffer_header (p2, LOAD);
 780             vlib_prefetch_buffer_header (p3, LOAD);
 781
 782             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 783             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
 784           }
 785
 786           /* speculatively enqueue b0 and b1 to the current next frame */
 787           to_next[0] = bi0 = from[0];
 788           to_next[1] = bi1 = from[1];
 789           from += 2;
 790           to_next += 2;
 791           n_left_from -= 2;
 792           n_left_to_next -= 2;
 793
 794           b0 = vlib_get_buffer (vm, bi0);
 795           b1 = vlib_get_buffer (vm, bi1);
 796
 797           vnet_buffer (b0)->snat.flags = 0;
 798           vnet_buffer (b1)->snat.flags = 0;
 799
 800           ip0 = vlib_buffer_get_current (b0);
 801           udp0 = ip4_next_header (ip0);
 802           tcp0 = (tcp_header_t *) udp0;
 803           icmp0 = (icmp46_header_t *) udp0;
 804
 805           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 806           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 807                                    sw_if_index0);
 808
 809           if (PREDICT_FALSE (ip0->ttl == 1))
 810             {
 811               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 812               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 813                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 814                                            0);
 815               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 816               goto trace0;
 817             }
 818
 819           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 820
 821           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 822             {
 823               if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
 824                 {
 825                   if (!sm->forwarding_enabled)
 826                     {
 827                       b0->error =
 828                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 829                       next0 = SNAT_OUT2IN_NEXT_DROP;
 830                     }
 831                 }
 832               other_packets++;
 833               goto trace0;
 834             }
 835
 836           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 837             {
 838               next0 = icmp_out2in_slow_path
 839                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 840                  next0, now, thread_index, &s0);
 841               icmp_packets++;
 842               goto trace0;
 843             }
 844
 845           init_nat_k (&kv0, ip0->dst_address,
 846                       vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
 847                       proto0);
 848           if (clib_bihash_search_8_8
 849               (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 850             {
 851               /* Try to match static mapping by external address and port,
 852                  destination address and port in packet */
 853               if (snat_static_mapping_match
 854                   (sm, ip0->dst_address,
 855                    vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
 856                    proto0, &sm_addr0, &sm_port0, &sm_fib_index0, 1, 0, 0, 0,
 857                    0, &identity_nat0))
 858                 {
 859                   /*
 860                    * Send DHCP packets to the ipv4 stack, or we won't
 861                    * be able to use dhcp client on the outside interface
 862                    */
 863                   if (PREDICT_FALSE
 864                       (proto0 == NAT_PROTOCOL_UDP
 865                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
 866                            clib_host_to_net_u16
 867                            (UDP_DST_PORT_dhcp_to_client))))
 868                     {
 869                       vnet_feature_next (&next0, b0);
 870                       goto trace0;
 871                     }
 872
 873                   if (!sm->forwarding_enabled)
 874                     {
 875                       b0->error =
 876                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 877                       next0 = SNAT_OUT2IN_NEXT_DROP;
 878                     }
 879                   goto trace0;
 880                 }
 881
 882               if (PREDICT_FALSE (identity_nat0))
 883                 goto trace0;
 884
 885               /* Create session initiated by host from external network */
 886               s0 = create_session_for_static_mapping (sm, b0,
 887                                                       sm_addr0, sm_port0,
 888                                                       sm_fib_index0,
 889                                                       ip0->dst_address,
 890                                                       vnet_buffer (b0)->
 891                                                       ip.reass.l4_dst_port,
 892                                                       rx_fib_index0, proto0,
 893                                                       node, thread_index,
 894                                                       now);
 895               if (!s0)
 896                 {
 897                   next0 = SNAT_OUT2IN_NEXT_DROP;
 898                   goto trace0;
 899                 }
 900             }
 901           else
 902             s0 = pool_elt_at_index (tsm->sessions, value0.value);
 903
 904           old_addr0 = ip0->dst_address.as_u32;
 905           ip0->dst_address = s0->in2out.addr;
 906           new_addr0 = ip0->dst_address.as_u32;
 907           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 908
 909           sum0 = ip0->checksum;
 910           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 911                                  ip4_header_t,
 912                                  dst_address /* changed member */ );
 913           ip0->checksum = ip_csum_fold (sum0);
 914
 915           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 916             {
 917               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 918                 {
 919                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 920                   new_port0 = udp0->dst_port = s0->in2out.port;
 921                   sum0 = tcp0->checksum;
 922                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 923                                          ip4_header_t,
 924                                          dst_address /* changed member */ );
 925
 926                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
 927                                          ip4_header_t /* cheat */ ,
 928                                          length /* changed member */ );
 929                   tcp0->checksum = ip_csum_fold (sum0);
 930                 }
 931               tcp_packets++;
 932             }
 933           else
 934             {
 935               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 936                 {
 937                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 938                   new_port0 = udp0->dst_port = s0->in2out.port;
 939                   if (PREDICT_FALSE (udp0->checksum))
 940                     {
 941                       sum0 = udp0->checksum;
 942                       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address      /* changed member */
 943                         );
 944                       sum0 =
 945                         ip_csum_update (sum0, old_port0, new_port0,
 946                                         ip4_header_t /* cheat */ ,
 947                                         length /* changed member */ );
 948                       udp0->checksum = ip_csum_fold (sum0);
 949                     }
 950                 }
 951               udp_packets++;
 952             }
 953
 954           /* Accounting */
 955           nat44_session_update_counters (s0, now,
 956                                          vlib_buffer_length_in_chain (vm, b0),
 957                                          thread_index);
 958           /* Per-user LRU list maintenance */
 959           nat44_session_update_lru (sm, s0, thread_index);
 960         trace0:
 961
 962           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 963                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 964             {
 965               snat_out2in_trace_t *t =
 966                 vlib_add_trace (vm, node, b0, sizeof (*t));
 967               t->sw_if_index = sw_if_index0;
 968               t->next_index = next0;
 969               t->session_index = ~0;
 970               if (s0)
 971                 t->session_index =
 972                   s0 - sm->per_thread_data[thread_index].sessions;
 973             }
 974
 975           pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
 976
 977
 978           ip1 = vlib_buffer_get_current (b1);
 979           udp1 = ip4_next_header (ip1);
 980           tcp1 = (tcp_header_t *) udp1;
 981           icmp1 = (icmp46_header_t *) udp1;
 982
 983           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 984           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 985                                    sw_if_index1);
 986
 987           if (PREDICT_FALSE (ip1->ttl == 1))
 988             {
 989               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 990               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 991                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 992                                            0);
 993               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 994               goto trace1;
 995             }
 996
 997           proto1 = ip_proto_to_nat_proto (ip1->protocol);
 998
 999           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1000             {
1001               if (nat_out2in_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1002                 {
1003                   if (!sm->forwarding_enabled)
1004                     {
1005                       b1->error =
1006                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1007                       next1 = SNAT_OUT2IN_NEXT_DROP;
1008                     }
1009                 }
1010               other_packets++;
1011               goto trace1;
1012             }
1013
1014           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1015             {
1016               next1 = icmp_out2in_slow_path
1017                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1018                  next1, now, thread_index, &s1);
1019               icmp_packets++;
1020               goto trace1;
1021             }
1022
1023           init_nat_k (&kv1, ip1->dst_address,
1024                       vnet_buffer (b1)->ip.reass.l4_dst_port, rx_fib_index1,
1025                       proto1);
1026
1027           if (clib_bihash_search_8_8
1028               (&sm->per_thread_data[thread_index].out2in, &kv1, &value1))
1029             {
1030               /* Try to match static mapping by external address and port,
1031                  destination address and port in packet */
1032               if (snat_static_mapping_match
1033                   (sm, ip1->dst_address,
1034                    vnet_buffer (b1)->ip.reass.l4_dst_port, proto1,
1035                    rx_fib_index1, &sm_addr1, &sm_port1, &sm_fib_index1, 1, 0,
1036                    0, 0, 0, &identity_nat1))
1037                 {
1038                   /*
1039                    * Send DHCP packets to the ipv4 stack, or we won't
1040                    * be able to use dhcp client on the outside interface
1041                    */
1042                   if (PREDICT_FALSE
1043                       (proto1 == NAT_PROTOCOL_UDP
1044                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1045                            clib_host_to_net_u16
1046                            (UDP_DST_PORT_dhcp_to_client))))
1047                     {
1048                       vnet_feature_next (&next1, b1);
1049                       goto trace1;
1050                     }
1051
1052                   if (!sm->forwarding_enabled)
1053                     {
1054                       b1->error =
1055                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1056                       next1 = SNAT_OUT2IN_NEXT_DROP;
1057                     }
1058                   goto trace1;
1059                 }
1060
1061               if (PREDICT_FALSE (identity_nat1))
1062                 goto trace1;
1063
1064               /* Create session initiated by host from external network */
1065               s1 =
1066                 create_session_for_static_mapping (sm, b1, sm_addr1, sm_port1,
1067                                                    sm_fib_index1,
1068                                                    ip1->dst_address,
1069                                                    vnet_buffer (b1)->ip.
1070                                                    reass.l4_dst_port,
1071                                                    rx_fib_index1, proto1,
1072                                                    node, thread_index, now);
1073               if (!s1)
1074                 {
1075                   next1 = SNAT_OUT2IN_NEXT_DROP;
1076                   goto trace1;
1077                 }
1078             }
1079           else
1080             s1 =
1081               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1082                                  value1.value);
1083
1084           old_addr1 = ip1->dst_address.as_u32;
1085           ip1->dst_address = s1->in2out.addr;
1086           new_addr1 = ip1->dst_address.as_u32;
1087           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1088
1089           sum1 = ip1->checksum;
1090           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1091                                  ip4_header_t,
1092                                  dst_address /* changed member */ );
1093           ip1->checksum = ip_csum_fold (sum1);
1094
1095           if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1096             {
1097               if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1098                 {
1099                   old_port1 = vnet_buffer (b1)->ip.reass.l4_dst_port;
1100                   new_port1 = udp1->dst_port = s1->in2out.port;
1101
1102                   sum1 = tcp1->checksum;
1103                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1104                                          ip4_header_t,
1105                                          dst_address /* changed member */ );
1106
1107                   sum1 = ip_csum_update (sum1, old_port1, new_port1,
1108                                          ip4_header_t /* cheat */ ,
1109                                          length /* changed member */ );
1110                   tcp1->checksum = ip_csum_fold (sum1);
1111                 }
1112               tcp_packets++;
1113             }
1114           else
1115             {
1116               if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1117                 {
1118                   old_port1 = vnet_buffer (b1)->ip.reass.l4_dst_port;
1119                   new_port1 = udp1->dst_port = s1->in2out.port;
1120                   if (PREDICT_FALSE (udp1->checksum))
1121                     {
1122
1123                       sum1 = udp1->checksum;
1124                       sum1 =
1125                         ip_csum_update (sum1, old_addr1, new_addr1,
1126                                         ip4_header_t,
1127                                         dst_address /* changed member */ );
1128                       sum1 =
1129                         ip_csum_update (sum1, old_port1, new_port1,
1130                                         ip4_header_t /* cheat */ ,
1131                                         length /* changed member */ );
1132                       udp1->checksum = ip_csum_fold (sum1);
1133                     }
1134                 }
1135               udp_packets++;
1136             }
1137
1138           /* Accounting */
1139           nat44_session_update_counters (s1, now,
1140                                          vlib_buffer_length_in_chain (vm, b1),
1141                                          thread_index);
1142           /* Per-user LRU list maintenance */
1143           nat44_session_update_lru (sm, s1, thread_index);
1144         trace1:
1145
1146           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1147                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1148             {
1149               snat_out2in_trace_t *t =
1150                 vlib_add_trace (vm, node, b1, sizeof (*t));
1151               t->sw_if_index = sw_if_index1;
1152               t->next_index = next1;
1153               t->session_index = ~0;
1154               if (s1)
1155                 t->session_index =
1156                   s1 - sm->per_thread_data[thread_index].sessions;
1157             }
1158
1159           pkts_processed += next1 == SNAT_OUT2IN_NEXT_LOOKUP;
1160
1161           /* verify speculative enqueues, maybe switch current next frame */
1162           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1163                                            to_next, n_left_to_next,
1164                                            bi0, bi1, next0, next1);
1165         }
1166
1167       while (n_left_from > 0 && n_left_to_next > 0)
1168         {
1169           u32 bi0;
1170           vlib_buffer_t *b0;
1171           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1172           u32 sw_if_index0;
1173           ip4_header_t *ip0;
1174           ip_csum_t sum0;
1175           u32 new_addr0, old_addr0;
1176           u16 new_port0, old_port0;
1177           udp_header_t *udp0;
1178           tcp_header_t *tcp0;
1179           icmp46_header_t *icmp0;
1180           u32 rx_fib_index0;
1181           u32 proto0;
1182           snat_session_t *s0 = 0;
1183           clib_bihash_kv_8_8_t kv0, value0;
1184           u8 identity_nat0;
1185           ip4_address_t sm_addr0;
1186           u16 sm_port0;
1187           u32 sm_fib_index0;
1188
1189           /* speculatively enqueue b0 to the current next frame */
1190           bi0 = from[0];
1191           to_next[0] = bi0;
1192           from += 1;
1193           to_next += 1;
1194           n_left_from -= 1;
1195           n_left_to_next -= 1;
1196
1197           b0 = vlib_get_buffer (vm, bi0);
1198
1199           vnet_buffer (b0)->snat.flags = 0;
1200
1201           ip0 = vlib_buffer_get_current (b0);
1202           udp0 = ip4_next_header (ip0);
1203           tcp0 = (tcp_header_t *) udp0;
1204           icmp0 = (icmp46_header_t *) udp0;
1205
1206           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1207           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1208                                    sw_if_index0);
1209
1210           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1211
1212           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1213             {
1214               if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1215                 {
1216                   if (!sm->forwarding_enabled)
1217                     {
1218                       b0->error =
1219                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1220                       next0 = SNAT_OUT2IN_NEXT_DROP;
1221                     }
1222                 }
1223               other_packets++;
1224               goto trace00;
1225             }
1226
1227           if (PREDICT_FALSE (ip0->ttl == 1))
1228             {
1229               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1230               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1231                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1232                                            0);
1233               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1234               goto trace00;
1235             }
1236
1237           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1238             {
1239               next0 = icmp_out2in_slow_path
1240                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1241                  next0, now, thread_index, &s0);
1242               icmp_packets++;
1243               goto trace00;
1244             }
1245
1246           init_nat_k (&kv0, ip0->dst_address,
1247                       vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1248                       proto0);
1249
1250           if (clib_bihash_search_8_8
1251               (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
1252             {
1253               /* Try to match static mapping by external address and port,
1254                  destination address and port in packet */
1255               if (snat_static_mapping_match
1256                   (sm, ip0->dst_address,
1257                    vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1258                    proto0, &sm_addr0, &sm_port0, &sm_fib_index0, 1, 0, 0, 0,
1259                    0, &identity_nat0))
1260                 {
1261                   /*
1262                    * Send DHCP packets to the ipv4 stack, or we won't
1263                    * be able to use dhcp client on the outside interface
1264                    */
1265                   if (PREDICT_FALSE
1266                       (proto0 == NAT_PROTOCOL_UDP
1267                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1268                            clib_host_to_net_u16
1269                            (UDP_DST_PORT_dhcp_to_client))))
1270                     {
1271                       vnet_feature_next (&next0, b0);
1272                       goto trace00;
1273                     }
1274
1275                   if (!sm->forwarding_enabled)
1276                     {
1277                       b0->error =
1278                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1279                       next0 = SNAT_OUT2IN_NEXT_DROP;
1280                     }
1281                   goto trace00;
1282                 }
1283
1284               if (PREDICT_FALSE (identity_nat0))
1285                 goto trace00;
1286
1287               /* Create session initiated by host from external network */
1288               s0 = create_session_for_static_mapping (sm, b0,
1289                                                       sm_addr0, sm_port0,
1290                                                       sm_fib_index0,
1291                                                       ip0->dst_address,
1292                                                       vnet_buffer (b0)->
1293                                                       ip.reass.l4_dst_port,
1294                                                       rx_fib_index0, proto0,
1295                                                       node, thread_index,
1296                                                       now);
1297               if (!s0)
1298                 {
1299                   next0 = SNAT_OUT2IN_NEXT_DROP;
1300                   goto trace00;
1301                 }
1302             }
1303           else
1304             s0 =
1305               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1306                                  value0.value);
1307
1308           old_addr0 = ip0->dst_address.as_u32;
1309           ip0->dst_address = s0->in2out.addr;
1310           new_addr0 = ip0->dst_address.as_u32;
1311           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1312
1313           sum0 = ip0->checksum;
1314           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1315                                  ip4_header_t,
1316                                  dst_address /* changed member */ );
1317           ip0->checksum = ip_csum_fold (sum0);
1318
1319           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1320             {
1321               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1322                 {
1323                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1324                   new_port0 = udp0->dst_port = s0->in2out.port;
1325
1326                   sum0 = tcp0->checksum;
1327                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1328                                          ip4_header_t,
1329                                          dst_address /* changed member */ );
1330
1331                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1332                                          ip4_header_t /* cheat */ ,
1333                                          length /* changed member */ );
1334                   tcp0->checksum = ip_csum_fold (sum0);
1335                 }
1336               tcp_packets++;
1337             }
1338           else
1339             {
1340               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1341                 {
1342                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1343                   new_port0 = udp0->dst_port = s0->in2out.port;
1344                   if (PREDICT_FALSE (udp0->checksum))
1345                     {
1346                       sum0 = udp0->checksum;
1347                       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address      /* changed member */
1348                         );
1349                       sum0 =
1350                         ip_csum_update (sum0, old_port0, new_port0,
1351                                         ip4_header_t /* cheat */ ,
1352                                         length /* changed member */ );
1353                       udp0->checksum = ip_csum_fold (sum0);
1354                     }
1355                 }
1356               udp_packets++;
1357             }
1358
1359           /* Accounting */
1360           nat44_session_update_counters (s0, now,
1361                                          vlib_buffer_length_in_chain (vm, b0),
1362                                          thread_index);
1363           /* Per-user LRU list maintenance */
1364           nat44_session_update_lru (sm, s0, thread_index);
1365         trace00:
1366
1367           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1368                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1369             {
1370               snat_out2in_trace_t *t =
1371                 vlib_add_trace (vm, node, b0, sizeof (*t));
1372               t->sw_if_index = sw_if_index0;
1373               t->next_index = next0;
1374               t->session_index = ~0;
1375               if (s0)
1376                 t->session_index =
1377                   s0 - sm->per_thread_data[thread_index].sessions;
1378             }
1379
1380           pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
1381
1382           /* verify speculative enqueue, maybe switch current next frame */
1383           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1384                                            to_next, n_left_to_next,
1385                                            bi0, next0);
1386         }
1387
1388       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1389     }
1390
1391   vlib_node_increment_counter (vm, sm->out2in_node_index,
1392                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1393                                pkts_processed);
1394   vlib_node_increment_counter (vm, sm->out2in_node_index,
1395                                SNAT_OUT2IN_ERROR_TCP_PACKETS, tcp_packets);
1396   vlib_node_increment_counter (vm, sm->out2in_node_index,
1397                                SNAT_OUT2IN_ERROR_UDP_PACKETS, udp_packets);
1398   vlib_node_increment_counter (vm, sm->out2in_node_index,
1399                                SNAT_OUT2IN_ERROR_ICMP_PACKETS, icmp_packets);
1400   vlib_node_increment_counter (vm, sm->out2in_node_index,
1401                                SNAT_OUT2IN_ERROR_OTHER_PACKETS,
1402                                other_packets);
1403   vlib_node_increment_counter (vm, sm->out2in_node_index,
1404                                SNAT_OUT2IN_ERROR_FRAGMENTS, fragments);
1405
1406   return frame->n_vectors;
1407 }
1408
1409 /* *INDENT-OFF* */
1410 VLIB_REGISTER_NODE (snat_out2in_node) = {
1411   .name = "nat44-out2in",
1412   .vector_size = sizeof (u32),
1413   .format_trace = format_snat_out2in_trace,
1414   .type = VLIB_NODE_TYPE_INTERNAL,
1415
1416   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1417   .error_strings = snat_out2in_error_strings,
1418
1419   .runtime_data_bytes = sizeof (snat_runtime_t),
1420
1421   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1422
1423   /* edit / add dispositions here */
1424   .next_nodes = {
1425     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1426     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1427     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1428   },
1429 };
1430 /* *INDENT-ON* */
1431
1432 VLIB_NODE_FN (snat_out2in_fast_node) (vlib_main_t * vm,
1433                                       vlib_node_runtime_t * node,
1434                                       vlib_frame_t * frame)
1435 {
1436   u32 n_left_from, *from, *to_next;
1437   snat_out2in_next_t next_index;
1438   u32 pkts_processed = 0;
1439   snat_main_t *sm = &snat_main;
1440
1441   from = vlib_frame_vector_args (frame);
1442   n_left_from = frame->n_vectors;
1443   next_index = node->cached_next_index;
1444
1445   while (n_left_from > 0)
1446     {
1447       u32 n_left_to_next;
1448
1449       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1450
1451       while (n_left_from > 0 && n_left_to_next > 0)
1452         {
1453           u32 bi0;
1454           vlib_buffer_t *b0;
1455           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1456           u32 sw_if_index0;
1457           ip4_header_t *ip0;
1458           ip_csum_t sum0;
1459           u32 new_addr0, old_addr0;
1460           u16 new_port0, old_port0;
1461           udp_header_t *udp0;
1462           tcp_header_t *tcp0;
1463           icmp46_header_t *icmp0;
1464           u32 proto0;
1465           u32 rx_fib_index0;
1466           ip4_address_t sm_addr0;
1467           u16 sm_port0;
1468           u32 sm_fib_index0;
1469
1470           /* speculatively enqueue b0 to the current next frame */
1471           bi0 = from[0];
1472           to_next[0] = bi0;
1473           from += 1;
1474           to_next += 1;
1475           n_left_from -= 1;
1476           n_left_to_next -= 1;
1477
1478           b0 = vlib_get_buffer (vm, bi0);
1479
1480           ip0 = vlib_buffer_get_current (b0);
1481           udp0 = ip4_next_header (ip0);
1482           tcp0 = (tcp_header_t *) udp0;
1483           icmp0 = (icmp46_header_t *) udp0;
1484
1485           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1486           rx_fib_index0 =
1487             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1488
1489           vnet_feature_next (&next0, b0);
1490
1491           if (PREDICT_FALSE (ip0->ttl == 1))
1492             {
1493               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1494               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1495                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1496                                            0);
1497               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1498               goto trace00;
1499             }
1500
1501           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1502
1503           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1504             goto trace00;
1505
1506           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1507             {
1508               next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0,
1509                                    rx_fib_index0, node, next0, ~0, 0, 0);
1510               goto trace00;
1511             }
1512
1513           if (snat_static_mapping_match
1514               (sm, ip0->dst_address, udp0->dst_port, rx_fib_index0, proto0,
1515                &sm_addr0, &sm_port0, &sm_fib_index0, 1, 0, 0, 0, 0, 0))
1516             {
1517               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1518               goto trace00;
1519             }
1520
1521           new_addr0 = sm_addr0.as_u32;
1522           new_port0 = sm_port0;
1523           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm_fib_index0;
1524           old_addr0 = ip0->dst_address.as_u32;
1525           ip0->dst_address.as_u32 = new_addr0;
1526
1527           sum0 = ip0->checksum;
1528           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1529                                  ip4_header_t,
1530                                  dst_address /* changed member */ );
1531           ip0->checksum = ip_csum_fold (sum0);
1532
1533           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1534             {
1535               old_port0 = udp0->dst_port;
1536               udp0->dst_port = new_port0;
1537
1538               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1539                 {
1540                   sum0 = tcp0->checksum;
1541                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1542                                          ip4_header_t,
1543                                          dst_address /* changed member */ );
1544                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1545                                          ip4_header_t /* cheat */ ,
1546                                          length /* changed member */ );
1547                   tcp0->checksum = ip_csum_fold (sum0);
1548                 }
1549               else if (udp0->checksum)
1550                 {
1551                   sum0 = udp0->checksum;
1552                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1553                                          ip4_header_t,
1554                                          dst_address /* changed member */ );
1555                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1556                                          ip4_header_t /* cheat */ ,
1557                                          length /* changed member */ );
1558                   udp0->checksum = ip_csum_fold (sum0);
1559                 }
1560             }
1561           else
1562             {
1563               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1564                 {
1565                   sum0 = tcp0->checksum;
1566                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1567                                          ip4_header_t,
1568                                          dst_address /* changed member */ );
1569                   tcp0->checksum = ip_csum_fold (sum0);
1570                 }
1571               else if (udp0->checksum)
1572                 {
1573                   sum0 = udp0->checksum;
1574                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1575                                          ip4_header_t,
1576                                          dst_address /* changed member */ );
1577                   udp0->checksum = ip_csum_fold (sum0);
1578                 }
1579             }
1580
1581         trace00:
1582
1583           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1584                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1585             {
1586               snat_out2in_trace_t *t =
1587                 vlib_add_trace (vm, node, b0, sizeof (*t));
1588               t->sw_if_index = sw_if_index0;
1589               t->next_index = next0;
1590             }
1591
1592           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1593
1594           /* verify speculative enqueue, maybe switch current next frame */
1595           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1596                                            to_next, n_left_to_next,
1597                                            bi0, next0);
1598         }
1599
1600       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1601     }
1602
1603   vlib_node_increment_counter (vm, sm->out2in_fast_node_index,
1604                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1605                                pkts_processed);
1606   return frame->n_vectors;
1607 }
1608
1609 /* *INDENT-OFF* */
1610 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1611   .name = "nat44-out2in-fast",
1612   .vector_size = sizeof (u32),
1613   .format_trace = format_snat_out2in_fast_trace,
1614   .type = VLIB_NODE_TYPE_INTERNAL,
1615
1616   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1617   .error_strings = snat_out2in_error_strings,
1618
1619   .runtime_data_bytes = sizeof (snat_runtime_t),
1620
1621   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1622
1623   /* edit / add dispositions here */
1624   .next_nodes = {
1625     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1626     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1627     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1628   },
1629 };
1630 /* *INDENT-ON* */
1631
1632 /*
1633  * fd.io coding-style-patch-verification: ON
1634  *
1635  * Local Variables:
1636  * eval: (c-set-style "gnu")
1637  * End:
1638  */