src/plugins/nat/out2in.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent outside to inside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23
  24 #include <vnet/ip/ip.h>
  25 #include <vnet/udp/udp.h>
  26 #include <vnet/ethernet/ethernet.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 #include <vppinfra/hash.h>
  36 #include <vppinfra/error.h>
  37 #include <vppinfra/elog.h>
  38
  39 typedef struct
  40 {
  41   u32 sw_if_index;
  42   u32 next_index;
  43   u32 session_index;
  44 } snat_out2in_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 *
  48 format_snat_out2in_trace (u8 * s, va_list * args)
  49 {
  50   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  51   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  52   snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
  53
  54   s =
  55     format (s,
  56             "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
  57             t->sw_if_index, t->next_index, t->session_index);
  58   return s;
  59 }
  60
  61 static u8 *
  62 format_snat_out2in_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_out2in_trace_t *t = va_arg (*args, snat_out2in_trace_t *);
  67
  68   s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70   return s;
  71 }
  72
  73 #define foreach_snat_out2in_error                       \
  74 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
  75 _(OUT2IN_PACKETS, "good out2in packets processed")      \
  76 _(OUT_OF_PORTS, "out of ports")                         \
  77 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
  78 _(NO_TRANSLATION, "no translation")                     \
  79 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
  80 _(DROP_FRAGMENT, "drop fragment")                       \
  81 _(MAX_REASS, "maximum reassemblies exceeded")           \
  82 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
  83 _(TCP_PACKETS, "TCP packets")                           \
  84 _(UDP_PACKETS, "UDP packets")                           \
  85 _(ICMP_PACKETS, "ICMP packets")                         \
  86 _(OTHER_PACKETS, "other protocol packets")              \
  87 _(FRAGMENTS, "fragments")                               \
  88 _(CACHED_FRAGMENTS, "cached fragments")                 \
  89 _(PROCESSED_FRAGMENTS, "processed fragments")
  90
  91 typedef enum
  92 {
  93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
  94   foreach_snat_out2in_error
  95 #undef _
  96     SNAT_OUT2IN_N_ERROR,
  97 } snat_out2in_error_t;
  98
  99 static char *snat_out2in_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_snat_out2in_error
 102 #undef _
 103 };
 104
 105 typedef enum
 106 {
 107   SNAT_OUT2IN_NEXT_DROP,
 108   SNAT_OUT2IN_NEXT_LOOKUP,
 109   SNAT_OUT2IN_NEXT_ICMP_ERROR,
 110   SNAT_OUT2IN_N_NEXT,
 111 } snat_out2in_next_t;
 112
 113 #ifndef CLIB_MARCH_VARIANT
 114 int
 115 nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 116 {
 117   snat_main_t *sm = &snat_main;
 118   nat44_is_idle_session_ctx_t *ctx = arg;
 119   snat_session_t *s;
 120   u64 sess_timeout_time;
 121   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
 122                                                        ctx->thread_index);
 123   clib_bihash_kv_8_8_t s_kv;
 124
 125   s = pool_elt_at_index (tsm->sessions, kv->value);
 126   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 127   if (ctx->now >= sess_timeout_time)
 128     {
 129       init_nat_i2o_k (&s_kv, s);
 130       if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
 131         nat_elog_warn ("out2in key del failed");
 132
 133       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 134                                            s->in2out.addr.as_u32,
 135                                            s->out2in.addr.as_u32,
 136                                            s->nat_proto,
 137                                            s->in2out.port,
 138                                            s->out2in.port,
 139                                            s->in2out.fib_index);
 140
 141       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 142                                &s->in2out.addr, s->in2out.port,
 143                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 144
 145       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 146                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 147                    ctx->thread_index);
 148
 149       if (!snat_is_session_static (s))
 150         snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 151                                             &s->out2in.addr, s->out2in.port,
 152                                             s->nat_proto);
 153
 154       nat44_delete_session (sm, s, ctx->thread_index);
 155       return 1;
 156     }
 157
 158   return 0;
 159 }
 160 #endif
 161
 162 /**
 163  * @brief Create session for static mapping.
 164  *
 165  * Create NAT session initiated by host from external network with static
 166  * mapping.
 167  *
 168  * @param sm     NAT main.
 169  * @param b0     Vlib buffer.
 170  * @param in2out In2out NAT44 session key.
 171  * @param out2in Out2in NAT44 session key.
 172  * @param node   Vlib node.
 173  *
 174  * @returns SNAT session if successfully created otherwise 0.
 175  */
 176 static inline snat_session_t *
 177 create_session_for_static_mapping (snat_main_t * sm,
 178                                    vlib_buffer_t * b0,
 179                                    ip4_address_t i2o_addr,
 180                                    u16 i2o_port,
 181                                    u32 i2o_fib_index,
 182                                    ip4_address_t o2i_addr,
 183                                    u16 o2i_port,
 184                                    u32 o2i_fib_index,
 185                                    nat_protocol_t proto,
 186                                    vlib_node_runtime_t * node,
 187                                    u32 thread_index, f64 now)
 188 {
 189   snat_user_t *u;
 190   snat_session_t *s;
 191   clib_bihash_kv_8_8_t kv0;
 192   ip4_header_t *ip0;
 193   udp_header_t *udp0;
 194   nat44_is_idle_session_ctx_t ctx0;
 195
 196   if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 197     {
 198       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
 199       nat_elog_notice ("maximum sessions exceeded");
 200       return 0;
 201     }
 202
 203   ip0 = vlib_buffer_get_current (b0);
 204   udp0 = ip4_next_header (ip0);
 205
 206   u = nat_user_get_or_create (sm, &i2o_addr, i2o_fib_index, thread_index);
 207   if (!u)
 208     {
 209       nat_elog_warn ("create NAT user failed");
 210       return 0;
 211     }
 212
 213   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
 214   if (!s)
 215     {
 216       nat44_delete_user_with_no_session (sm, u, thread_index);
 217       nat_elog_warn ("create NAT session failed");
 218       return 0;
 219     }
 220
 221   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 222   s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
 223   s->ext_host_port = udp0->src_port;
 224   user_session_increment (sm, u, 1 /* static */ );
 225   s->in2out.addr = i2o_addr;
 226   s->in2out.port = i2o_port;
 227   s->in2out.fib_index = i2o_fib_index;
 228   s->out2in.addr = o2i_addr;
 229   s->out2in.port = o2i_port;
 230   s->out2in.fib_index = o2i_fib_index;
 231   s->nat_proto = proto;
 232
 233   /* Add to translation hashes */
 234   ctx0.now = now;
 235   ctx0.thread_index = thread_index;
 236   init_nat_i2o_kv (&kv0, s, s - sm->per_thread_data[thread_index].sessions);
 237   if (clib_bihash_add_or_overwrite_stale_8_8
 238       (&sm->per_thread_data[thread_index].in2out, &kv0,
 239        nat44_i2o_is_idle_session_cb, &ctx0))
 240     nat_elog_notice ("in2out key add failed");
 241
 242   init_nat_o2i_kv (&kv0, s, s - sm->per_thread_data[thread_index].sessions);
 243   if (clib_bihash_add_or_overwrite_stale_8_8
 244       (&sm->per_thread_data[thread_index].out2in, &kv0,
 245        nat44_o2i_is_idle_session_cb, &ctx0))
 246     nat_elog_notice ("out2in key add failed");
 247
 248   /* log NAT event */
 249   snat_ipfix_logging_nat44_ses_create (thread_index,
 250                                        s->in2out.addr.as_u32,
 251                                        s->out2in.addr.as_u32,
 252                                        s->nat_proto,
 253                                        s->in2out.port,
 254                                        s->out2in.port, s->in2out.fib_index);
 255
 256   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
 257                            &s->in2out.addr, s->in2out.port, &s->out2in.addr,
 258                            s->out2in.port, s->nat_proto);
 259
 260   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 261                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 262                &s->ext_host_nat_addr, s->ext_host_nat_port,
 263                s->nat_proto, s->in2out.fib_index, s->flags, thread_index, 0);
 264
 265   return s;
 266 }
 267
 268 #ifndef CLIB_MARCH_VARIANT
 269 static_always_inline snat_out2in_error_t
 270 icmp_get_key (vlib_buffer_t * b, ip4_header_t * ip0,
 271               ip4_address_t * addr, u16 * port, nat_protocol_t * nat_proto)
 272 {
 273   icmp46_header_t *icmp0;
 274   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 275   ip4_header_t *inner_ip0;
 276   void *l4_header = 0;
 277   icmp46_header_t *inner_icmp0;
 278
 279   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 280   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 281
 282   if (!icmp_type_is_error_message
 283       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 284     {
 285       *nat_proto = NAT_PROTOCOL_ICMP;
 286       *addr = ip0->dst_address;
 287       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 288     }
 289   else
 290     {
 291       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 292       l4_header = ip4_next_header (inner_ip0);
 293       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 294       *addr = inner_ip0->src_address;
 295       switch (*nat_proto)
 296         {
 297         case NAT_PROTOCOL_ICMP:
 298           inner_icmp0 = (icmp46_header_t *) l4_header;
 299           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 300           *port = inner_echo0->identifier;
 301           break;
 302         case NAT_PROTOCOL_UDP:
 303         case NAT_PROTOCOL_TCP:
 304           *port = ((tcp_udp_header_t *) l4_header)->src_port;
 305           break;
 306         default:
 307           return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
 308         }
 309     }
 310   return -1;                    /* success */
 311 }
 312
 313 /**
 314  * Get address and port values to be used for ICMP packet translation
 315  * and create session if needed
 316  *
 317  * @param[in,out] sm             NAT main
 318  * @param[in,out] node           NAT node runtime
 319  * @param[in] thread_index       thread index
 320  * @param[in,out] b0             buffer containing packet to be translated
 321  * @param[in,out] ip0            ip header
 322  * @param[out] p_proto           protocol used for matching
 323  * @param[out] p_value           address and port after NAT translation
 324  * @param[out] p_dont_translate  if packet should not be translated
 325  * @param d                      optional parameter
 326  * @param e                      optional parameter
 327  */
 328 u32
 329 icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
 330                         u32 thread_index, vlib_buffer_t * b0,
 331                         ip4_header_t * ip0, ip4_address_t * addr,
 332                         u16 * port, u32 * fib_index,
 333                         nat_protocol_t * proto, void *d, void *e,
 334                         u8 * dont_translate)
 335 {
 336   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 337   u32 sw_if_index0;
 338   snat_session_t *s0 = 0;
 339   clib_bihash_kv_8_8_t kv0, value0;
 340   u8 is_addr_only;
 341   u32 next0 = ~0;
 342   int err;
 343   u8 identity_nat;
 344   vlib_main_t *vm = vlib_get_main ();
 345   *dont_translate = 0;
 346
 347   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 348   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 349
 350   *proto = 0;
 351
 352   err = icmp_get_key (b0, ip0, addr, port, proto);
 353   if (err != -1)
 354     {
 355       b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 356       next0 = SNAT_OUT2IN_NEXT_DROP;
 357       goto out;
 358     }
 359
 360   ip4_address_t mapping_addr;
 361   u16 mapping_port;
 362   u32 mapping_fib_index;
 363
 364   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 365   if (clib_bihash_search_8_8 (&tsm->out2in, &kv0, &value0))
 366     {
 367       /* Try to match static mapping by external address and port,
 368          destination address and port in packet */
 369       if (snat_static_mapping_match
 370           (sm, *addr, *port, *fib_index, *proto,
 371            &mapping_addr, &mapping_port, &mapping_fib_index, 1, &is_addr_only,
 372            0, 0, 0, &identity_nat))
 373         {
 374           if (!sm->forwarding_enabled)
 375             {
 376               /* Don't NAT packet aimed at the intfc address */
 377               if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index0,
 378                                                     ip0->dst_address.as_u32)))
 379                 {
 380                   *dont_translate = 1;
 381                   goto out;
 382                 }
 383               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 384               next0 = SNAT_OUT2IN_NEXT_DROP;
 385               goto out;
 386             }
 387           else
 388             {
 389               *dont_translate = 1;
 390               goto out;
 391             }
 392         }
 393
 394       if (PREDICT_FALSE
 395           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 396            ICMP4_echo_reply
 397            && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 398                ICMP4_echo_request || !is_addr_only)))
 399         {
 400           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 401           next0 = SNAT_OUT2IN_NEXT_DROP;
 402           goto out;
 403         }
 404
 405       if (PREDICT_FALSE (identity_nat))
 406         {
 407           *dont_translate = 1;
 408           goto out;
 409         }
 410       /* Create session initiated by host from external network */
 411       s0 =
 412         create_session_for_static_mapping (sm, b0, mapping_addr, mapping_port,
 413                                            mapping_fib_index, *addr, *port,
 414                                            *fib_index, *proto, node,
 415                                            thread_index, vlib_time_now (vm));
 416
 417       if (!s0)
 418         {
 419           next0 = SNAT_OUT2IN_NEXT_DROP;
 420           goto out;
 421         }
 422     }
 423   else
 424     {
 425       if (PREDICT_FALSE
 426           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 427            ICMP4_echo_reply
 428            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 429            ICMP4_echo_request
 430            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 431                                            reass.icmp_type_or_tcp_flags)))
 432         {
 433           b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 434           next0 = SNAT_OUT2IN_NEXT_DROP;
 435           goto out;
 436         }
 437
 438       s0 = pool_elt_at_index (tsm->sessions, value0.value);
 439     }
 440
 441 out:
 442   if (s0)
 443     {
 444       *addr = s0->in2out.addr;
 445       *port = s0->in2out.port;
 446       *fib_index = s0->in2out.fib_index;
 447     }
 448   if (d)
 449     *(snat_session_t **) d = s0;
 450   return next0;
 451 }
 452 #endif
 453
 454 #ifndef CLIB_MARCH_VARIANT
 455 /**
 456  * Get address and port values to be used for ICMP packet translation
 457  *
 458  * @param[in] sm                 NAT main
 459  * @param[in,out] node           NAT node runtime
 460  * @param[in] thread_index       thread index
 461  * @param[in,out] b0             buffer containing packet to be translated
 462  * @param[in,out] ip0            ip header
 463  * @param[out] p_proto           protocol used for matching
 464  * @param[out] p_value           address and port after NAT translation
 465  * @param[out] p_dont_translate  if packet should not be translated
 466  * @param d                      optional parameter
 467  * @param e                      optional parameter
 468  */
 469 u32
 470 icmp_match_out2in_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 471                         u32 thread_index, vlib_buffer_t * b0,
 472                         ip4_header_t * ip0, ip4_address_t * mapping_addr,
 473                         u16 * mapping_port, u32 * mapping_fib_index,
 474                         nat_protocol_t * proto, void *d, void *e,
 475                         u8 * dont_translate)
 476 {
 477   u32 sw_if_index0;
 478   u32 rx_fib_index0;
 479   u8 is_addr_only;
 480   u32 next0 = ~0;
 481   int err;
 482   ip4_address_t addr;
 483   u16 port;
 484   *dont_translate = 0;
 485
 486   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 487   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 488
 489   err = icmp_get_key (b0, ip0, &addr, &port, proto);
 490   if (err != -1)
 491     {
 492       b0->error = node->errors[err];
 493       next0 = SNAT_OUT2IN_NEXT_DROP;
 494       goto out;
 495     }
 496   if (snat_static_mapping_match
 497       (sm, addr, port, rx_fib_index0, *proto, mapping_addr, mapping_port,
 498        mapping_fib_index, 1, &is_addr_only, 0, 0, 0, 0))
 499     {
 500       /* Don't NAT packet aimed at the intfc address */
 501       if (is_interface_addr (sm, node, sw_if_index0, ip0->dst_address.as_u32))
 502         {
 503           *dont_translate = 1;
 504           goto out;
 505         }
 506       b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 507       next0 = SNAT_OUT2IN_NEXT_DROP;
 508       goto out;
 509     }
 510
 511   if (PREDICT_FALSE
 512       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_reply
 513        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 514            ICMP4_echo_request || !is_addr_only)
 515        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 516                                        reass.icmp_type_or_tcp_flags)))
 517     {
 518       b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
 519       next0 = SNAT_OUT2IN_NEXT_DROP;
 520       goto out;
 521     }
 522
 523 out:
 524   return next0;
 525 }
 526 #endif
 527
 528 #ifndef CLIB_MARCH_VARIANT
 529 u32
 530 icmp_out2in (snat_main_t * sm,
 531              vlib_buffer_t * b0,
 532              ip4_header_t * ip0,
 533              icmp46_header_t * icmp0,
 534              u32 sw_if_index0,
 535              u32 rx_fib_index0,
 536              vlib_node_runtime_t * node,
 537              u32 next0, u32 thread_index, void *d, void *e)
 538 {
 539   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 540   ip4_header_t *inner_ip0 = 0;
 541   void *l4_header = 0;
 542   icmp46_header_t *inner_icmp0;
 543   u8 dont_translate;
 544   u32 new_addr0, old_addr0;
 545   u16 old_id0, new_id0;
 546   ip_csum_t sum0;
 547   u16 checksum0;
 548   u32 next0_tmp;
 549   vlib_main_t *vm = vlib_get_main ();
 550   ip4_address_t addr;
 551   u16 port;
 552   u32 fib_index;
 553   nat_protocol_t proto;
 554
 555   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 556
 557   next0_tmp = sm->icmp_match_out2in_cb (sm, node, thread_index, b0, ip0,
 558                                         &addr, &port, &fib_index, &proto,
 559                                         d, e, &dont_translate);
 560   if (next0_tmp != ~0)
 561     next0 = next0_tmp;
 562   if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
 563     goto out;
 564
 565   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 566     {
 567       sum0 =
 568         ip_incremental_checksum_buffer (vm, b0,
 569                                         (u8 *) icmp0 -
 570                                         (u8 *) vlib_buffer_get_current (b0),
 571                                         ntohs (ip0->length) -
 572                                         ip4_header_bytes (ip0), 0);
 573       checksum0 = ~ip_csum_fold (sum0);
 574       if (checksum0 != 0 && checksum0 != 0xffff)
 575         {
 576           next0 = SNAT_OUT2IN_NEXT_DROP;
 577           goto out;
 578         }
 579     }
 580
 581   old_addr0 = ip0->dst_address.as_u32;
 582   new_addr0 = ip0->dst_address.as_u32 = addr.as_u32;
 583   vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
 584
 585   sum0 = ip0->checksum;
 586   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 587                          dst_address /* changed member */ );
 588   ip0->checksum = ip_csum_fold (sum0);
 589
 590
 591   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 592     {
 593       if (icmp0->checksum == 0)
 594         icmp0->checksum = 0xffff;
 595
 596       if (!icmp_type_is_error_message (icmp0->type))
 597         {
 598           new_id0 = port;
 599           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 600             {
 601               old_id0 = echo0->identifier;
 602               new_id0 = port;
 603               echo0->identifier = new_id0;
 604
 605               sum0 = icmp0->checksum;
 606               sum0 =
 607                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 608                                 identifier /* changed member */ );
 609               icmp0->checksum = ip_csum_fold (sum0);
 610             }
 611         }
 612       else
 613         {
 614           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 615           l4_header = ip4_next_header (inner_ip0);
 616
 617           if (!ip4_header_checksum_is_valid (inner_ip0))
 618             {
 619               next0 = SNAT_OUT2IN_NEXT_DROP;
 620               goto out;
 621             }
 622
 623           old_addr0 = inner_ip0->src_address.as_u32;
 624           inner_ip0->src_address = addr;
 625           new_addr0 = inner_ip0->src_address.as_u32;
 626
 627           sum0 = icmp0->checksum;
 628           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 629                                  src_address /* changed member */ );
 630           icmp0->checksum = ip_csum_fold (sum0);
 631
 632           switch (proto)
 633             {
 634             case NAT_PROTOCOL_ICMP:
 635               inner_icmp0 = (icmp46_header_t *) l4_header;
 636               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 637
 638               old_id0 = inner_echo0->identifier;
 639               new_id0 = port;
 640               inner_echo0->identifier = new_id0;
 641
 642               sum0 = icmp0->checksum;
 643               sum0 =
 644                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 645                                 identifier);
 646               icmp0->checksum = ip_csum_fold (sum0);
 647               break;
 648             case NAT_PROTOCOL_UDP:
 649             case NAT_PROTOCOL_TCP:
 650               old_id0 = ((tcp_udp_header_t *) l4_header)->src_port;
 651               new_id0 = port;
 652               ((tcp_udp_header_t *) l4_header)->src_port = new_id0;
 653
 654               sum0 = icmp0->checksum;
 655               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 656                                      src_port);
 657               icmp0->checksum = ip_csum_fold (sum0);
 658               break;
 659             default:
 660               ASSERT (0);
 661             }
 662         }
 663     }
 664
 665 out:
 666   return next0;
 667 }
 668 #endif
 669
 670 static inline u32
 671 icmp_out2in_slow_path (snat_main_t * sm,
 672                        vlib_buffer_t * b0,
 673                        ip4_header_t * ip0,
 674                        icmp46_header_t * icmp0,
 675                        u32 sw_if_index0,
 676                        u32 rx_fib_index0,
 677                        vlib_node_runtime_t * node,
 678                        u32 next0, f64 now,
 679                        u32 thread_index, snat_session_t ** p_s0)
 680 {
 681   vlib_main_t *vm = vlib_get_main ();
 682
 683   next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 684                        next0, thread_index, p_s0, 0);
 685   snat_session_t *s0 = *p_s0;
 686   if (PREDICT_TRUE (next0 != SNAT_OUT2IN_NEXT_DROP && s0))
 687     {
 688       /* Accounting */
 689       nat44_session_update_counters (s0, now,
 690                                      vlib_buffer_length_in_chain
 691                                      (vm, b0), thread_index);
 692       /* Per-user LRU list maintenance */
 693       nat44_session_update_lru (sm, s0, thread_index);
 694     }
 695   return next0;
 696 }
 697
 698 static int
 699 nat_out2in_sm_unknown_proto (snat_main_t * sm,
 700                              vlib_buffer_t * b,
 701                              ip4_header_t * ip, u32 rx_fib_index)
 702 {
 703   clib_bihash_kv_8_8_t kv, value;
 704   snat_static_mapping_t *m;
 705   u32 old_addr, new_addr;
 706   ip_csum_t sum;
 707
 708   init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 709   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 710     return 1;
 711
 712   m = pool_elt_at_index (sm->static_mappings, value.value);
 713
 714   old_addr = ip->dst_address.as_u32;
 715   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 716   sum = ip->checksum;
 717   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 718   ip->checksum = ip_csum_fold (sum);
 719
 720   vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 721   return 0;
 722 }
 723
 724 VLIB_NODE_FN (snat_out2in_node) (vlib_main_t * vm,
 725                                  vlib_node_runtime_t * node,
 726                                  vlib_frame_t * frame)
 727 {
 728   u32 n_left_from, *from, *to_next;
 729   snat_out2in_next_t next_index;
 730   u32 pkts_processed = 0;
 731   snat_main_t *sm = &snat_main;
 732   f64 now = vlib_time_now (vm);
 733   u32 thread_index = vm->thread_index;
 734   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 735   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 736     0, fragments = 0;
 737
 738   from = vlib_frame_vector_args (frame);
 739   n_left_from = frame->n_vectors;
 740   next_index = node->cached_next_index;
 741
 742   while (n_left_from > 0)
 743     {
 744       u32 n_left_to_next;
 745
 746       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 747
 748       while (n_left_from >= 4 && n_left_to_next >= 2)
 749         {
 750           u32 bi0, bi1;
 751           vlib_buffer_t *b0, *b1;
 752           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
 753           u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
 754           u32 sw_if_index0, sw_if_index1;
 755           ip4_header_t *ip0, *ip1;
 756           ip_csum_t sum0, sum1;
 757           u32 new_addr0, old_addr0;
 758           u16 new_port0, old_port0;
 759           u32 new_addr1, old_addr1;
 760           u16 new_port1, old_port1;
 761           udp_header_t *udp0, *udp1;
 762           tcp_header_t *tcp0, *tcp1;
 763           icmp46_header_t *icmp0, *icmp1;
 764           u32 rx_fib_index0, rx_fib_index1;
 765           u32 proto0, proto1;
 766           snat_session_t *s0 = 0, *s1 = 0;
 767           clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
 768           u8 identity_nat0, identity_nat1;
 769           ip4_address_t sm_addr0, sm_addr1;
 770           u16 sm_port0, sm_port1;
 771           u32 sm_fib_index0, sm_fib_index1;
 772
 773           /* Prefetch next iteration. */
 774           {
 775             vlib_buffer_t *p2, *p3;
 776
 777             p2 = vlib_get_buffer (vm, from[2]);
 778             p3 = vlib_get_buffer (vm, from[3]);
 779
 780             vlib_prefetch_buffer_header (p2, LOAD);
 781             vlib_prefetch_buffer_header (p3, LOAD);
 782
 783             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 784             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
 785           }
 786
 787           /* speculatively enqueue b0 and b1 to the current next frame */
 788           to_next[0] = bi0 = from[0];
 789           to_next[1] = bi1 = from[1];
 790           from += 2;
 791           to_next += 2;
 792           n_left_from -= 2;
 793           n_left_to_next -= 2;
 794
 795           b0 = vlib_get_buffer (vm, bi0);
 796           b1 = vlib_get_buffer (vm, bi1);
 797
 798           vnet_buffer (b0)->snat.flags = 0;
 799           vnet_buffer (b1)->snat.flags = 0;
 800
 801           ip0 = vlib_buffer_get_current (b0);
 802           udp0 = ip4_next_header (ip0);
 803           tcp0 = (tcp_header_t *) udp0;
 804           icmp0 = (icmp46_header_t *) udp0;
 805
 806           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 807           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 808                                    sw_if_index0);
 809
 810           if (PREDICT_FALSE (ip0->ttl == 1))
 811             {
 812               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 813               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 814                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 815                                            0);
 816               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 817               goto trace0;
 818             }
 819
 820           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 821
 822           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 823             {
 824               if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
 825                 {
 826                   if (!sm->forwarding_enabled)
 827                     {
 828                       b0->error =
 829                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
 830                       next0 = SNAT_OUT2IN_NEXT_DROP;
 831                     }
 832                 }
 833               other_packets++;
 834               goto trace0;
 835             }
 836
 837           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
 838             {
 839               next0 = icmp_out2in_slow_path
 840                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 841                  next0, now, thread_index, &s0);
 842               icmp_packets++;
 843               goto trace0;
 844             }
 845
 846           init_nat_k (&kv0, ip0->dst_address,
 847                       vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
 848                       proto0);
 849           if (clib_bihash_search_8_8
 850               (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
 851             {
 852               /* Try to match static mapping by external address and port,
 853                  destination address and port in packet */
 854               if (snat_static_mapping_match
 855                   (sm, ip0->dst_address,
 856                    vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
 857                    proto0, &sm_addr0, &sm_port0, &sm_fib_index0, 1, 0, 0, 0,
 858                    0, &identity_nat0))
 859                 {
 860                   /*
 861                    * Send DHCP packets to the ipv4 stack, or we won't
 862                    * be able to use dhcp client on the outside interface
 863                    */
 864                   if (PREDICT_FALSE
 865                       (proto0 == NAT_PROTOCOL_UDP
 866                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
 867                            clib_host_to_net_u16
 868                            (UDP_DST_PORT_dhcp_to_client))))
 869                     {
 870                       vnet_feature_next (&next0, b0);
 871                       goto trace0;
 872                     }
 873
 874                   if (!sm->forwarding_enabled)
 875                     {
 876                       b0->error =
 877                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
 878                       next0 = SNAT_OUT2IN_NEXT_DROP;
 879                     }
 880                   goto trace0;
 881                 }
 882
 883               if (PREDICT_FALSE (identity_nat0))
 884                 goto trace0;
 885
 886               /* Create session initiated by host from external network */
 887               s0 = create_session_for_static_mapping (sm, b0,
 888                                                       sm_addr0, sm_port0,
 889                                                       sm_fib_index0,
 890                                                       ip0->dst_address,
 891                                                       vnet_buffer (b0)->
 892                                                       ip.reass.l4_dst_port,
 893                                                       rx_fib_index0, proto0,
 894                                                       node, thread_index,
 895                                                       now);
 896               if (!s0)
 897                 {
 898                   next0 = SNAT_OUT2IN_NEXT_DROP;
 899                   goto trace0;
 900                 }
 901             }
 902           else
 903             s0 = pool_elt_at_index (tsm->sessions, value0.value);
 904
 905           old_addr0 = ip0->dst_address.as_u32;
 906           ip0->dst_address = s0->in2out.addr;
 907           new_addr0 = ip0->dst_address.as_u32;
 908           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 909
 910           sum0 = ip0->checksum;
 911           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 912                                  ip4_header_t,
 913                                  dst_address /* changed member */ );
 914           ip0->checksum = ip_csum_fold (sum0);
 915
 916           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 917             {
 918               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 919                 {
 920                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 921                   new_port0 = udp0->dst_port = s0->in2out.port;
 922                   sum0 = tcp0->checksum;
 923                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
 924                                          ip4_header_t,
 925                                          dst_address /* changed member */ );
 926
 927                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
 928                                          ip4_header_t /* cheat */ ,
 929                                          length /* changed member */ );
 930                   tcp0->checksum = ip_csum_fold (sum0);
 931                 }
 932               tcp_packets++;
 933             }
 934           else
 935             {
 936               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 937                 {
 938                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
 939                   new_port0 = udp0->dst_port = s0->in2out.port;
 940                   if (PREDICT_FALSE (udp0->checksum))
 941                     {
 942                       sum0 = udp0->checksum;
 943                       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address      /* changed member */
 944                         );
 945                       sum0 =
 946                         ip_csum_update (sum0, old_port0, new_port0,
 947                                         ip4_header_t /* cheat */ ,
 948                                         length /* changed member */ );
 949                       udp0->checksum = ip_csum_fold (sum0);
 950                     }
 951                 }
 952               udp_packets++;
 953             }
 954
 955           /* Accounting */
 956           nat44_session_update_counters (s0, now,
 957                                          vlib_buffer_length_in_chain (vm, b0),
 958                                          thread_index);
 959           /* Per-user LRU list maintenance */
 960           nat44_session_update_lru (sm, s0, thread_index);
 961         trace0:
 962
 963           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 964                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 965             {
 966               snat_out2in_trace_t *t =
 967                 vlib_add_trace (vm, node, b0, sizeof (*t));
 968               t->sw_if_index = sw_if_index0;
 969               t->next_index = next0;
 970               t->session_index = ~0;
 971               if (s0)
 972                 t->session_index =
 973                   s0 - sm->per_thread_data[thread_index].sessions;
 974             }
 975
 976           pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
 977
 978
 979           ip1 = vlib_buffer_get_current (b1);
 980           udp1 = ip4_next_header (ip1);
 981           tcp1 = (tcp_header_t *) udp1;
 982           icmp1 = (icmp46_header_t *) udp1;
 983
 984           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 985           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
 986                                    sw_if_index1);
 987
 988           if (PREDICT_FALSE (ip1->ttl == 1))
 989             {
 990               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 991               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
 992                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 993                                            0);
 994               next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
 995               goto trace1;
 996             }
 997
 998           proto1 = ip_proto_to_nat_proto (ip1->protocol);
 999
1000           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1001             {
1002               if (nat_out2in_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1003                 {
1004                   if (!sm->forwarding_enabled)
1005                     {
1006                       b1->error =
1007                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1008                       next1 = SNAT_OUT2IN_NEXT_DROP;
1009                     }
1010                 }
1011               other_packets++;
1012               goto trace1;
1013             }
1014
1015           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1016             {
1017               next1 = icmp_out2in_slow_path
1018                 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1019                  next1, now, thread_index, &s1);
1020               icmp_packets++;
1021               goto trace1;
1022             }
1023
1024           init_nat_k (&kv1, ip1->dst_address,
1025                       vnet_buffer (b1)->ip.reass.l4_dst_port, rx_fib_index1,
1026                       proto1);
1027
1028           if (clib_bihash_search_8_8
1029               (&sm->per_thread_data[thread_index].out2in, &kv1, &value1))
1030             {
1031               /* Try to match static mapping by external address and port,
1032                  destination address and port in packet */
1033               if (snat_static_mapping_match
1034                   (sm, ip1->dst_address,
1035                    vnet_buffer (b1)->ip.reass.l4_dst_port, proto1,
1036                    rx_fib_index1, &sm_addr1, &sm_port1, &sm_fib_index1, 1, 0,
1037                    0, 0, 0, &identity_nat1))
1038                 {
1039                   /*
1040                    * Send DHCP packets to the ipv4 stack, or we won't
1041                    * be able to use dhcp client on the outside interface
1042                    */
1043                   if (PREDICT_FALSE
1044                       (proto1 == NAT_PROTOCOL_UDP
1045                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1046                            clib_host_to_net_u16
1047                            (UDP_DST_PORT_dhcp_to_client))))
1048                     {
1049                       vnet_feature_next (&next1, b1);
1050                       goto trace1;
1051                     }
1052
1053                   if (!sm->forwarding_enabled)
1054                     {
1055                       b1->error =
1056                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1057                       next1 = SNAT_OUT2IN_NEXT_DROP;
1058                     }
1059                   goto trace1;
1060                 }
1061
1062               if (PREDICT_FALSE (identity_nat1))
1063                 goto trace1;
1064
1065               /* Create session initiated by host from external network */
1066               s1 =
1067                 create_session_for_static_mapping (sm, b1, sm_addr1, sm_port1,
1068                                                    sm_fib_index1,
1069                                                    ip1->dst_address,
1070                                                    vnet_buffer (b1)->ip.
1071                                                    reass.l4_dst_port,
1072                                                    rx_fib_index1, proto1,
1073                                                    node, thread_index, now);
1074               if (!s1)
1075                 {
1076                   next1 = SNAT_OUT2IN_NEXT_DROP;
1077                   goto trace1;
1078                 }
1079             }
1080           else
1081             s1 =
1082               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1083                                  value1.value);
1084
1085           old_addr1 = ip1->dst_address.as_u32;
1086           ip1->dst_address = s1->in2out.addr;
1087           new_addr1 = ip1->dst_address.as_u32;
1088           vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1089
1090           sum1 = ip1->checksum;
1091           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1092                                  ip4_header_t,
1093                                  dst_address /* changed member */ );
1094           ip1->checksum = ip_csum_fold (sum1);
1095
1096           if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1097             {
1098               if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1099                 {
1100                   old_port1 = vnet_buffer (b1)->ip.reass.l4_dst_port;
1101                   new_port1 = udp1->dst_port = s1->in2out.port;
1102
1103                   sum1 = tcp1->checksum;
1104                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1105                                          ip4_header_t,
1106                                          dst_address /* changed member */ );
1107
1108                   sum1 = ip_csum_update (sum1, old_port1, new_port1,
1109                                          ip4_header_t /* cheat */ ,
1110                                          length /* changed member */ );
1111                   tcp1->checksum = ip_csum_fold (sum1);
1112                 }
1113               tcp_packets++;
1114             }
1115           else
1116             {
1117               if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1118                 {
1119                   old_port1 = vnet_buffer (b1)->ip.reass.l4_dst_port;
1120                   new_port1 = udp1->dst_port = s1->in2out.port;
1121                   if (PREDICT_FALSE (udp1->checksum))
1122                     {
1123
1124                       sum1 = udp1->checksum;
1125                       sum1 =
1126                         ip_csum_update (sum1, old_addr1, new_addr1,
1127                                         ip4_header_t,
1128                                         dst_address /* changed member */ );
1129                       sum1 =
1130                         ip_csum_update (sum1, old_port1, new_port1,
1131                                         ip4_header_t /* cheat */ ,
1132                                         length /* changed member */ );
1133                       udp1->checksum = ip_csum_fold (sum1);
1134                     }
1135                 }
1136               udp_packets++;
1137             }
1138
1139           /* Accounting */
1140           nat44_session_update_counters (s1, now,
1141                                          vlib_buffer_length_in_chain (vm, b1),
1142                                          thread_index);
1143           /* Per-user LRU list maintenance */
1144           nat44_session_update_lru (sm, s1, thread_index);
1145         trace1:
1146
1147           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1148                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1149             {
1150               snat_out2in_trace_t *t =
1151                 vlib_add_trace (vm, node, b1, sizeof (*t));
1152               t->sw_if_index = sw_if_index1;
1153               t->next_index = next1;
1154               t->session_index = ~0;
1155               if (s1)
1156                 t->session_index =
1157                   s1 - sm->per_thread_data[thread_index].sessions;
1158             }
1159
1160           pkts_processed += next1 == SNAT_OUT2IN_NEXT_LOOKUP;
1161
1162           /* verify speculative enqueues, maybe switch current next frame */
1163           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1164                                            to_next, n_left_to_next,
1165                                            bi0, bi1, next0, next1);
1166         }
1167
1168       while (n_left_from > 0 && n_left_to_next > 0)
1169         {
1170           u32 bi0;
1171           vlib_buffer_t *b0;
1172           u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1173           u32 sw_if_index0;
1174           ip4_header_t *ip0;
1175           ip_csum_t sum0;
1176           u32 new_addr0, old_addr0;
1177           u16 new_port0, old_port0;
1178           udp_header_t *udp0;
1179           tcp_header_t *tcp0;
1180           icmp46_header_t *icmp0;
1181           u32 rx_fib_index0;
1182           u32 proto0;
1183           snat_session_t *s0 = 0;
1184           clib_bihash_kv_8_8_t kv0, value0;
1185           u8 identity_nat0;
1186           ip4_address_t sm_addr0;
1187           u16 sm_port0;
1188           u32 sm_fib_index0;
1189
1190           /* speculatively enqueue b0 to the current next frame */
1191           bi0 = from[0];
1192           to_next[0] = bi0;
1193           from += 1;
1194           to_next += 1;
1195           n_left_from -= 1;
1196           n_left_to_next -= 1;
1197
1198           b0 = vlib_get_buffer (vm, bi0);
1199
1200           vnet_buffer (b0)->snat.flags = 0;
1201
1202           ip0 = vlib_buffer_get_current (b0);
1203           udp0 = ip4_next_header (ip0);
1204           tcp0 = (tcp_header_t *) udp0;
1205           icmp0 = (icmp46_header_t *) udp0;
1206
1207           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1208           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1209                                    sw_if_index0);
1210
1211           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1212
1213           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1214             {
1215               if (nat_out2in_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1216                 {
1217                   if (!sm->forwarding_enabled)
1218                     {
1219                       b0->error =
1220                         node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1221                       next0 = SNAT_OUT2IN_NEXT_DROP;
1222                     }
1223                 }
1224               other_packets++;
1225               goto trace00;
1226             }
1227
1228           if (PREDICT_FALSE (ip0->ttl == 1))
1229             {
1230               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1231               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1232                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1233                                            0);
1234               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1235               goto trace00;
1236             }
1237
1238           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1239             {
1240               next0 = icmp_out2in_slow_path
1241                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1242                  next0, now, thread_index, &s0);
1243               icmp_packets++;
1244               goto trace00;
1245             }
1246
1247           init_nat_k (&kv0, ip0->dst_address,
1248                       vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1249                       proto0);
1250
1251           if (clib_bihash_search_8_8
1252               (&sm->per_thread_data[thread_index].out2in, &kv0, &value0))
1253             {
1254               /* Try to match static mapping by external address and port,
1255                  destination address and port in packet */
1256               if (snat_static_mapping_match
1257                   (sm, ip0->dst_address,
1258                    vnet_buffer (b0)->ip.reass.l4_dst_port, rx_fib_index0,
1259                    proto0, &sm_addr0, &sm_port0, &sm_fib_index0, 1, 0, 0, 0,
1260                    0, &identity_nat0))
1261                 {
1262                   /*
1263                    * Send DHCP packets to the ipv4 stack, or we won't
1264                    * be able to use dhcp client on the outside interface
1265                    */
1266                   if (PREDICT_FALSE
1267                       (proto0 == NAT_PROTOCOL_UDP
1268                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1269                            clib_host_to_net_u16
1270                            (UDP_DST_PORT_dhcp_to_client))))
1271                     {
1272                       vnet_feature_next (&next0, b0);
1273                       goto trace00;
1274                     }
1275
1276                   if (!sm->forwarding_enabled)
1277                     {
1278                       b0->error =
1279                         node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1280                       next0 = SNAT_OUT2IN_NEXT_DROP;
1281                     }
1282                   goto trace00;
1283                 }
1284
1285               if (PREDICT_FALSE (identity_nat0))
1286                 goto trace00;
1287
1288               /* Create session initiated by host from external network */
1289               s0 = create_session_for_static_mapping (sm, b0,
1290                                                       sm_addr0, sm_port0,
1291                                                       sm_fib_index0,
1292                                                       ip0->dst_address,
1293                                                       vnet_buffer (b0)->
1294                                                       ip.reass.l4_dst_port,
1295                                                       rx_fib_index0, proto0,
1296                                                       node, thread_index,
1297                                                       now);
1298               if (!s0)
1299                 {
1300                   next0 = SNAT_OUT2IN_NEXT_DROP;
1301                   goto trace00;
1302                 }
1303             }
1304           else
1305             s0 =
1306               pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1307                                  value0.value);
1308
1309           old_addr0 = ip0->dst_address.as_u32;
1310           ip0->dst_address = s0->in2out.addr;
1311           new_addr0 = ip0->dst_address.as_u32;
1312           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1313
1314           sum0 = ip0->checksum;
1315           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1316                                  ip4_header_t,
1317                                  dst_address /* changed member */ );
1318           ip0->checksum = ip_csum_fold (sum0);
1319
1320           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1321             {
1322               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1323                 {
1324                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1325                   new_port0 = udp0->dst_port = s0->in2out.port;
1326
1327                   sum0 = tcp0->checksum;
1328                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1329                                          ip4_header_t,
1330                                          dst_address /* changed member */ );
1331
1332                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1333                                          ip4_header_t /* cheat */ ,
1334                                          length /* changed member */ );
1335                   tcp0->checksum = ip_csum_fold (sum0);
1336                 }
1337               tcp_packets++;
1338             }
1339           else
1340             {
1341               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1342                 {
1343                   old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1344                   new_port0 = udp0->dst_port = s0->in2out.port;
1345                   if (PREDICT_FALSE (udp0->checksum))
1346                     {
1347                       sum0 = udp0->checksum;
1348                       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address      /* changed member */
1349                         );
1350                       sum0 =
1351                         ip_csum_update (sum0, old_port0, new_port0,
1352                                         ip4_header_t /* cheat */ ,
1353                                         length /* changed member */ );
1354                       udp0->checksum = ip_csum_fold (sum0);
1355                     }
1356                 }
1357               udp_packets++;
1358             }
1359
1360           /* Accounting */
1361           nat44_session_update_counters (s0, now,
1362                                          vlib_buffer_length_in_chain (vm, b0),
1363                                          thread_index);
1364           /* Per-user LRU list maintenance */
1365           nat44_session_update_lru (sm, s0, thread_index);
1366         trace00:
1367
1368           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1369                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1370             {
1371               snat_out2in_trace_t *t =
1372                 vlib_add_trace (vm, node, b0, sizeof (*t));
1373               t->sw_if_index = sw_if_index0;
1374               t->next_index = next0;
1375               t->session_index = ~0;
1376               if (s0)
1377                 t->session_index =
1378                   s0 - sm->per_thread_data[thread_index].sessions;
1379             }
1380
1381           pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
1382
1383           /* verify speculative enqueue, maybe switch current next frame */
1384           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1385                                            to_next, n_left_to_next,
1386                                            bi0, next0);
1387         }
1388
1389       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1390     }
1391
1392   vlib_node_increment_counter (vm, sm->out2in_node_index,
1393                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1394                                pkts_processed);
1395   vlib_node_increment_counter (vm, sm->out2in_node_index,
1396                                SNAT_OUT2IN_ERROR_TCP_PACKETS, tcp_packets);
1397   vlib_node_increment_counter (vm, sm->out2in_node_index,
1398                                SNAT_OUT2IN_ERROR_UDP_PACKETS, udp_packets);
1399   vlib_node_increment_counter (vm, sm->out2in_node_index,
1400                                SNAT_OUT2IN_ERROR_ICMP_PACKETS, icmp_packets);
1401   vlib_node_increment_counter (vm, sm->out2in_node_index,
1402                                SNAT_OUT2IN_ERROR_OTHER_PACKETS,
1403                                other_packets);
1404   vlib_node_increment_counter (vm, sm->out2in_node_index,
1405                                SNAT_OUT2IN_ERROR_FRAGMENTS, fragments);
1406
1407   return frame->n_vectors;
1408 }
1409
1410 /* *INDENT-OFF* */
1411 VLIB_REGISTER_NODE (snat_out2in_node) = {
1412   .name = "nat44-out2in",
1413   .vector_size = sizeof (u32),
1414   .format_trace = format_snat_out2in_trace,
1415   .type = VLIB_NODE_TYPE_INTERNAL,
1416
1417   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1418   .error_strings = snat_out2in_error_strings,
1419
1420   .runtime_data_bytes = sizeof (snat_runtime_t),
1421
1422   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1423
1424   /* edit / add dispositions here */
1425   .next_nodes = {
1426     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1427     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1428     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1429   },
1430 };
1431 /* *INDENT-ON* */
1432
1433 VLIB_NODE_FN (snat_out2in_fast_node) (vlib_main_t * vm,
1434                                       vlib_node_runtime_t * node,
1435                                       vlib_frame_t * frame)
1436 {
1437   u32 n_left_from, *from, *to_next;
1438   snat_out2in_next_t next_index;
1439   u32 pkts_processed = 0;
1440   snat_main_t *sm = &snat_main;
1441
1442   from = vlib_frame_vector_args (frame);
1443   n_left_from = frame->n_vectors;
1444   next_index = node->cached_next_index;
1445
1446   while (n_left_from > 0)
1447     {
1448       u32 n_left_to_next;
1449
1450       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1451
1452       while (n_left_from > 0 && n_left_to_next > 0)
1453         {
1454           u32 bi0;
1455           vlib_buffer_t *b0;
1456           u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1457           u32 sw_if_index0;
1458           ip4_header_t *ip0;
1459           ip_csum_t sum0;
1460           u32 new_addr0, old_addr0;
1461           u16 new_port0, old_port0;
1462           udp_header_t *udp0;
1463           tcp_header_t *tcp0;
1464           icmp46_header_t *icmp0;
1465           u32 proto0;
1466           u32 rx_fib_index0;
1467           ip4_address_t sm_addr0;
1468           u16 sm_port0;
1469           u32 sm_fib_index0;
1470
1471           /* speculatively enqueue b0 to the current next frame */
1472           bi0 = from[0];
1473           to_next[0] = bi0;
1474           from += 1;
1475           to_next += 1;
1476           n_left_from -= 1;
1477           n_left_to_next -= 1;
1478
1479           b0 = vlib_get_buffer (vm, bi0);
1480
1481           ip0 = vlib_buffer_get_current (b0);
1482           udp0 = ip4_next_header (ip0);
1483           tcp0 = (tcp_header_t *) udp0;
1484           icmp0 = (icmp46_header_t *) udp0;
1485
1486           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1487           rx_fib_index0 =
1488             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1489
1490           vnet_feature_next (&next0, b0);
1491
1492           if (PREDICT_FALSE (ip0->ttl == 1))
1493             {
1494               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1495               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1496                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1497                                            0);
1498               next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1499               goto trace00;
1500             }
1501
1502           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1503
1504           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1505             goto trace00;
1506
1507           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1508             {
1509               next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0,
1510                                    rx_fib_index0, node, next0, ~0, 0, 0);
1511               goto trace00;
1512             }
1513
1514           if (snat_static_mapping_match
1515               (sm, ip0->dst_address, udp0->dst_port, rx_fib_index0, proto0,
1516                &sm_addr0, &sm_port0, &sm_fib_index0, 1, 0, 0, 0, 0, 0))
1517             {
1518               b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1519               goto trace00;
1520             }
1521
1522           new_addr0 = sm_addr0.as_u32;
1523           new_port0 = sm_port0;
1524           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm_fib_index0;
1525           old_addr0 = ip0->dst_address.as_u32;
1526           ip0->dst_address.as_u32 = new_addr0;
1527
1528           sum0 = ip0->checksum;
1529           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1530                                  ip4_header_t,
1531                                  dst_address /* changed member */ );
1532           ip0->checksum = ip_csum_fold (sum0);
1533
1534           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1535             {
1536               old_port0 = udp0->dst_port;
1537               udp0->dst_port = new_port0;
1538
1539               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1540                 {
1541                   sum0 = tcp0->checksum;
1542                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1543                                          ip4_header_t,
1544                                          dst_address /* changed member */ );
1545                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1546                                          ip4_header_t /* cheat */ ,
1547                                          length /* changed member */ );
1548                   tcp0->checksum = ip_csum_fold (sum0);
1549                 }
1550               else if (udp0->checksum)
1551                 {
1552                   sum0 = udp0->checksum;
1553                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1554                                          ip4_header_t,
1555                                          dst_address /* changed member */ );
1556                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1557                                          ip4_header_t /* cheat */ ,
1558                                          length /* changed member */ );
1559                   udp0->checksum = ip_csum_fold (sum0);
1560                 }
1561             }
1562           else
1563             {
1564               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1565                 {
1566                   sum0 = tcp0->checksum;
1567                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1568                                          ip4_header_t,
1569                                          dst_address /* changed member */ );
1570                   tcp0->checksum = ip_csum_fold (sum0);
1571                 }
1572               else if (udp0->checksum)
1573                 {
1574                   sum0 = udp0->checksum;
1575                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1576                                          ip4_header_t,
1577                                          dst_address /* changed member */ );
1578                   udp0->checksum = ip_csum_fold (sum0);
1579                 }
1580             }
1581
1582         trace00:
1583
1584           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1585                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1586             {
1587               snat_out2in_trace_t *t =
1588                 vlib_add_trace (vm, node, b0, sizeof (*t));
1589               t->sw_if_index = sw_if_index0;
1590               t->next_index = next0;
1591             }
1592
1593           pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1594
1595           /* verify speculative enqueue, maybe switch current next frame */
1596           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1597                                            to_next, n_left_to_next,
1598                                            bi0, next0);
1599         }
1600
1601       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1602     }
1603
1604   vlib_node_increment_counter (vm, sm->out2in_fast_node_index,
1605                                SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1606                                pkts_processed);
1607   return frame->n_vectors;
1608 }
1609
1610 /* *INDENT-OFF* */
1611 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
1612   .name = "nat44-out2in-fast",
1613   .vector_size = sizeof (u32),
1614   .format_trace = format_snat_out2in_fast_trace,
1615   .type = VLIB_NODE_TYPE_INTERNAL,
1616
1617   .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1618   .error_strings = snat_out2in_error_strings,
1619
1620   .runtime_data_bytes = sizeof (snat_runtime_t),
1621
1622   .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1623
1624   /* edit / add dispositions here */
1625   .next_nodes = {
1626     [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1627     [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1628     [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1629   },
1630 };
1631 /* *INDENT-ON* */
1632
1633 /*
1634  * fd.io coding-style-patch-verification: ON
1635  *
1636  * Local Variables:
1637  * eval: (c-set-style "gnu")
1638  * End:
1639  */