nat: fix dual-loop tcp checksum botch
[vpp.git] / src / plugins / nat / out2in.c
index 985333f..9549a45 100755 (executable)
@@ -29,6 +29,8 @@
 #include <nat/nat_ipfix_logging.h>
 #include <nat/nat_reass.h>
 #include <nat/nat_inlines.h>
+#include <nat/nat_syslog.h>
+#include <nat/nat_ha.h>
 
 #include <vppinfra/hash.h>
 #include <vppinfra/error.h>
@@ -68,20 +70,23 @@ format_snat_out2in_fast_trace (u8 * s, va_list * args)
   return s;
 }
 
-vlib_node_registration_t snat_out2in_node;
-vlib_node_registration_t snat_out2in_fast_node;
-vlib_node_registration_t nat44_out2in_reass_node;
-
 #define foreach_snat_out2in_error                       \
-_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
-_(OUT2IN_PACKETS, "Good out2in packets processed")      \
-_(OUT_OF_PORTS, "Out of ports")                         \
+_(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
+_(OUT2IN_PACKETS, "good out2in packets processed")      \
+_(OUT_OF_PORTS, "out of ports")                         \
 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
-_(NO_TRANSLATION, "No translation")                     \
-_(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
-_(DROP_FRAGMENT, "Drop fragment")                       \
-_(MAX_REASS, "Maximum reassemblies exceeded")           \
-_(MAX_FRAG, "Maximum fragments per reassembly exceeded")
+_(NO_TRANSLATION, "no translation")                     \
+_(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
+_(DROP_FRAGMENT, "drop fragment")                       \
+_(MAX_REASS, "maximum reassemblies exceeded")           \
+_(MAX_FRAG, "maximum fragments per reassembly exceeded")\
+_(TCP_PACKETS, "TCP packets")                           \
+_(UDP_PACKETS, "UDP packets")                           \
+_(ICMP_PACKETS, "ICMP packets")                         \
+_(OTHER_PACKETS, "other protocol packets")              \
+_(FRAGMENTS, "fragments")                               \
+_(CACHED_FRAGMENTS, "cached fragments")                 \
+_(PROCESSED_FRAGMENTS, "processed fragments")
 
 typedef enum
 {
@@ -106,6 +111,7 @@ typedef enum
   SNAT_OUT2IN_N_NEXT,
 } snat_out2in_next_t;
 
+#ifndef CLIB_MARCH_VARIANT
 int
 nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 {
@@ -123,15 +129,25 @@ nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
     {
       s_kv.key = s->in2out.as_u64;
       if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
-       nat_log_warn ("out2in key del failed");
+       nat_elog_warn ("out2in key del failed");
 
-      snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
+      snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
+                                          s->in2out.addr.as_u32,
                                           s->out2in.addr.as_u32,
                                           s->in2out.protocol,
                                           s->in2out.port,
                                           s->out2in.port,
                                           s->in2out.fib_index);
 
+      nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
+                              &s->in2out.addr, s->in2out.port,
+                              &s->out2in.addr, s->out2in.port,
+                              s->in2out.protocol);
+
+      nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
+                  s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
+                  ctx->thread_index);
+
       if (!snat_is_session_static (s))
        snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
                                            &s->out2in);
@@ -142,6 +158,7 @@ nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 
   return 0;
 }
+#endif
 
 /**
  * @brief Create session for static mapping.
@@ -175,7 +192,7 @@ create_session_for_static_mapping (snat_main_t * sm,
   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
     {
       b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
-      nat_log_notice ("maximum sessions exceeded");
+      nat_elog_notice ("maximum sessions exceeded");
       return 0;
     }
 
@@ -186,15 +203,15 @@ create_session_for_static_mapping (snat_main_t * sm,
     nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
   if (!u)
     {
-      nat_log_warn ("create NAT user failed");
+      nat_elog_warn ("create NAT user failed");
       return 0;
     }
 
-  s = nat_session_alloc_or_recycle (sm, u, thread_index);
+  s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
   if (!s)
     {
       nat44_delete_user_with_no_session (sm, u, thread_index);
-      nat_log_warn ("create NAT session failed");
+      nat_elog_warn ("create NAT session failed");
       return 0;
     }
 
@@ -214,24 +231,37 @@ create_session_for_static_mapping (snat_main_t * sm,
   if (clib_bihash_add_or_overwrite_stale_8_8
       (&sm->per_thread_data[thread_index].in2out, &kv0,
        nat44_i2o_is_idle_session_cb, &ctx0))
-    nat_log_notice ("in2out key add failed");
+    nat_elog_notice ("in2out key add failed");
 
   kv0.key = s->out2in.as_u64;
 
   if (clib_bihash_add_or_overwrite_stale_8_8
       (&sm->per_thread_data[thread_index].out2in, &kv0,
        nat44_o2i_is_idle_session_cb, &ctx0))
-    nat_log_notice ("out2in key add failed");
+    nat_elog_notice ("out2in key add failed");
 
   /* log NAT event */
-  snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
+  snat_ipfix_logging_nat44_ses_create (thread_index,
+                                      s->in2out.addr.as_u32,
                                       s->out2in.addr.as_u32,
                                       s->in2out.protocol,
                                       s->in2out.port,
                                       s->out2in.port, s->in2out.fib_index);
+
+  nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index,
+                          &s->in2out.addr, s->in2out.port, &s->out2in.addr,
+                          s->out2in.port, s->in2out.protocol);
+
+  nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
+              s->out2in.port, &s->ext_host_addr, s->ext_host_port,
+              &s->ext_host_nat_addr, s->ext_host_nat_port,
+              s->in2out.protocol, s->in2out.fib_index, s->flags,
+              thread_index, 0);
+
   return s;
 }
 
+#ifndef CLIB_MARCH_VARIANT
 static_always_inline
   snat_out2in_error_t icmp_get_key (ip4_header_t * ip0,
                                    snat_session_key_t * p_key0)
@@ -309,6 +339,7 @@ icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
   u8 is_addr_only;
   u32 next0 = ~0;
   int err;
+  u8 identity_nat;
 
   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
@@ -333,7 +364,7 @@ icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
       /* Try to match static mapping by external address and port,
          destination address and port in packet */
       if (snat_static_mapping_match
-         (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0))
+         (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0, &identity_nat))
        {
          if (!sm->forwarding_enabled)
            {
@@ -364,6 +395,11 @@ icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
          goto out;
        }
 
+      if (PREDICT_FALSE (identity_nat))
+       {
+         dont_translate = 1;
+         goto out;
+       }
       /* Create session initiated by host from external network */
       s0 = create_session_for_static_mapping (sm, b0, sm0, key0,
                                              node, thread_index,
@@ -399,7 +435,9 @@ out:
     *(snat_session_t **) d = s0;
   return next0;
 }
+#endif
 
+#ifndef CLIB_MARCH_VARIANT
 /**
  * Get address and port values to be used for ICMP packet translation
  *
@@ -443,7 +481,8 @@ icmp_match_out2in_fast (snat_main_t * sm, vlib_node_runtime_t * node,
     }
   key0.fib_index = rx_fib_index0;
 
-  if (snat_static_mapping_match (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0))
+  if (snat_static_mapping_match
+      (sm, key0, &sm0, 1, &is_addr_only, 0, 0, 0, 0))
     {
       /* Don't NAT packet aimed at the intfc address */
       if (is_interface_addr (sm, node, sw_if_index0, ip0->dst_address.as_u32))
@@ -472,7 +511,9 @@ out2:
   *p_dont_translate = dont_translate;
   return next0;
 }
+#endif
 
+#ifndef CLIB_MARCH_VARIANT
 u32
 icmp_out2in (snat_main_t * sm,
             vlib_buffer_t * b0,
@@ -508,9 +549,11 @@ icmp_out2in (snat_main_t * sm,
 
   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
     {
-      sum0 = ip_incremental_checksum (0, icmp0,
-                                     ntohs (ip0->length) -
-                                     ip4_header_bytes (ip0));
+      sum0 = ip_incremental_checksum_buffer (sm->vlib_main, b0, (u8 *) icmp0 -
+                                            (u8 *)
+                                            vlib_buffer_get_current (b0),
+                                            ntohs (ip0->length) -
+                                            ip4_header_bytes (ip0), 0);
       checksum0 = ~ip_csum_fold (sum0);
       if (checksum0 != 0 && checksum0 != 0xffff)
        {
@@ -600,7 +643,7 @@ icmp_out2in (snat_main_t * sm,
 out:
   return next0;
 }
-
+#endif
 
 static inline u32
 icmp_out2in_slow_path (snat_main_t * sm,
@@ -621,7 +664,7 @@ icmp_out2in_slow_path (snat_main_t * sm,
       /* Accounting */
       nat44_session_update_counters (s0, now,
                                     vlib_buffer_length_in_chain
-                                    (sm->vlib_main, b0));
+                                    (sm->vlib_main, b0), thread_index);
       /* Per-user LRU list maintenance */
       nat44_session_update_lru (sm, s0, thread_index);
     }
@@ -659,9 +702,9 @@ nat_out2in_sm_unknown_proto (snat_main_t * sm,
   return 0;
 }
 
-static uword
-snat_out2in_node_fn (vlib_main_t * vm,
-                    vlib_node_runtime_t * node, vlib_frame_t * frame)
+VLIB_NODE_FN (snat_out2in_node) (vlib_main_t * vm,
+                                vlib_node_runtime_t * node,
+                                vlib_frame_t * frame)
 {
   u32 n_left_from, *from, *to_next;
   snat_out2in_next_t next_index;
@@ -669,6 +712,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
   snat_main_t *sm = &snat_main;
   f64 now = vlib_time_now (vm);
   u32 thread_index = vm->thread_index;
+  u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
+    0, fragments = 0;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
@@ -701,6 +746,7 @@ snat_out2in_node_fn (vlib_main_t * vm,
          u32 proto0, proto1;
          snat_session_t *s0 = 0, *s1 = 0;
          clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
+         u8 identity_nat0, identity_nat1;
 
          /* Prefetch next iteration. */
          {
@@ -762,20 +808,23 @@ snat_out2in_node_fn (vlib_main_t * vm,
                      next0 = SNAT_OUT2IN_NEXT_DROP;
                    }
                }
+             other_packets++;
              goto trace0;
            }
 
-         if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+         if (PREDICT_FALSE (ip4_is_fragment (ip0)))
            {
-             next0 = icmp_out2in_slow_path
-               (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
-                next0, now, thread_index, &s0);
+             next0 = SNAT_OUT2IN_NEXT_REASS;
+             fragments++;
              goto trace0;
            }
 
-         if (PREDICT_FALSE (ip4_is_fragment (ip0)))
+         if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
            {
-             next0 = SNAT_OUT2IN_NEXT_REASS;
+             next0 = icmp_out2in_slow_path
+               (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                next0, now, thread_index, &s0);
+             icmp_packets++;
              goto trace0;
            }
 
@@ -791,7 +840,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
            {
              /* Try to match static mapping by external address and port,
                 destination address and port in packet */
-             if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
+             if (snat_static_mapping_match
+                 (sm, key0, &sm0, 1, 0, 0, 0, 0, &identity_nat0))
                {
                  /*
                   * Send DHCP packets to the ipv4 stack, or we won't
@@ -815,6 +865,9 @@ snat_out2in_node_fn (vlib_main_t * vm,
                  goto trace0;
                }
 
+             if (PREDICT_FALSE (identity_nat0))
+               goto trace0;
+
              /* Create session initiated by host from external network */
              s0 = create_session_for_static_mapping (sm, b0, sm0, key0, node,
                                                      thread_index, now);
@@ -855,18 +908,31 @@ snat_out2in_node_fn (vlib_main_t * vm,
                                     ip4_header_t /* cheat */ ,
                                     length /* changed member */ );
              tcp0->checksum = ip_csum_fold (sum0);
+             tcp_packets++;
            }
          else
            {
-             old_port0 = udp0->dst_port;
-             udp0->dst_port = s0->in2out.port;
-             udp0->checksum = 0;
+             if (PREDICT_FALSE (udp0->checksum))
+               {
+                 old_port0 = udp0->dst_port;
+                 new_port0 = udp0->dst_port = s0->in2out.port;
+
+                 sum0 = udp0->checksum;
+                 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                        ip4_header_t,
+                                        dst_address /* changed member */ );
+                 sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                        ip4_header_t /* cheat */ ,
+                                        length /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
+               }
+             udp_packets++;
            }
 
          /* Accounting */
          nat44_session_update_counters (s0, now,
-                                        vlib_buffer_length_in_chain (vm,
-                                                                     b0));
+                                        vlib_buffer_length_in_chain (vm, b0),
+                                        thread_index);
          /* Per-user LRU list maintenance */
          nat44_session_update_lru (sm, s0, thread_index);
        trace0:
@@ -884,7 +950,7 @@ snat_out2in_node_fn (vlib_main_t * vm,
                  s0 - sm->per_thread_data[thread_index].sessions;
            }
 
-         pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+         pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
 
 
          ip1 = vlib_buffer_get_current (b1);
@@ -919,20 +985,23 @@ snat_out2in_node_fn (vlib_main_t * vm,
                      next1 = SNAT_OUT2IN_NEXT_DROP;
                    }
                }
+             other_packets++;
              goto trace1;
            }
 
-         if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
+         if (PREDICT_FALSE (ip4_is_fragment (ip1)))
            {
-             next1 = icmp_out2in_slow_path
-               (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
-                next1, now, thread_index, &s1);
+             next1 = SNAT_OUT2IN_NEXT_REASS;
+             fragments++;
              goto trace1;
            }
 
-         if (PREDICT_FALSE (ip4_is_fragment (ip1)))
+         if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
            {
-             next1 = SNAT_OUT2IN_NEXT_REASS;
+             next1 = icmp_out2in_slow_path
+               (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
+                next1, now, thread_index, &s1);
+             icmp_packets++;
              goto trace1;
            }
 
@@ -948,7 +1017,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
            {
              /* Try to match static mapping by external address and port,
                 destination address and port in packet */
-             if (snat_static_mapping_match (sm, key1, &sm1, 1, 0, 0, 0, 0))
+             if (snat_static_mapping_match
+                 (sm, key1, &sm1, 1, 0, 0, 0, 0, &identity_nat1))
                {
                  /*
                   * Send DHCP packets to the ipv4 stack, or we won't
@@ -972,6 +1042,9 @@ snat_out2in_node_fn (vlib_main_t * vm,
                  goto trace1;
                }
 
+             if (PREDICT_FALSE (identity_nat1))
+               goto trace1;
+
              /* Create session initiated by host from external network */
              s1 = create_session_for_static_mapping (sm, b1, sm1, key1, node,
                                                      thread_index, now);
@@ -1012,18 +1085,31 @@ snat_out2in_node_fn (vlib_main_t * vm,
                                     ip4_header_t /* cheat */ ,
                                     length /* changed member */ );
              tcp1->checksum = ip_csum_fold (sum1);
+             tcp_packets++;
            }
          else
            {
-             old_port1 = udp1->dst_port;
-             udp1->dst_port = s1->in2out.port;
-             udp1->checksum = 0;
+             if (PREDICT_FALSE (udp1->checksum))
+               {
+                 old_port1 = udp1->dst_port;
+                 new_port1 = udp1->dst_port = s1->in2out.port;
+
+                 sum1 = udp1->checksum;
+                 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
+                                        ip4_header_t,
+                                        dst_address /* changed member */ );
+                 sum1 = ip_csum_update (sum1, old_port1, new_port1,
+                                        ip4_header_t /* cheat */ ,
+                                        length /* changed member */ );
+                 udp1->checksum = ip_csum_fold (sum1);
+               }
+             udp_packets++;
            }
 
          /* Accounting */
          nat44_session_update_counters (s1, now,
-                                        vlib_buffer_length_in_chain (vm,
-                                                                     b1));
+                                        vlib_buffer_length_in_chain (vm, b1),
+                                        thread_index);
          /* Per-user LRU list maintenance */
          nat44_session_update_lru (sm, s1, thread_index);
        trace1:
@@ -1041,7 +1127,7 @@ snat_out2in_node_fn (vlib_main_t * vm,
                  s1 - sm->per_thread_data[thread_index].sessions;
            }
 
-         pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
+         pkts_processed += next1 == SNAT_OUT2IN_NEXT_LOOKUP;
 
          /* verify speculative enqueues, maybe switch current next frame */
          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
@@ -1067,6 +1153,7 @@ snat_out2in_node_fn (vlib_main_t * vm,
          u32 proto0;
          snat_session_t *s0 = 0;
          clib_bihash_kv_8_8_t kv0, value0;
+         u8 identity_nat0;
 
          /* speculatively enqueue b0 to the current next frame */
          bi0 = from[0];
@@ -1102,6 +1189,7 @@ snat_out2in_node_fn (vlib_main_t * vm,
                      next0 = SNAT_OUT2IN_NEXT_DROP;
                    }
                }
+             other_packets++;
              goto trace00;
            }
 
@@ -1115,17 +1203,19 @@ snat_out2in_node_fn (vlib_main_t * vm,
              goto trace00;
            }
 
-         if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+         if (PREDICT_FALSE (ip4_is_fragment (ip0)))
            {
-             next0 = icmp_out2in_slow_path
-               (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
-                next0, now, thread_index, &s0);
+             next0 = SNAT_OUT2IN_NEXT_REASS;
+             fragments++;
              goto trace00;
            }
 
-         if (PREDICT_FALSE (ip4_is_fragment (ip0)))
+         if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
            {
-             next0 = SNAT_OUT2IN_NEXT_REASS;
+             next0 = icmp_out2in_slow_path
+               (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                next0, now, thread_index, &s0);
+             icmp_packets++;
              goto trace00;
            }
 
@@ -1141,7 +1231,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
            {
              /* Try to match static mapping by external address and port,
                 destination address and port in packet */
-             if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
+             if (snat_static_mapping_match
+                 (sm, key0, &sm0, 1, 0, 0, 0, 0, &identity_nat0))
                {
                  /*
                   * Send DHCP packets to the ipv4 stack, or we won't
@@ -1165,6 +1256,9 @@ snat_out2in_node_fn (vlib_main_t * vm,
                  goto trace00;
                }
 
+             if (PREDICT_FALSE (identity_nat0))
+               goto trace00;
+
              /* Create session initiated by host from external network */
              s0 = create_session_for_static_mapping (sm, b0, sm0, key0, node,
                                                      thread_index, now);
@@ -1205,18 +1299,31 @@ snat_out2in_node_fn (vlib_main_t * vm,
                                     ip4_header_t /* cheat */ ,
                                     length /* changed member */ );
              tcp0->checksum = ip_csum_fold (sum0);
+             tcp_packets++;
            }
          else
            {
-             old_port0 = udp0->dst_port;
-             udp0->dst_port = s0->in2out.port;
-             udp0->checksum = 0;
+             if (PREDICT_FALSE (udp0->checksum))
+               {
+                 old_port0 = udp0->dst_port;
+                 new_port0 = udp0->dst_port = s0->in2out.port;
+
+                 sum0 = udp0->checksum;
+                 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                        ip4_header_t,
+                                        dst_address /* changed member */ );
+                 sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                        ip4_header_t /* cheat */ ,
+                                        length /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
+               }
+             udp_packets++;
            }
 
          /* Accounting */
          nat44_session_update_counters (s0, now,
-                                        vlib_buffer_length_in_chain (vm,
-                                                                     b0));
+                                        vlib_buffer_length_in_chain (vm, b0),
+                                        thread_index);
          /* Per-user LRU list maintenance */
          nat44_session_update_lru (sm, s0, thread_index);
        trace00:
@@ -1234,7 +1341,7 @@ snat_out2in_node_fn (vlib_main_t * vm,
                  s0 - sm->per_thread_data[thread_index].sessions;
            }
 
-         pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+         pkts_processed += next0 == SNAT_OUT2IN_NEXT_LOOKUP;
 
          /* verify speculative enqueue, maybe switch current next frame */
          vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
@@ -1245,15 +1352,26 @@ snat_out2in_node_fn (vlib_main_t * vm,
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
-  vlib_node_increment_counter (vm, snat_out2in_node.index,
+  vlib_node_increment_counter (vm, sm->out2in_node_index,
                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
                               pkts_processed);
+  vlib_node_increment_counter (vm, sm->out2in_node_index,
+                              SNAT_OUT2IN_ERROR_TCP_PACKETS, tcp_packets);
+  vlib_node_increment_counter (vm, sm->out2in_node_index,
+                              SNAT_OUT2IN_ERROR_UDP_PACKETS, udp_packets);
+  vlib_node_increment_counter (vm, sm->out2in_node_index,
+                              SNAT_OUT2IN_ERROR_ICMP_PACKETS, icmp_packets);
+  vlib_node_increment_counter (vm, sm->out2in_node_index,
+                              SNAT_OUT2IN_ERROR_OTHER_PACKETS,
+                              other_packets);
+  vlib_node_increment_counter (vm, sm->out2in_node_index,
+                              SNAT_OUT2IN_ERROR_FRAGMENTS, fragments);
+
   return frame->n_vectors;
 }
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (snat_out2in_node) = {
-  .function = snat_out2in_node_fn,
   .name = "nat44-out2in",
   .vector_size = sizeof (u32),
   .format_trace = format_snat_out2in_trace,
@@ -1276,15 +1394,13 @@ VLIB_REGISTER_NODE (snat_out2in_node) = {
 };
 /* *INDENT-ON* */
 
-VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
-
-static uword
-nat44_out2in_reass_node_fn (vlib_main_t * vm,
-                           vlib_node_runtime_t * node, vlib_frame_t * frame)
+VLIB_NODE_FN (nat44_out2in_reass_node) (vlib_main_t * vm,
+                                       vlib_node_runtime_t * node,
+                                       vlib_frame_t * frame)
 {
   u32 n_left_from, *from, *to_next;
   snat_out2in_next_t next_index;
-  u32 pkts_processed = 0;
+  u32 pkts_processed = 0, cached_fragments = 0;
   snat_main_t *sm = &snat_main;
   f64 now = vlib_time_now (vm);
   u32 thread_index = vm->thread_index;
@@ -1313,11 +1429,13 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
          nat_reass_ip4_t *reass0;
          udp_header_t *udp0;
          tcp_header_t *tcp0;
+         icmp46_header_t *icmp0;
          snat_session_key_t key0, sm0;
          clib_bihash_kv_8_8_t kv0, value0;
          snat_session_t *s0 = 0;
          u16 old_port0, new_port0;
          ip_csum_t sum0;
+         u8 identity_nat0;
 
          /* speculatively enqueue b0 to the current next frame */
          bi0 = from[0];
@@ -1345,6 +1463,7 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
          ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
          udp0 = ip4_next_header (ip0);
          tcp0 = (tcp_header_t *) udp0;
+         icmp0 = (icmp46_header_t *) udp0;
          proto0 = ip_proto_to_snat_proto (ip0->protocol);
 
          reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
@@ -1357,12 +1476,32 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
            {
              next0 = SNAT_OUT2IN_NEXT_DROP;
              b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
-             nat_log_notice ("maximum reassemblies exceeded");
+             nat_elog_notice ("maximum reassemblies exceeded");
              goto trace0;
            }
 
          if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
            {
+             if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+               {
+                 next0 = icmp_out2in_slow_path
+                   (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                    next0, now, thread_index, &s0);
+
+                 if (PREDICT_TRUE (next0 != SNAT_OUT2IN_NEXT_DROP))
+                   {
+                     if (s0)
+                       reass0->sess_index = s0 - per_thread_data->sessions;
+                     else
+                       reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
+                     reass0->thread_index = thread_index;
+                     nat_ip4_reass_get_frags (reass0,
+                                              &fragments_to_loopback);
+                   }
+
+                 goto trace0;
+               }
+
              key0.addr = ip0->dst_address;
              key0.port = udp0->dst_port;
              key0.protocol = proto0;
@@ -1375,7 +1514,7 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
                  /* Try to match static mapping by external address and port,
                     destination address and port in packet */
                  if (snat_static_mapping_match
-                     (sm, key0, &sm0, 1, 0, 0, 0, 0))
+                     (sm, key0, &sm0, 1, 0, 0, 0, 0, &identity_nat0))
                    {
                      /*
                       * Send DHCP packets to the ipv4 stack, or we won't
@@ -1397,9 +1536,18 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
                            node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
                          next0 = SNAT_OUT2IN_NEXT_DROP;
                        }
+                     else
+                       {
+                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
+                         nat_ip4_reass_get_frags (reass0,
+                                                  &fragments_to_loopback);
+                       }
                      goto trace0;
                    }
 
+                 if (PREDICT_FALSE (identity_nat0))
+                   goto trace0;
+
                  /* Create session initiated by host from external network */
                  s0 =
                    create_session_for_static_mapping (sm, b0, sm0, key0,
@@ -1425,13 +1573,15 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
            }
          else
            {
+             if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
+               goto trace0;
              if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
                {
                  if (nat_ip4_reass_add_fragment
-                     (reass0, bi0, &fragments_to_drop))
+                     (thread_index, reass0, bi0, &fragments_to_drop))
                    {
                      b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
-                     nat_log_notice
+                     nat_elog_notice
                        ("maximum fragments per reassembly exceeded");
                      next0 = SNAT_OUT2IN_NEXT_DROP;
                      goto trace0;
@@ -1456,12 +1606,11 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
 
          if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
            {
+             old_port0 = udp0->dst_port;
+             new_port0 = udp0->dst_port = s0->in2out.port;
+
              if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
                {
-                 old_port0 = tcp0->dst_port;
-                 tcp0->dst_port = s0->in2out.port;
-                 new_port0 = tcp0->dst_port;
-
                  sum0 = tcp0->checksum;
                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
                                         ip4_header_t,
@@ -1472,18 +1621,23 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
                                         length /* changed member */ );
                  tcp0->checksum = ip_csum_fold (sum0);
                }
-             else
+             else if (udp0->checksum)
                {
-                 old_port0 = udp0->dst_port;
-                 udp0->dst_port = s0->in2out.port;
-                 udp0->checksum = 0;
+                 sum0 = udp0->checksum;
+                 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                        ip4_header_t,
+                                        dst_address /* changed member */ );
+                 sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                        ip4_header_t /* cheat */ ,
+                                        length /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
                }
            }
 
          /* Accounting */
          nat44_session_update_counters (s0, now,
-                                        vlib_buffer_length_in_chain (vm,
-                                                                     b0));
+                                        vlib_buffer_length_in_chain (vm, b0),
+                                        thread_index);
          /* Per-user LRU list maintenance */
          nat44_session_update_lru (sm, s0, thread_index);
 
@@ -1502,6 +1656,7 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
            {
              n_left_to_next++;
              to_next--;
+             cached_fragments++;
            }
          else
            {
@@ -1519,17 +1674,16 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
              u32 len = vec_len (fragments_to_loopback);
              if (len <= VLIB_FRAME_SIZE)
                {
-                 clib_memcpy (from, fragments_to_loopback,
-                              sizeof (u32) * len);
+                 clib_memcpy_fast (from, fragments_to_loopback,
+                                   sizeof (u32) * len);
                  n_left_from = len;
                  vec_reset_length (fragments_to_loopback);
                }
              else
                {
-                 clib_memcpy (from,
-                              fragments_to_loopback + (len -
-                                                       VLIB_FRAME_SIZE),
-                              sizeof (u32) * VLIB_FRAME_SIZE);
+                 clib_memcpy_fast (from, fragments_to_loopback +
+                                   (len - VLIB_FRAME_SIZE),
+                                   sizeof (u32) * VLIB_FRAME_SIZE);
                  n_left_from = VLIB_FRAME_SIZE;
                  _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
                }
@@ -1539,9 +1693,12 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
-  vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
-                              SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
+  vlib_node_increment_counter (vm, sm->out2in_reass_node_index,
+                              SNAT_OUT2IN_ERROR_PROCESSED_FRAGMENTS,
                               pkts_processed);
+  vlib_node_increment_counter (vm, sm->out2in_reass_node_index,
+                              SNAT_OUT2IN_ERROR_CACHED_FRAGMENTS,
+                              cached_fragments);
 
   nat_send_all_to_node (vm, fragments_to_drop, node,
                        &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
@@ -1554,7 +1711,6 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm,
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
-  .function = nat44_out2in_reass_node_fn,
   .name = "nat44-out2in-reass",
   .vector_size = sizeof (u32),
   .format_trace = format_nat44_reass_trace,
@@ -1575,12 +1731,9 @@ VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
 };
 /* *INDENT-ON* */
 
-VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
-                             nat44_out2in_reass_node_fn);
-
-static uword
-snat_out2in_fast_node_fn (vlib_main_t * vm,
-                         vlib_node_runtime_t * node, vlib_frame_t * frame)
+VLIB_NODE_FN (snat_out2in_fast_node) (vlib_main_t * vm,
+                                     vlib_node_runtime_t * node,
+                                     vlib_frame_t * frame)
 {
   u32 n_left_from, *from, *to_next;
   snat_out2in_next_t next_index;
@@ -1661,7 +1814,7 @@ snat_out2in_fast_node_fn (vlib_main_t * vm,
          key0.port = udp0->dst_port;
          key0.fib_index = rx_fib_index0;
 
-         if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
+         if (snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
            {
              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
              goto trace00;
@@ -1681,26 +1834,30 @@ snat_out2in_fast_node_fn (vlib_main_t * vm,
 
          if (PREDICT_FALSE (new_port0 != udp0->dst_port))
            {
+             old_port0 = udp0->dst_port;
+             udp0->dst_port = new_port0;
+
              if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
                {
-                 old_port0 = tcp0->dst_port;
-                 tcp0->dst_port = new_port0;
-
                  sum0 = tcp0->checksum;
                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
                                         ip4_header_t,
                                         dst_address /* changed member */ );
-
                  sum0 = ip_csum_update (sum0, old_port0, new_port0,
                                         ip4_header_t /* cheat */ ,
                                         length /* changed member */ );
                  tcp0->checksum = ip_csum_fold (sum0);
                }
-             else
+             else if (udp0->checksum)
                {
-                 old_port0 = udp0->dst_port;
-                 udp0->dst_port = new_port0;
-                 udp0->checksum = 0;
+                 sum0 = udp0->checksum;
+                 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                        ip4_header_t,
+                                        dst_address /* changed member */ );
+                 sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                        ip4_header_t /* cheat */ ,
+                                        length /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
                }
            }
          else
@@ -1711,9 +1868,16 @@ snat_out2in_fast_node_fn (vlib_main_t * vm,
                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
                                         ip4_header_t,
                                         dst_address /* changed member */ );
-
                  tcp0->checksum = ip_csum_fold (sum0);
                }
+             else if (udp0->checksum)
+               {
+                 sum0 = udp0->checksum;
+                 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                        ip4_header_t,
+                                        dst_address /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
+               }
            }
 
        trace00:
@@ -1738,7 +1902,7 @@ snat_out2in_fast_node_fn (vlib_main_t * vm,
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
-  vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
+  vlib_node_increment_counter (vm, sm->out2in_fast_node_index,
                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
                               pkts_processed);
   return frame->n_vectors;
@@ -1746,7 +1910,6 @@ snat_out2in_fast_node_fn (vlib_main_t * vm,
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
-  .function = snat_out2in_fast_node_fn,
   .name = "nat44-out2in-fast",
   .vector_size = sizeof (u32),
   .format_trace = format_snat_out2in_fast_trace,
@@ -1769,9 +1932,6 @@ VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
 };
 /* *INDENT-ON* */
 
-VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node,
-                             snat_out2in_fast_node_fn);
-
 /*
  * fd.io coding-style-patch-verification: ON
  *