api: multiple connections per process
[vpp.git] / src / plugins / nat / nat.c
index 85c4202..ee70b0f 100755 (executable)
 
 snat_main_t snat_main;
 
-/* *INDENT-OFF* */
+fib_source_t nat_fib_src_hi;
+fib_source_t nat_fib_src_low;
 
+/* *INDENT-OFF* */
 /* Hook up input features */
+VNET_FEATURE_INIT (nat_pre_in2out, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat-pre-in2out",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
+};
+VNET_FEATURE_INIT (nat_pre_out2in, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat-pre-out2in",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
+                               "ip4-dhcp-client-detect"),
+};
+VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-in2out-worker-handoff",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
+};
+VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-out2in-worker-handoff",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
+                               "ip4-dhcp-client-detect"),
+};
 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
   .arc_name = "ip4-unicast",
   .node_name = "nat44-in2out",
@@ -89,17 +113,6 @@ VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
   .node_name = "nat44-ed-classify",
   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 };
-VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "nat44-in2out-worker-handoff",
-  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
-};
-VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "nat44-out2in-worker-handoff",
-  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
-                               "ip4-dhcp-client-detect"),
-};
 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
   .arc_name = "ip4-unicast",
   .node_name = "nat44-handoff-classify",
@@ -526,7 +539,7 @@ snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
   if (is_add)
     fib_table_entry_update_one_path (fib_index,
                                     &prefix,
-                                    FIB_SOURCE_PLUGIN_LOW,
+                                    nat_fib_src_low,
                                     (FIB_ENTRY_FLAG_CONNECTED |
                                      FIB_ENTRY_FLAG_LOCAL |
                                      FIB_ENTRY_FLAG_EXCLUSIVE),
@@ -535,7 +548,7 @@ snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
                                     sw_if_index,
                                     ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
   else
-    fib_table_entry_delete (fib_index, &prefix, FIB_SOURCE_PLUGIN_LOW);
+    fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 }
 
 int
@@ -567,7 +580,7 @@ snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
   if (vrf_id != ~0)
     ap->fib_index =
       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
-                                        FIB_SOURCE_PLUGIN_LOW);
+                                        nat_fib_src_low);
   else
     ap->fib_index = ~0;
 #define _(N, i, n, s) \
@@ -803,7 +816,7 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
              local->vrf_id = vrf_id;
              local->fib_index =
                fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
-                                                  FIB_SOURCE_PLUGIN_LOW);
+                                                  nat_fib_src_low);
              m_key.addr = m->local_addr;
              m_key.port = m->local_port;
              m_key.protocol = m->proto;
@@ -824,13 +837,13 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
       if (vrf_id != ~0)
        fib_index =
          fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
-                                            FIB_SOURCE_PLUGIN_LOW);
+                                            nat_fib_src_low);
       /* If not specified use inside VRF id from SNAT plugin startup config */
       else
        {
          fib_index = sm->inside_fib_index;
          vrf_id = sm->inside_vrf_id;
-         fib_table_lock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
+         fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
        }
 
       if (!(out2in_only || identity_nat))
@@ -935,7 +948,7 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
          ip4_header_t ip = {
            .src_address = m->local_addr,
          };
-         vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
+         vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
          tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
        }
       else
@@ -1123,7 +1136,7 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
            }
        }
 
-      fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
+      fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
       if (pool_elts (m->locals))
        return 0;
 
@@ -1281,7 +1294,7 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
          locals[i].fib_index =
            fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
                                               locals[i].vrf_id,
-                                              FIB_SOURCE_PLUGIN_LOW);
+                                              nat_fib_src_low);
          m_key.addr = locals[i].addr;
          m_key.fib_index = locals[i].fib_index;
          if (!out2in_only)
@@ -1302,7 +1315,8 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
              };
              bitmap =
                clib_bitmap_set (bitmap,
-                                sm->worker_in2out_cb (&ip, m->fib_index), 1);
+                                sm->worker_in2out_cb (&ip, m->fib_index, 0),
+                                1);
            }
        }
 
@@ -1370,7 +1384,7 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       pool_foreach (local, m->locals,
       ({
           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
-                            FIB_SOURCE_PLUGIN_LOW);
+                            nat_fib_src_low);
           m_key.addr = local->addr;
           if (!out2in_only)
             {
@@ -1390,7 +1404,7 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
                 .src_address = local->addr,
               };
               tsm = vec_elt_at_index (sm->per_thread_data,
-                                      sm->worker_in2out_cb (&ip, m->fib_index));
+                                      sm->worker_in2out_cb (&ip, m->fib_index, 0));
             }
           else
             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
@@ -1503,7 +1517,7 @@ nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
       local->vrf_id = vrf_id;
       local->fib_index =
        fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
-                                          FIB_SOURCE_PLUGIN_LOW);
+                                          nat_fib_src_low);
 
       if (!is_out2in_only_static_mapping (m))
        {
@@ -1525,7 +1539,7 @@ nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
        return VNET_API_ERROR_UNSPECIFIED;
 
       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
-                       FIB_SOURCE_PLUGIN_LOW);
+                       nat_fib_src_low);
 
       if (!is_out2in_only_static_mapping (m))
        {
@@ -1543,7 +1557,8 @@ nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
            .src_address = local->addr,
          };
          tsm = vec_elt_at_index (sm->per_thread_data,
-                                 sm->worker_in2out_cb (&ip, m->fib_index));
+                                 sm->worker_in2out_cb (&ip, m->fib_index,
+                                                       0));
        }
       else
        tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
@@ -1596,7 +1611,7 @@ nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
         ip4_header_t ip;
         ip.src_address.as_u32 = local->addr.as_u32,
         bitmap = clib_bitmap_set (bitmap,
-                                  sm->worker_in2out_cb (&ip, local->fib_index),
+                                  sm->worker_in2out_cb (&ip, local->fib_index, 0),
                                   1);
       }
   }));
@@ -1675,7 +1690,7 @@ snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
     }
 
   if (a->fib_index != ~0)
-    fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
+    fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 
   /* Delete sessions using address */
   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
@@ -1773,18 +1788,20 @@ snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
       else if (sm->deterministic)
        feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
       else if (sm->endpoint_dependent)
-       feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
+       {
+         feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
+       }
       else
        feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
     }
 
   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
-    sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
-                                                     NAT_FQ_NELTS);
+    sm->fq_in2out_index =
+      vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
 
   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
-    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
-                                                     NAT_FQ_NELTS);
+    sm->fq_out2in_index =
+      vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
 
   if (!is_inside)
     {
@@ -1842,8 +1859,8 @@ feature_set:
                 else if (sm->endpoint_dependent)
                   {
                     del_feature_name = "nat44-ed-classify";
-                    feature_name = !is_inside ?  "nat44-ed-in2out" :
-                                                 "nat44-ed-out2in";
+                    feature_name = !is_inside ?  "nat-pre-in2out" :
+                                                 "nat-pre-out2in";
                   }
                 else
                   {
@@ -1905,8 +1922,9 @@ feature_set:
               }
             else if (sm->endpoint_dependent)
               {
-                del_feature_name = !is_inside ?  "nat44-ed-in2out" :
-                                                 "nat44-ed-out2in";
+                del_feature_name = !is_inside ?  "nat-pre-in2out" :
+                                                 "nat-pre-out2in";
+
                 feature_name = "nat44-ed-classify";
               }
             else
@@ -2072,7 +2090,7 @@ feature_set:
     {
       if (sm->endpoint_dependent)
        {
-         vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
+         vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
                                       sw_if_index, !is_del, 0, 0);
          vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
                                       sw_if_index, !is_del, 0, 0);
@@ -2089,11 +2107,11 @@ feature_set:
 fq:
   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
     sm->fq_in2out_output_index =
-      vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
+      vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
 
   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
     sm->fq_out2in_index =
-      vlib_frame_queue_main_init (sm->out2in_node_index, 0);
+      vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
 
   /* *INDENT-OFF* */
   pool_foreach (i, sm->output_feature_interfaces,
@@ -2160,6 +2178,7 @@ snat_set_workers (uword * bitmap)
     ({
       vec_add1(sm->workers, i);
       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
+      sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
       j++;
     }));
   /* *INDENT-ON* */
@@ -2196,6 +2215,16 @@ snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
           match = 1;
         }
     }));
+
+  pool_foreach (i, sm->output_feature_interfaces,
+    ({
+      if (i->sw_if_index == sw_if_index)
+        {
+          if (!(nat_interface_is_outside (i)))
+           return;
+          match = 1;
+        }
+    }));
   /* *INDENT-ON* */
 
   if (!match)
@@ -2280,7 +2309,7 @@ snat_init (vlib_main_t * vm)
   sm->vnet_main = vnet_get_main ();
   sm->ip4_main = im;
   sm->ip4_lookup_main = lm;
-  sm->api_main = &api_main;
+  sm->api_main = vlibapi_get_main ();
   sm->first_worker_index = 0;
   sm->num_workers = 0;
   sm->num_snat_thread = 1;
@@ -2297,12 +2326,25 @@ snat_init (vlib_main_t * vm)
   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
   sm->forwarding_enabled = 0;
   sm->log_class = vlib_log_register_class ("nat", 0);
-  sm->log_level = SNAT_LOG_NONE;
+  sm->log_level = SNAT_LOG_ERROR;
   sm->mss_clamping = 0;
 
   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
   sm->error_node_index = node->index;
 
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
+  sm->pre_in2out_node_index = node->index;
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
+  sm->pre_out2in_node_index = node->index;
+
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
+  sm->pre_in2out_node_index = node->index;
+
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
+  sm->pre_out2in_node_index = node->index;
+
+  // TODO: output ?? (special node)
+
   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
   sm->in2out_node_index = node->index;
   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
@@ -2425,6 +2467,13 @@ snat_init (vlib_main_t * vm)
   };
   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
 
+  nat_fib_src_hi = fib_source_allocate ("nat-hi",
+                                       FIB_SOURCE_PRIORITY_HI,
+                                       FIB_SOURCE_BH_SIMPLE);
+  nat_fib_src_low = fib_source_allocate ("nat-low",
+                                        FIB_SOURCE_PRIORITY_LOW,
+                                        FIB_SOURCE_BH_SIMPLE);
+
   /* Init virtual fragmenentation reassembly */
   return nat_reass_init (vm);
 }
@@ -2579,7 +2628,7 @@ snat_static_mapping_match (snat_main_t * sm,
                  .src_address = local->addr,
                };
 
-               if (sm->worker_in2out_cb (&ip, m->fib_index) ==
+               if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
                    thread_index)
                   {
                     vec_add1 (tmp, i);
@@ -2876,13 +2925,13 @@ nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
   if (is_add)
     {
       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
-      fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
+      fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
                                       FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
       dpo_reset (&dpo_v4);
     }
   else
     {
-      fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
+      fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
     }
 }
 
@@ -2946,7 +2995,8 @@ format_ed_session_kvp (u8 * s, va_list * args)
 }
 
 static u32
-snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
+snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
+                          u8 is_output)
 {
   snat_main_t *sm = &snat_main;
   u32 next_worker_index = 0;
@@ -2965,7 +3015,8 @@ snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
 }
 
 static u32
-snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
+snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0,
+                          u8 is_output)
 {
   snat_main_t *sm = &snat_main;
   udp_header_t *udp;
@@ -3101,16 +3152,178 @@ no_reass:
 }
 
 static u32
-nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
+nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
+                              u8 is_output)
+{
+  snat_main_t *sm = &snat_main;
+  u32 next_worker_index = sm->first_worker_index;
+  u32 hash;
+
+  clib_bihash_kv_16_8_t kv16, value16;
+  snat_main_per_thread_data_t *tsm;
+  udp_header_t *udp;
+
+  if (PREDICT_FALSE (is_output))
+    {
+      u32 fib_index = sm->outside_fib_index;
+      nat_outside_fib_t *outside_fib;
+      fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
+      fib_prefix_t pfx = {
+       .fp_proto = FIB_PROTOCOL_IP4,
+       .fp_len = 32,
+       .fp_addr = {
+                   .ip4.as_u32 = ip->dst_address.as_u32,
+                   }
+       ,
+      };
+
+      udp = ip4_next_header (ip);
+
+      switch (vec_len (sm->outside_fibs))
+       {
+       case 0:
+         fib_index = sm->outside_fib_index;
+         break;
+       case 1:
+         fib_index = sm->outside_fibs[0].fib_index;
+         break;
+       default:
+            /* *INDENT-OFF* */
+            vec_foreach (outside_fib, sm->outside_fibs)
+              {
+                fei = fib_table_lookup (outside_fib->fib_index, &pfx);
+                if (FIB_NODE_INDEX_INVALID != fei)
+                  {
+                    if (fib_entry_get_resolving_interface (fei) != ~0)
+                      {
+                        fib_index = outside_fib->fib_index;
+                        break;
+                      }
+                  }
+              }
+            /* *INDENT-ON* */
+         break;
+       }
+
+      make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
+                 ip->protocol, fib_index, udp->src_port, udp->dst_port);
+
+      /* *INDENT-OFF* */
+      vec_foreach (tsm, sm->per_thread_data)
+        {
+          if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
+                                                      &kv16, &value16)))
+            {
+              next_worker_index += tsm->thread_index;
+
+              nat_elog_debug_handoff (
+                "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
+                next_worker_index, fib_index,
+               clib_net_to_host_u32 (ip->src_address.as_u32),
+               clib_net_to_host_u32 (ip->dst_address.as_u32));
+
+              return next_worker_index;
+            }
+        }
+      /* *INDENT-ON* */
+    }
+
+  hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
+    (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
+
+  if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
+    next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
+  else
+    next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
+
+  if (PREDICT_TRUE (!is_output))
+    {
+      nat_elog_debug_handoff ("HANDOFF IN2OUT",
+                             next_worker_index, rx_fib_index,
+                             clib_net_to_host_u32 (ip->src_address.as_u32),
+                             clib_net_to_host_u32 (ip->dst_address.as_u32));
+    }
+  else
+    {
+      nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
+                             next_worker_index, rx_fib_index,
+                             clib_net_to_host_u32 (ip->src_address.as_u32),
+                             clib_net_to_host_u32 (ip->dst_address.as_u32));
+    }
+
+  return next_worker_index;
+}
+
+static u32
+nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index,
+                              u8 is_output)
 {
   snat_main_t *sm = &snat_main;
   clib_bihash_kv_8_8_t kv, value;
+  clib_bihash_kv_16_8_t kv16, value16;
+  snat_main_per_thread_data_t *tsm;
+
   u32 proto, next_worker_index = 0;
   udp_header_t *udp;
   u16 port;
   snat_static_mapping_t *m;
   u32 hash;
 
+  proto = ip_proto_to_snat_proto (ip->protocol);
+
+  if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
+    {
+      udp = ip4_next_header (ip);
+
+      make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
+                 ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
+
+      /* *INDENT-OFF* */
+      vec_foreach (tsm, sm->per_thread_data)
+        {
+          if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
+                                                      &kv16, &value16)))
+            {
+              next_worker_index = sm->first_worker_index + tsm->thread_index;
+              nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
+                          next_worker_index, rx_fib_index,
+                         clib_net_to_host_u32 (ip->src_address.as_u32),
+                         clib_net_to_host_u32 (ip->dst_address.as_u32));
+              return next_worker_index;
+            }
+          }
+        /* *INDENT-ON* */
+    }
+  else if (proto == SNAT_PROTOCOL_ICMP)
+    {
+      nat_ed_ses_key_t key;
+
+      if (!get_icmp_o2i_ed_key (ip, &key))
+       {
+
+         key.fib_index = rx_fib_index;
+         kv16.key[0] = key.as_u64[0];
+         kv16.key[1] = key.as_u64[1];
+
+          /* *INDENT-OFF* */
+          vec_foreach (tsm, sm->per_thread_data)
+            {
+              if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
+                                                          &kv16, &value16)))
+                {
+                  next_worker_index = sm->first_worker_index +
+                                      tsm->thread_index;
+                  nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
+                              next_worker_index, rx_fib_index,
+                             clib_net_to_host_u32 (ip->src_address.as_u32),
+                             clib_net_to_host_u32 (ip->dst_address.as_u32));
+                  return next_worker_index;
+                }
+            }
+          /* *INDENT-ON* */
+       }
+    }
+
   /* first try static mappings without port */
   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
     {
@@ -3119,17 +3332,17 @@ nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
          (&sm->static_mapping_by_external, &kv, &value))
        {
          m = pool_elt_at_index (sm->static_mappings, value.value);
-         return m->workers[0];
+         next_worker_index = m->workers[0];
+         goto done;
        }
     }
 
-  proto = ip_proto_to_snat_proto (ip->protocol);
-
   /* unknown protocol */
   if (PREDICT_FALSE (proto == ~0))
     {
       /* use current thread */
-      return vlib_get_thread_index ();
+      next_worker_index = vlib_get_thread_index ();
+      goto done;
     }
 
   udp = ip4_next_header (ip);
@@ -3158,7 +3371,8 @@ nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
              port = ((tcp_udp_header_t *) l4_header)->src_port;
              break;
            default:
-             return vlib_get_thread_index ();
+             next_worker_index = vlib_get_thread_index ();
+             goto done;
            }
        }
     }
@@ -3173,15 +3387,20 @@ nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
        {
          m = pool_elt_at_index (sm->static_mappings, value.value);
          if (!is_lb_static_mapping (m))
-           return m->workers[0];
+           {
+             next_worker_index = m->workers[0];
+             goto done;
+           }
 
          hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
            (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
 
          if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
-           return m->workers[hash & (_vec_len (m->workers) - 1)];
+           next_worker_index =
+             m->workers[hash & (_vec_len (m->workers) - 1)];
          else
-           return m->workers[hash % _vec_len (m->workers)];
+           next_worker_index = m->workers[hash % _vec_len (m->workers)];
+         goto done;
        }
     }
 
@@ -3190,6 +3409,10 @@ nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
   next_worker_index +=
     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
 
+done:
+  nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
+                         clib_net_to_host_u32 (ip->src_address.as_u32),
+                         clib_net_to_host_u32 (ip->dst_address.as_u32));
   return next_worker_index;
 }
 
@@ -3623,15 +3846,15 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
   sm->outside_vrf_id = outside_vrf_id;
   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
                                                             outside_vrf_id,
-                                                            FIB_SOURCE_PLUGIN_HI);
+                                                            nat_fib_src_hi);
   nm->outside_vrf_id = outside_ip6_vrf_id;
   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
                                                             outside_ip6_vrf_id,
-                                                            FIB_SOURCE_PLUGIN_HI);
+                                                            nat_fib_src_hi);
   sm->inside_vrf_id = inside_vrf_id;
   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
                                                            inside_vrf_id,
-                                                           FIB_SOURCE_PLUGIN_HI);
+                                                           nat_fib_src_hi);
   sm->static_mapping_only = static_mapping_only;
   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
 
@@ -3650,11 +3873,18 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
     {
       if (sm->endpoint_dependent)
        {
-         sm->worker_in2out_cb = snat_get_worker_in2out_cb;
+         sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
          sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
+
+         sm->handoff_out2in_index = nat_pre_out2in_node.index;
+         sm->handoff_in2out_index = nat_pre_in2out_node.index;
+         // TODO: test
+         sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
+
          sm->in2out_node_index = nat44_ed_in2out_node.index;
          sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
          sm->out2in_node_index = nat44_ed_out2in_node.index;
+
          sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
          sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
          nat_affinity_init (vm);
@@ -3665,6 +3895,12 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
        {
          sm->worker_in2out_cb = snat_get_worker_in2out_cb;
          sm->worker_out2in_cb = snat_get_worker_out2in_cb;
+
+         sm->handoff_out2in_index = snat_in2out_node.index;
+         sm->handoff_in2out_index = snat_out2in_node.index;
+         // TODO: test
+         sm->handoff_in2out_output_index = snat_in2out_output_node.index;
+
          sm->in2out_node_index = snat_in2out_node.index;
          sm->in2out_output_node_index = snat_in2out_output_node.index;
          sm->out2in_node_index = snat_out2in_node.index;
@@ -3976,7 +4212,7 @@ nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
   if (sm->num_workers > 1)
     tsm =
       vec_elt_at_index (sm->per_thread_data,
-                       sm->worker_in2out_cb (&ip, fib_index));
+                       sm->worker_in2out_cb (&ip, fib_index, 0));
   else
     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 
@@ -4020,7 +4256,7 @@ nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
   if (sm->num_workers > 1)
     tsm =
       vec_elt_at_index (sm->per_thread_data,
-                       sm->worker_in2out_cb (&ip, fib_index));
+                       sm->worker_in2out_cb (&ip, fib_index, 0));
   else
     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 
@@ -4076,6 +4312,38 @@ nat_set_alloc_addr_and_port_default (void)
   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
 }
 
+VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
+                                vlib_node_runtime_t * node,
+                                vlib_frame_t * frame)
+{
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat_default_node) = {
+  .name = "nat-default",
+  .vector_size = sizeof (u32),
+  .format_trace = 0,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = 0,
+  .n_next_nodes = NAT_N_NEXT,
+  .next_nodes = {
+    [NAT_NEXT_DROP] = "error-drop",
+    [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+    [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
+    [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
+    [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
+    [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
+    [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
+    [NAT_NEXT_IN2OUT_ED_REASS] = "nat44-ed-in2out-reass",
+    [NAT_NEXT_IN2OUT_ED_OUTPUT_REASS] = "nat44-ed-in2out-reass-output",
+    [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
+    [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
+    [NAT_NEXT_OUT2IN_ED_REASS] = "nat44-ed-out2in-reass",
+  },
+};
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *