nat: cleanup & reorganization
[vpp.git] / src / plugins / nat / nat.h
index d2b114a..710c86f 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Copyright (c) 2020 Cisco and/or its affiliates.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at:
@@ -13,7 +13,8 @@
  * limitations under the License.
  */
 /**
- * @file NAT plugin global declarations
+ * @file nat.c
+ * NAT plugin global declarations
  */
 #ifndef __included_nat_h__
 #define __included_nat_h__
 #include <vlibapi/api.h>
 #include <vlib/log.h>
 #include <vppinfra/bihash_16_8.h>
-
-/* default session timeouts */
-#define SNAT_UDP_TIMEOUT 300
-#define SNAT_TCP_TRANSITORY_TIMEOUT 240
-#define SNAT_TCP_ESTABLISHED_TIMEOUT 7440
-#define SNAT_ICMP_TIMEOUT 60
+#include <nat/lib/lib.h>
+#include <nat/lib/inlines.h>
 
 /* number of worker handoff frame queue elements */
 #define NAT_FQ_NELTS 64
 /* NAT buffer flags */
 #define SNAT_FLAG_HAIRPINNING (1 << 0)
 
+/* NAT44 API Configuration flags */
+#define foreach_nat44_config_flag  \
+  _(0x00, IS_ENDPOINT_INDEPENDENT) \
+  _(0x01, IS_ENDPOINT_DEPENDENT)   \
+  _(0x02, IS_STATIC_MAPPING_ONLY)  \
+  _(0x04, IS_CONNECTION_TRACKING)  \
+  _(0x08, IS_OUT2IN_DPO)
+
+typedef enum nat44_config_flags_t_
+{
+#define _(n,f) NAT44_API_##f = n,
+  foreach_nat44_config_flag
+#undef _
+} nat44_config_flags_t;
+
 typedef struct
 {
-  u32 arc_next;
-} nat_buffer_opaque_t;
+  /* nat44 plugin features */
+  u8 static_mapping_only;
+  u8 connection_tracking;
+  u8 endpoint_dependent;
+  u8 out2in_dpo;
+
+  u32 inside_vrf;
+  u32 outside_vrf;
+
+  /* maximum number of users */
+  u32 users;
+
+  /* maximum number of sessions */
+  u32 sessions;
+
+  /* maximum number of ssessions per user */
+  u32 user_sessions;
+} nat44_config_t;
 
 typedef enum
 {
   NAT_NEXT_DROP,
   NAT_NEXT_ICMP_ERROR,
-  NAT_NEXT_IN2OUT_PRE,
-  NAT_NEXT_OUT2IN_PRE,
   NAT_NEXT_IN2OUT_ED_FAST_PATH,
   NAT_NEXT_IN2OUT_ED_SLOW_PATH,
+  NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH,
   NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH,
   NAT_NEXT_OUT2IN_ED_FAST_PATH,
   NAT_NEXT_OUT2IN_ED_SLOW_PATH,
+  NAT_NEXT_IN2OUT_CLASSIFY,
+  NAT_NEXT_OUT2IN_CLASSIFY,
   NAT_N_NEXT,
 } nat_next_t;
 
 typedef struct
 {
   u32 next_index;
+  u32 arc_next_index;
 } nat_pre_trace_t;
 
-#define nat_buffer_opaque(b) \
-  ((nat_buffer_opaque_t *)((vnet_buffer_opaque2_t *)b->opaque2)->__unused2)
-
-/*
-STATIC_ASSERT (sizeof (nat_buffer_opaque_t) <=
-               STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused),
-               "Custom meta-data too large for vnet_buffer_opaque_t");
-
-#define nat_buffer_opaque(b) \
-  ((nat_buffer_opaque_t *)((u8 *)((b)->opaque) + \
-    STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))*/
-
-/* session key (4-tuple) */
-typedef struct
-{
-  union
-  {
-    struct
-    {
-      ip4_address_t addr;
-      u16 port;
-      u16 protocol:3, fib_index:13;
-    };
-    u64 as_u64;
-  };
-} snat_session_key_t;
-
-/* endpoint-dependent session key (6-tuple) */
-typedef struct
-{
-  union
-  {
-    struct
-    {
-      ip4_address_t l_addr;
-      ip4_address_t r_addr;
-      u32 proto:8, fib_index:24;
-      u16 l_port;
-      u16 r_port;
-    };
-    u64 as_u64[2];
-  };
-} nat_ed_ses_key_t;
-
-/* deterministic session outside key */
-typedef struct
-{
-  union
-  {
-    struct
-    {
-      ip4_address_t ext_host_addr;
-      u16 ext_host_port;
-      u16 out_port;
-    };
-    u64 as_u64;
-  };
-} snat_det_out_key_t;
-
 /* user (internal host) key */
 typedef struct
 {
@@ -142,31 +113,6 @@ typedef struct
   };
 } snat_user_key_t;
 
-typedef struct
-{
-  u32 sw_if_index;
-  u32 next_index;
-  u8 cached;
-} nat44_reass_trace_t;
-
-/* NAT API Configuration flags */
-#define foreach_nat_config_flag \
-  _(0x01, IS_TWICE_NAT)         \
-  _(0x02, IS_SELF_TWICE_NAT)    \
-  _(0x04, IS_OUT2IN_ONLY)       \
-  _(0x08, IS_ADDR_ONLY)         \
-  _(0x10, IS_OUTSIDE)           \
-  _(0x20, IS_INSIDE)            \
-  _(0x40, IS_STATIC)            \
-  _(0x80, IS_EXT_HOST_VALID)    \
-
-typedef enum nat_config_flags_t_
-{
-#define _(n,f) NAT_API_##f = n,
-  foreach_nat_config_flag
-#undef _
-} nat_config_flags_t;
-
 /* External address and port allocation modes */
 #define foreach_nat_addr_and_port_alloc_alg \
   _(0, DEFAULT, "default")         \
@@ -180,21 +126,6 @@ typedef enum
 #undef _
 } nat_addr_and_port_alloc_alg_t;
 
-
-/* Supported L4 protocols */
-#define foreach_snat_protocol \
-  _(UDP, 0, udp, "udp")       \
-  _(TCP, 1, tcp, "tcp")       \
-  _(ICMP, 2, icmp, "icmp")
-
-typedef enum
-{
-#define _(N, i, n, s) SNAT_PROTOCOL_##N = i,
-  foreach_snat_protocol
-#undef _
-} snat_protocol_t;
-
-
 /* Session state */
 #define foreach_snat_session_state          \
   _(0, UNKNOWN, "unknown")                 \
@@ -217,24 +148,11 @@ typedef enum
 
 #define foreach_nat_in2out_ed_error                     \
 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
-_(IN2OUT_PACKETS, "good in2out packets processed")      \
 _(OUT_OF_PORTS, "out of ports")                         \
 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
-_(MAX_USER_SESS_EXCEEDED, "max user sessions exceeded") \
-_(DROP_FRAGMENT, "drop fragment")                       \
-_(MAX_REASS, "maximum reassemblies exceeded")           \
-_(MAX_FRAG, "maximum fragments per reassembly exceeded")\
-_(CANNOT_CREATE_USER, "cannot create NAT user")         \
 _(NON_SYN, "non-SYN packet try to create session")      \
-_(TCP_PACKETS, "TCP packets")                           \
-_(TCP_CLOSED, "drops due to TCP in transitory timeout") \
-_(UDP_PACKETS, "UDP packets")                           \
-_(ICMP_PACKETS, "ICMP packets")                         \
-_(OTHER_PACKETS, "other protocol packets")              \
-_(FRAGMENTS, "fragments")                               \
-_(CACHED_FRAGMENTS, "cached fragments")                 \
-_(PROCESSED_FRAGMENTS, "processed fragments")
+_(TCP_CLOSED, "drops due to TCP in transitory timeout")
 
 typedef enum
 {
@@ -246,25 +164,14 @@ typedef enum
 
 #define foreach_nat_out2in_ed_error                     \
 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
-_(OUT2IN_PACKETS, "good out2in packets processed")      \
 _(OUT_OF_PORTS, "out of ports")                         \
 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 _(NO_TRANSLATION, "no translation")                     \
 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")   \
 _(MAX_USER_SESS_EXCEEDED, "max user sessions exceeded") \
-_(DROP_FRAGMENT, "drop fragment")                       \
-_(MAX_REASS, "maximum reassemblies exceeded")           \
-_(MAX_FRAG, "maximum fragments per reassembly exceeded")\
 _(CANNOT_CREATE_USER, "cannot create NAT user")         \
 _(NON_SYN, "non-SYN packet try to create session")      \
-_(TCP_PACKETS, "TCP packets")                           \
-_(TCP_CLOSED, "drops due to TCP in transitory timeout") \
-_(UDP_PACKETS, "UDP packets")                           \
-_(ICMP_PACKETS, "ICMP packets")                         \
-_(OTHER_PACKETS, "other protocol packets")              \
-_(FRAGMENTS, "fragments")                               \
-_(CACHED_FRAGMENTS, "cached fragments")                 \
-_(PROCESSED_FRAGMENTS, "processed fragments")
+_(TCP_CLOSED, "drops due to TCP in transitory timeout")
 
 typedef enum
 {
@@ -293,25 +200,53 @@ typedef enum
 #define SNAT_SESSION_FLAG_FWD_BYPASS           32
 #define SNAT_SESSION_FLAG_AFFINITY             64
 #define SNAT_SESSION_FLAG_OUTPUT_FEATURE       128
+#define SNAT_SESSION_FLAG_EXACT_ADDRESS        256
 
 /* NAT interface flags */
 #define NAT_INTERFACE_FLAG_IS_INSIDE 1
 #define NAT_INTERFACE_FLAG_IS_OUTSIDE 2
 
 /* Static mapping flags */
-#define NAT_STATIC_MAPPING_FLAG_ADDR_ONLY    1
-#define NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY  2
-#define NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT 4
-#define NAT_STATIC_MAPPING_FLAG_LB           8
+#define NAT_STATIC_MAPPING_FLAG_ADDR_ONLY      1
+#define NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY    2
+#define NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT   4
+#define NAT_STATIC_MAPPING_FLAG_LB             8
+#define NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS  16
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED(struct
+{
+  // number of sessions in this vrf
+  u32 ses_count;
+
+  u32 rx_fib_index;
+  u32 tx_fib_index;
+
+  // is this vrf expired
+  u8 expired;
+}) per_vrf_sessions_t;
+/* *INDENT-ON* */
 
 /* *INDENT-OFF* */
 typedef CLIB_PACKED(struct
 {
-  /* Outside network key */
-  snat_session_key_t out2in;
+  /* Outside network tuple */
+  struct
+  {
+    ip4_address_t addr;
+    u32 fib_index;
+    u16 port;
+  } out2in;
+
+  /* Inside network tuple */
+  struct
+  {
+    ip4_address_t addr;
+    u32 fib_index;
+    u16 port;
+  } in2out;
 
-  /* Inside network key */
-  snat_session_key_t in2out;
+  nat_protocol_t nat_proto;
 
   /* Flags */
   u32 flags;
@@ -320,8 +255,10 @@ typedef CLIB_PACKED(struct
   u32 per_user_index;
   u32 per_user_list_head_index;
 
+  /* head of LRU list in which this session is tracked */
+  u32 lru_head_index;
   /* index in global LRU list */
-  u32 global_lru_index;
+  u32 lru_index;
   f64 last_lru_update;
 
   /* Last heard timer */
@@ -346,14 +283,17 @@ typedef CLIB_PACKED(struct
   u8 state;
   u32 i2o_fin_seq;
   u32 o2i_fin_seq;
-  u32 tcp_close_timestamp;
+  u64 tcp_closed_timestamp;
 
   /* user index */
   u32 user_index;
+
+  /* per vrf sessions index */
+  u32 per_vrf_sessions_index;
+
 }) snat_session_t;
 /* *INDENT-ON* */
 
-
 typedef struct
 {
   ip4_address_t addr;
@@ -361,8 +301,6 @@ typedef struct
   u32 sessions_per_user_list_head_index;
   u32 nsessions;
   u32 nstaticsessions;
-  /* discovered minimum session timeout time */
-  u64 min_session_timeout;
 } snat_user_t;
 
 typedef struct
@@ -371,10 +309,10 @@ typedef struct
   u32 fib_index;
 /* *INDENT-OFF* */
 #define _(N, i, n, s) \
-  u16 busy_##n##_ports; \
-  u16 * busy_##n##_ports_per_thread; \
+  u32 busy_##n##_ports; \
+  u32 * busy_##n##_ports_per_thread; \
   u32 busy_##n##_port_refcounts[65535];
-  foreach_snat_protocol
+  foreach_nat_protocol
 #undef _
 /* *INDENT-ON* */
 } snat_address_t;
@@ -382,38 +320,14 @@ typedef struct
 typedef struct
 {
   u32 fib_index;
-  u32 refcount;
-} nat_outside_fib_t;
-
-typedef struct
-{
-  /* Inside network port */
-  u16 in_port;
-  /* Outside network address and port */
-  snat_det_out_key_t out;
-  /* Session state */
-  u8 state;
-  /* Expire timeout */
-  u32 expire;
-} snat_det_session_t;
+  u32 ref_count;
+} nat_fib_t;
 
 typedef struct
 {
-  /* inside IP address range */
-  ip4_address_t in_addr;
-  u8 in_plen;
-  /* outside IP address range */
-  ip4_address_t out_addr;
-  u8 out_plen;
-  /* inside IP addresses / outside IP addresses */
-  u32 sharing_ratio;
-  /* number of ports available to internal host */
-  u16 ports_per_host;
-  /* session counter */
-  u32 ses_num;
-  /* vector of sessions */
-  snat_det_session_t *sessions;
-} snat_det_map_t;
+  u32 fib_index;
+  u32 refcount;
+} nat_outside_fib_t;
 
 typedef struct
 {
@@ -451,6 +365,8 @@ typedef enum
 
 typedef struct
 {
+  /* prefered pool address */
+  ip4_address_t pool_addr;
   /* local IP address */
   ip4_address_t local_addr;
   /* external IP address */
@@ -465,7 +381,7 @@ typedef struct
   u32 vrf_id;
   u32 fib_index;
   /* protocol */
-  snat_protocol_t proto;
+  nat_protocol_t proto;
   /* 0 = disabled, otherwise client IP affinity sticky time in seconds */
   u32 affinity;
   /* worker threads used by backends/local host */
@@ -489,17 +405,19 @@ typedef struct
 typedef struct
 {
   ip4_address_t l_addr;
+  ip4_address_t pool_addr;
   u16 l_port;
   u16 e_port;
   u32 sw_if_index;
   u32 vrf_id;
-  snat_protocol_t proto;
+  nat_protocol_t proto;
   u32 flags;
   int addr_only;
   int twice_nat;
   int is_add;
   int out2in_only;
   int identity_nat;
+  int exact;
   u8 *tag;
 } snat_static_map_resolve_t;
 
@@ -510,7 +428,6 @@ typedef struct
   clib_bihash_8_8_t in2out;
 
   /* Endpoint dependent sessions lookup tables */
-  clib_bihash_16_8_t out2in_ed;
   clib_bihash_16_8_t in2out_ed;
 
   /* Find-a-user => src address lookup */
@@ -526,8 +443,12 @@ typedef struct
   dlist_elt_t *list_pool;
 
   /* LRU session list - head is stale, tail is fresh */
-  dlist_elt_t *global_lru_pool;
-  u32 global_lru_head_index;
+  dlist_elt_t *lru_pool;
+  u32 tcp_trans_lru_head_index;
+  u32 tcp_estab_lru_head_index;
+  u32 udp_lru_head_index;
+  u32 icmp_lru_head_index;
+  u32 unk_proto_lru_head_index;
 
   /* NAT thread index */
   u32 snat_thread_index;
@@ -535,13 +456,7 @@ typedef struct
   /* real thread index */
   u32 thread_index;
 
-  /* discovered minimum session timeout time */
-  u64 min_session_timeout;
-
-  /* session scavenging */
-  u32 cleared;
-  u32 cleanup_runs;
-  f64 cleanup_timeout;
+  per_vrf_sessions_t *per_vrf_sessions_vec;
 
 } snat_main_per_thread_data_t;
 
@@ -552,10 +467,13 @@ typedef u32 (snat_icmp_match_function_t) (struct snat_main_s * sm,
                                          vlib_node_runtime_t * node,
                                          u32 thread_index,
                                          vlib_buffer_t * b0,
-                                         ip4_header_t * ip0, u8 * p_proto,
-                                         snat_session_key_t * p_value,
-                                         u8 * p_dont_translate, void *d,
-                                         void *e);
+                                         ip4_header_t * ip0,
+                                         ip4_address_t * addr,
+                                         u16 * port,
+                                         u32 * fib_index,
+                                         nat_protocol_t * proto,
+                                         void *d, void *e,
+                                         u8 * dont_translate);
 
 /* Return worker thread index for given packet */
 typedef u32 (snat_get_worker_in2out_function_t) (ip4_header_t * ip,
@@ -567,36 +485,18 @@ typedef u32 (snat_get_worker_out2in_function_t) (vlib_buffer_t * b,
                                                 u32 rx_fib_index,
                                                 u8 is_output);
 
-/* NAT address and port allacotaion function */
+/* NAT address and port allocation function */
 typedef int (nat_alloc_out_addr_and_port_function_t) (snat_address_t *
                                                      addresses,
                                                      u32 fib_index,
                                                      u32 thread_index,
-                                                     snat_session_key_t * k,
+                                                     nat_protocol_t proto,
+                                                     ip4_address_t * addr,
+                                                     u16 * port,
                                                      u16 port_per_thread,
                                                      u32 snat_thread_index);
 
-typedef struct ed_bihash_key_s
-{
-  u32 src_address;
-  u32 dst_address;
-  u16 src_port;
-  u16 dst_port;
-  u8 protocol;
-} ed_bihash_key_t;
-
-typedef struct ed_bihash_kv_s
-{
-  union
-  {
-    ed_bihash_key_t k;
-    clib_bihash_kv_16_8_t kv;
-  };
-} ed_bihash_kv_t;
-
-STATIC_ASSERT (STRUCT_SIZE_OF (ed_bihash_kv_t, k) <=
-              STRUCT_SIZE_OF (ed_bihash_kv_t, kv.key),
-              "ed key needs to fit in bihash key");
+#define foreach_nat_counter _ (tcp) _ (udp) _ (icmp) _ (other) _ (drops)
 
 typedef struct snat_main_s
 {
@@ -611,7 +511,6 @@ typedef struct snat_main_s
   snat_get_worker_in2out_function_t *worker_in2out_cb;
   snat_get_worker_out2in_function_t *worker_out2in_cb;
   u16 port_per_thread;
-  u32 num_snat_thread;
 
   /* Per thread data */
   snat_main_per_thread_data_t *per_thread_data;
@@ -625,6 +524,9 @@ typedef struct snat_main_s
   /* Static mapping pool */
   snat_static_mapping_t *static_mappings;
 
+  /* Endpoint-dependent out2in mappings */
+  clib_bihash_16_8_t out2in_ed;
+
   /* Interface pool */
   snat_interface_t *interfaces;
   snat_interface_t *output_feature_interfaces;
@@ -643,10 +545,13 @@ typedef struct snat_main_s
   u16 start_port;
   u16 end_port;
 
+  /* vector of fibs */
+  nat_fib_t *fibs;
+
   /* vector of outside fibs */
   nat_outside_fib_t *outside_fibs;
 
-  /* Vector of twice NAT addresses for extenal hosts */
+  /* Vector of twice NAT addresses for external hosts */
   snat_address_t *twice_nat_addresses;
 
   /* sw_if_indices whose intfc addresses should be auto-added */
@@ -676,19 +581,25 @@ typedef struct snat_main_s
   u32 pre_out2in_node_index;
   u32 pre_in2out_node_index;
 
+  u32 out2in_node_index;
   u32 in2out_node_index;
   u32 in2out_output_node_index;
+
   u32 in2out_fast_node_index;
   u32 in2out_slowpath_node_index;
   u32 in2out_slowpath_output_node_index;
-  u32 ed_in2out_node_index;
-  u32 ed_in2out_slowpath_node_index;
-  u32 out2in_node_index;
   u32 out2in_fast_node_index;
+
+  u32 ei_out2in_node_index;
+  u32 ei_in2out_node_index;
+  u32 ei_in2out_output_node_index;
+
   u32 ed_out2in_node_index;
+  u32 ed_in2out_node_index;
+  u32 ed_in2out_output_node_index;
+
+  u32 ed_in2out_slowpath_node_index;
   u32 ed_out2in_slowpath_node_index;
-  u32 det_in2out_node_index;
-  u32 det_out2in_node_index;
 
   u32 hairpinning_node_index;
   u32 hairpin_dst_node_index;
@@ -697,47 +608,97 @@ typedef struct snat_main_s
   u32 ed_hairpin_dst_node_index;
   u32 ed_hairpin_src_node_index;
 
-
-  /* Deterministic NAT mappings */
-  snat_det_map_t *det_maps;
+  nat44_config_t rconfig;
+  //nat44_config_t cconfig;
 
   /* If forwarding is enabled */
   u8 forwarding_enabled;
 
   /* Config parameters */
+  u8 endpoint_dependent;
+
+  u8 out2in_dpo;
+  /* static mapping config */
   u8 static_mapping_only;
   u8 static_mapping_connection_tracking;
-  u8 deterministic;
-  u8 out2in_dpo;
-  u8 endpoint_dependent;
+
+  /* Is translation memory size calculated or user defined */
+  u8 translation_memory_size_set;
+
   u32 translation_buckets;
-  u32 translation_memory_size;
-  u32 max_translations;
+  u32 max_translations_per_thread;
+  u32 *max_translations_per_fib;
+  u32 max_users_per_thread;
   u32 user_buckets;
-  u32 user_memory_size;
   u32 max_translations_per_user;
+
   u32 outside_vrf_id;
   u32 outside_fib_index;
   u32 inside_vrf_id;
   u32 inside_fib_index;
 
-  /* values of various timeouts */
-
-  // min timeout of all proto timeouts
-  u32 min_timeout;
-  // proto timeouts
-  u32 udp_timeout;
-  u32 tcp_transitory_timeout;
-  u32 tcp_established_timeout;
-  u32 icmp_timeout;
+  nat_timeouts_t timeouts;
 
   /* TCP MSS clamping */
   u16 mss_clamping;
-  u16 mss_value_net;
 
-  /* counters/gauges */
+  /* counters */
   vlib_simple_counter_main_t total_users;
   vlib_simple_counter_main_t total_sessions;
+  vlib_simple_counter_main_t user_limit_reached;
+
+#define _(x) vlib_simple_counter_main_t x;
+  struct
+  {
+    struct
+    {
+      struct
+      {
+       foreach_nat_counter;
+      } in2out;
+
+      struct
+      {
+       foreach_nat_counter;
+      } out2in;
+
+      struct
+      {
+       foreach_nat_counter;
+      } in2out_ed;
+
+      struct
+      {
+       foreach_nat_counter;
+      } out2in_ed;
+    } fastpath;
+
+    struct
+    {
+      struct
+      {
+       foreach_nat_counter;
+      } in2out;
+
+      struct
+      {
+       foreach_nat_counter;
+      } out2in;
+
+      struct
+      {
+       foreach_nat_counter;
+      } in2out_ed;
+
+      struct
+      {
+       foreach_nat_counter;
+      } out2in_ed;
+    } slowpath;
+
+    vlib_simple_counter_main_t hairpinning;
+  } counters;
+#undef _
 
   /* API message ID base */
   u16 msg_id_base;
@@ -748,13 +709,17 @@ typedef struct snat_main_s
   u8 log_level;
 
   /* convenience */
-  vlib_main_t *vlib_main;
-  vnet_main_t *vnet_main;
+  api_main_t *api_main;
   ip4_main_t *ip4_main;
   ip_lookup_main_t *ip4_lookup_main;
-  api_main_t *api_main;
 
-  clib_bihash_16_8_t ed_ext_ports;
+  fib_source_t fib_src_hi;
+  fib_source_t fib_src_low;
+
+  /* nat44 plugin enabled */
+  u8 enabled;
+
+  vnet_main_t *vnet_main;
 } snat_main_t;
 
 typedef struct
@@ -782,8 +747,6 @@ extern vlib_node_registration_t snat_out2in_node;
 extern vlib_node_registration_t snat_in2out_worker_handoff_node;
 extern vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 extern vlib_node_registration_t snat_out2in_worker_handoff_node;
-extern vlib_node_registration_t snat_det_in2out_node;
-extern vlib_node_registration_t snat_det_out2in_node;
 extern vlib_node_registration_t nat44_ed_in2out_node;
 extern vlib_node_registration_t nat44_ed_in2out_output_node;
 extern vlib_node_registration_t nat44_ed_out2in_node;
@@ -793,17 +756,15 @@ extern fib_source_t nat_fib_src_low;
 
 /* format functions */
 format_function_t format_snat_user;
-format_function_t format_snat_user_v2;
 format_function_t format_snat_static_mapping;
 format_function_t format_snat_static_map_to_resolve;
 format_function_t format_snat_session;
-format_function_t format_det_map_ses;
 format_function_t format_snat_key;
 format_function_t format_static_mapping_key;
-format_function_t format_snat_protocol;
+format_function_t format_nat_protocol;
 format_function_t format_nat_addr_and_port_alloc_alg;
 /* unformat functions */
-unformat_function_t unformat_snat_protocol;
+unformat_function_t unformat_nat_protocol;
 
 /** \brief Check if SNAT session is created from static mapping.
     @param s SNAT session
@@ -847,14 +808,20 @@ unformat_function_t unformat_snat_protocol;
 */
 #define is_affinity_sessions(s) (s->flags & SNAT_SESSION_FLAG_AFFINITY)
 
+/** \brief Check if exact pool address should be used.
+    @param s SNAT session
+    @return 1 if exact pool address or 0
+*/
+#define is_exact_address_session(s) (s->flags & SNAT_SESSION_FLAG_EXACT_ADDRESS)
+
 /** \brief Check if NAT interface is inside.
-    @param i NAT interfce
+    @param i NAT interface
     @return 1 if inside interface
 */
 #define nat_interface_is_inside(i) i->flags & NAT_INTERFACE_FLAG_IS_INSIDE
 
 /** \brief Check if NAT interface is outside.
-    @param i NAT interfce
+    @param i NAT interface
     @return 1 if outside interface
 */
 #define nat_interface_is_outside(i) i->flags & NAT_INTERFACE_FLAG_IS_OUTSIDE
@@ -889,6 +856,12 @@ unformat_function_t unformat_snat_protocol;
 */
 #define is_lb_static_mapping(sm) (sm->flags & NAT_STATIC_MAPPING_FLAG_LB)
 
+/** \brief Check if exact pool address should be used.
+    @param s SNAT session
+    @return 1 if exact pool address or 0
+*/
+#define is_exact_address(s) (s->flags & NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS)
+
 /** \brief Check if client initiating TCP connection (received SYN from client)
     @param t TCP header
     @return 1 if client initiating TCP connection
@@ -927,19 +900,19 @@ typedef enum nat_log_level_t_
 #undef _
 } nat_log_level_t;
 
-#define nat_elog(_level, _str)                    \
-do                                                \
-  {                                               \
-    snat_main_t *sm = &snat_main;                 \
-    if (PREDICT_FALSE (sm->log_level >= _level))  \
-      {                                           \
-        ELOG_TYPE_DECLARE (e) =                   \
-          {                                       \
-            .format = "nat-msg " _str,            \
-            .format_args = "",                    \
-          };                                      \
-        ELOG_DATA (&sm->vlib_main->elog_main, e); \
-      }                                           \
+#define nat_elog(_level, _str)                           \
+do                                                       \
+  {                                                      \
+    snat_main_t *sm = &snat_main;                        \
+    if (PREDICT_FALSE (sm->log_level >= _level))         \
+      {                                                  \
+        ELOG_TYPE_DECLARE (e) =                          \
+          {                                              \
+            .format = "nat-msg " _str,                   \
+            .format_args = "",                           \
+          };                                             \
+        ELOG_DATA (&vlib_global_main.elog_main, e);      \
+      }                                                  \
   } while (0);
 
 #define nat_elog_addr(_level, _str, _addr)               \
@@ -1047,24 +1020,24 @@ do                                                                           \
     }                                                                        \
   } while (0);
 
-#define nat_elog_X1(_level, _fmt, _arg, _val1)         \
-do                                                     \
-  {                                                    \
-    snat_main_t *sm = &snat_main;                      \
-    if (PREDICT_FALSE (sm->log_level >= _level))       \
-      {                                                \
-        ELOG_TYPE_DECLARE (e) =                        \
-          {                                            \
-            .format = "nat-msg " _fmt,                 \
-            .format_args = _arg,                       \
-          };                                           \
-        CLIB_PACKED(struct                             \
-          {                                            \
-            typeof (_val1) val1;                       \
-          }) *ed;                                      \
-        ed = ELOG_DATA (&sm->vlib_main->elog_main, e); \
-        ed->val1 = _val1;                              \
-      }                                                \
+#define nat_elog_X1(_level, _fmt, _arg, _val1)            \
+do                                                        \
+  {                                                       \
+    snat_main_t *sm = &snat_main;                         \
+    if (PREDICT_FALSE (sm->log_level >= _level))          \
+      {                                                   \
+        ELOG_TYPE_DECLARE (e) =                           \
+          {                                               \
+            .format = "nat-msg " _fmt,                    \
+            .format_args = _arg,                          \
+          };                                              \
+        CLIB_PACKED(struct                                \
+          {                                               \
+            typeof (_val1) val1;                          \
+          }) *ed;                                         \
+        ed = ELOG_DATA (&vlib_global_main.elog_main, e);  \
+        ed->val1 = _val1;                                 \
+      }                                                   \
   } while (0);
 
 #define nat_elog_notice(nat_elog_str) \
@@ -1092,48 +1065,40 @@ do                                                     \
 /* ICMP session match functions */
 u32 icmp_match_in2out_fast (snat_main_t * sm, vlib_node_runtime_t * node,
                            u32 thread_index, vlib_buffer_t * b0,
-                           ip4_header_t * ip0, u8 * p_proto,
-                           snat_session_key_t * p_value,
-                           u8 * p_dont_translate, void *d, void *e);
+                           ip4_header_t * ip0, ip4_address_t * addr,
+                           u16 * port, u32 * fib_index,
+                           nat_protocol_t * proto, void *d, void *e,
+                           u8 * dont_translate);
 u32 icmp_match_in2out_slow (snat_main_t * sm, vlib_node_runtime_t * node,
                            u32 thread_index, vlib_buffer_t * b0,
-                           ip4_header_t * ip0, u8 * p_proto,
-                           snat_session_key_t * p_value,
-                           u8 * p_dont_translate, void *d, void *e);
+                           ip4_header_t * ip0, ip4_address_t * addr,
+                           u16 * port, u32 * fib_index,
+                           nat_protocol_t * proto, void *d, void *e,
+                           u8 * dont_translate);
 u32 icmp_match_out2in_fast (snat_main_t * sm, vlib_node_runtime_t * node,
                            u32 thread_index, vlib_buffer_t * b0,
-                           ip4_header_t * ip0, u8 * p_proto,
-                           snat_session_key_t * p_value,
-                           u8 * p_dont_translate, void *d, void *e);
+                           ip4_header_t * ip0, ip4_address_t * addr,
+                           u16 * port, u32 * fib_index,
+                           nat_protocol_t * proto, void *d, void *e,
+                           u8 * dont_translate);
 u32 icmp_match_out2in_slow (snat_main_t * sm, vlib_node_runtime_t * node,
                            u32 thread_index, vlib_buffer_t * b0,
-                           ip4_header_t * ip0, u8 * p_proto,
-                           snat_session_key_t * p_value,
-                           u8 * p_dont_translate, void *d, void *e);
-
-/* ICMP deterministic NAT session match functions */
-u32 icmp_match_out2in_det (snat_main_t * sm, vlib_node_runtime_t * node,
-                          u32 thread_index, vlib_buffer_t * b0,
-                          ip4_header_t * ip0, u8 * p_proto,
-                          snat_session_key_t * p_value,
-                          u8 * p_dont_translate, void *d, void *e);
-u32 icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
-                          u32 thread_index, vlib_buffer_t * b0,
-                          ip4_header_t * ip0, u8 * p_proto,
-                          snat_session_key_t * p_value,
-                          u8 * p_dont_translate, void *d, void *e);
+                           ip4_header_t * ip0, ip4_address_t * addr,
+                           u16 * port, u32 * fib_index,
+                           nat_protocol_t * proto, void *d, void *e,
+                           u8 * dont_translate);
 
 /* ICMP endpoint-dependent session match functions */
 u32 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
                          u32 thread_index, vlib_buffer_t * b0,
-                         ip4_header_t * ip0, u8 * p_proto,
-                         snat_session_key_t * p_value,
-                         u8 * p_dont_translate, void *d, void *e);
+                         ip4_header_t * ip0, ip4_address_t * addr,
+                         u16 * port, u32 * fib_index, nat_protocol_t * proto,
+                         void *d, void *e, u8 * dont_translate);
 u32 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
                          u32 thread_index, vlib_buffer_t * b0,
-                         ip4_header_t * ip0, u8 * p_proto,
-                         snat_session_key_t * p_value,
-                         u8 * p_dont_translate, void *d, void *e);
+                         ip4_header_t * ip0, ip4_address_t * addr,
+                         u16 * port, u32 * fib_index, nat_protocol_t * proto,
+                         void *d, void *e, u8 * dont_translate);
 
 u32 icmp_in2out (snat_main_t * sm, vlib_buffer_t * b0, ip4_header_t * ip0,
                 icmp46_header_t * icmp0, u32 sw_if_index0, u32 rx_fib_index0,
@@ -1153,12 +1118,11 @@ void nat_hairpinning_sm_unknown_proto (snat_main_t * sm, vlib_buffer_t * b,
                                       ip4_header_t * ip);
 void nat44_ed_hairpinning_unknown_proto (snat_main_t * sm, vlib_buffer_t * b,
                                         ip4_header_t * ip);
-int snat_hairpinning (snat_main_t * sm, vlib_buffer_t * b0,
+int snat_hairpinning (vlib_main_t * vm, vlib_node_runtime_t * node,
+                     snat_main_t * sm, vlib_buffer_t * b0,
                      ip4_header_t * ip0, udp_header_t * udp0,
-                     tcp_header_t * tcp0, u32 proto0, int is_ed);
-void nat44_reass_hairpinning (snat_main_t * sm, vlib_buffer_t * b0,
-                             ip4_header_t * ip0, u16 sport, u16 dport,
-                             u32 proto0, int is_ed);
+                     tcp_header_t * tcp0, u32 proto0, int is_ed,
+                     int do_trace);
 
 /* Call back functions for clib_bihash_add_or_overwrite_stale */
 int nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg);
@@ -1166,9 +1130,27 @@ int nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg);
 int nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg);
 int nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg);
 
+
+/**
+ * @brief Enable NAT44 plugin
+ *
+ * @param c         nat44_config_t
+ *
+ * @return 0 on success, non-zero value otherwise
+ */
+int nat44_plugin_enable (nat44_config_t c);
+
+/**
+ * @brief Disable NAT44 plugin
+ *
+ * @return 0 on success, non-zero value otherwise
+ */
+int nat44_plugin_disable ();
+
 /**
  * @brief Add external address to NAT44 pool
  *
+ * @param sm        snat global configuration data
  * @param addr      IPv4 address
  * @param vrf_id    VRF id of tenant, ~0 means independent of VRF
  * @param twice_nat 1 if twice NAT address
@@ -1181,6 +1163,7 @@ int snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 /**
  * @brief Delete external address from NAT44 pool
  *
+ * @param sm        snat global configuration data
  * @param addr      IPv4 address
  * @param delete_sm 1 if delete static mapping using address
  * @param twice_nat 1 if twice NAT address
@@ -1216,15 +1199,18 @@ void nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add);
  * @param out2in_only  if 1 rule match only out2in direction
  * @param tag          opaque string tag
  * @param identity_nat identity NAT
+ * @param pool_addr    pool IPv4 address
+ * @param exact        1 = exact pool address
  *
  * @return 0 on success, non-zero value otherwise
  */
 int snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
                             u16 l_port, u16 e_port, u32 vrf_id,
                             int addr_only, u32 sw_if_index,
-                            snat_protocol_t proto, int is_add,
+                            nat_protocol_t proto, int is_add,
                             twice_nat_type_t twice_nat, u8 out2in_only,
-                            u8 * tag, u8 identity_nat);
+                            u8 * tag, u8 identity_nat,
+                            ip4_address_t pool_addr, int exact);
 
 /**
  * @brief Add/delete static mapping with load-balancing (multiple backends)
@@ -1242,17 +1228,17 @@ int snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
  * @return 0 on success, non-zero value otherwise
  */
 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
-                                    snat_protocol_t proto,
+                                    nat_protocol_t proto,
                                     nat44_lb_addr_port_t * locals, u8 is_add,
                                     twice_nat_type_t twice_nat,
                                     u8 out2in_only, u8 * tag, u32 affinity);
 
 int nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
                                           ip4_address_t l_addr, u16 l_port,
-                                          snat_protocol_t proto, u32 vrf_id,
+                                          nat_protocol_t proto, u32 vrf_id,
                                           u8 probability, u8 is_add);
 
-clib_error_t *snat_api_init (vlib_main_t * vm, snat_main_t * sm);
+clib_error_t *nat44_api_hookup (vlib_main_t * vm);
 
 /**
  * @brief Set NAT plugin workers
@@ -1287,11 +1273,11 @@ int snat_interface_add_del_output_feature (u32 sw_if_index, u8 is_inside,
                                           int is_del);
 
 /**
- * @brief Add/delete NAT44 pool address from specific interfce
+ * @brief Add/delete NAT44 pool address from specific interface
  *
  * @param sw_if_index software index of the interface
  * @param is_del      1 = delete, 0 = add
- * @param twice_nat   1 = twice NAT address for extenal hosts
+ * @param twice_nat   1 = twice NAT address for external hosts
  *
  * @return 0 on success, non-zero value otherwise
  */
@@ -1310,11 +1296,12 @@ int snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
  * @return 0 on success, non-zero value otherwise
  */
 int nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
-                      snat_protocol_t proto, u32 vrf_id, int is_in);
+                      nat_protocol_t proto, u32 vrf_id, int is_in);
 
 /**
  * @brief Delete NAT44 endpoint-dependent session
  *
+ * @param sm     snat global configuration data
  * @param addr   IPv4 address
  * @param port   L4 port number
  * @param proto  L4 protocol
@@ -1328,8 +1315,9 @@ int nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
                          u32 vrf_id, int is_in);
 
 /**
- * @brief Free NAT44 session data (lookup keys, external addrres port)
+ * @brief Free NAT44 session data (lookup keys, external address port)
  *
+ * @param sm           snat global configuration data
  * @param s            NAT session
  * @param thread_index thread index
  * @param is_ha        is HA event
@@ -1338,18 +1326,54 @@ void nat_free_session_data (snat_main_t * sm, snat_session_t * s,
                            u32 thread_index, u8 is_ha);
 
 /**
- * @brief Free NAT44 ED session data (lookup keys, external addrres port)
+ * @brief Set NAT44 session limit (session limit, vrf id)
  *
- * @param s            NAT session
- * @param thread_index thread index
- * @param is_ha        is HA event
+ * @param session_limit Session limit
+ * @param vrf_id        VRF id
+ * @return 0 on success, non-zero value otherwise
  */
-void
-nat44_free_session_data (snat_main_t * sm, snat_session_t * s,
-                        u32 thread_index, u8 is_ha);
+int nat44_set_session_limit (u32 session_limit, u32 vrf_id);
+
+/**
+ * @brief Update NAT44 session limit flushing all data (session limit, vrf id)
+ *
+ * @param session_limit Session limit
+ * @param vrf_id        VRF id
+ * @return 0 on success, non-zero value otherwise
+ */
+int nat44_update_session_limit (u32 session_limit, u32 vrf_id);
+
+/**
+ * @brief Initialize NAT44 data
+ *
+ * @param tsm          per thread data
+ */
+void nat44_db_init (snat_main_per_thread_data_t * tsm);
+
+/**
+ * @brief Free NAT44 data
+ *
+ * @param tsm          per thread data
+ */
+void nat44_db_free (snat_main_per_thread_data_t * tsm);
+
+/**
+ * @brief Delete specific NAT44 EI user and his sessions
+ *
+ * @param addr         IPv4 address
+ * @param fib_index    FIB table index
+ */
+int nat44_ei_user_del (ip4_address_t * addr, u32 fib_index);
+
+/**
+ * @brief Free all NAT44 sessions
+ */
+void nat44_sessions_clear ();
+
 /**
  * @brief Find or create NAT user
  *
+ * @param sm           snat global configuration data
  * @param addr         IPv4 address
  * @param fib_index    FIB table index
  * @param thread_index thread index
@@ -1363,8 +1387,10 @@ snat_user_t *nat_user_get_or_create (snat_main_t * sm,
 /**
  * @brief Allocate new NAT session or recycle last used
  *
+ * @param sm           snat global configuration data
  * @param u            NAT user
  * @param thread_index thread index
+ * @param now          time now
  *
  * @return session data structure on success otherwise zero value
  */
@@ -1372,17 +1398,6 @@ snat_session_t *nat_session_alloc_or_recycle (snat_main_t * sm,
                                              snat_user_t * u,
                                              u32 thread_index, f64 now);
 
-/**
- * @brief Allocate NAT endpoint-dependent session
- *
- * @param u            NAT user
- * @param thread_index thread index
- *
- * @return session data structure on success otherwise zero value
- */
-snat_session_t *nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u,
-                                     u32 thread_index, f64 now);
-
 /**
  * @brief Set address and port assignment algorithm for MAP-E CE
  *
@@ -1411,11 +1426,13 @@ void nat_set_alloc_addr_and_port_default (void);
  *
  * @param addresses    vector of outside addresses
  * @param thread_index thread index
- * @param k            address, port and protocol
+ * @param key          address, port and protocol
  */
-void snat_free_outside_address_and_port (snat_address_t * addresses,
-                                        u32 thread_index,
-                                        snat_session_key_t * k);
+void
+snat_free_outside_address_and_port (snat_address_t * addresses,
+                                   u32 thread_index,
+                                   ip4_address_t * addr,
+                                   u16 port, nat_protocol_t protocol);
 
 /**
  * @brief Alloc outside address and port
@@ -1423,8 +1440,7 @@ void snat_free_outside_address_and_port (snat_address_t * addresses,
  * @param addresses         vector of outside addresses
  * @param fib_index         FIB table index
  * @param thread_index      thread index
- * @param k                 allocated address and port pair
- * @param port_per_thread   number of ports per threead
+ * @param port_per_thread   number of ports per thread
  * @param snat_thread_index NAT thread index
  *
  * @return 0 on success, non-zero value otherwise
@@ -1432,33 +1448,47 @@ void snat_free_outside_address_and_port (snat_address_t * addresses,
 int snat_alloc_outside_address_and_port (snat_address_t * addresses,
                                         u32 fib_index,
                                         u32 thread_index,
-                                        snat_session_key_t * k,
+                                        nat_protocol_t proto,
+                                        ip4_address_t * addr,
+                                        u16 * port,
                                         u16 port_per_thread,
                                         u32 snat_thread_index);
 
+void expire_per_vrf_sessions (u32 fib_index);
+
 /**
  * @brief Match NAT44 static mapping.
  *
- * @param match         address and port to match
- * @param mapping       external/local address and port of the matched mapping
- * @param by_external   if 0 match by local address otherwise match by external
- *                      address
- * @param is_addr_only  1 if matched mapping is address only
- * @param twice_nat     matched mapping is twice NAT type
- * @param lb            1 if matched mapping is load-balanced
- * @param ext_host_addr external host address
+ * @param key             address and port to match
+ * @param addr            external/local address of the matched mapping
+ * @param port            port of the matched mapping
+ * @param fib_index       fib index of the matched mapping
+ * @param by_external     if 0 match by local address otherwise match by external
+ *                        address
+ * @param is_addr_only    1 if matched mapping is address only
+ * @param twice_nat       matched mapping is twice NAT type
+ * @param lb              1 if matched mapping is load-balanced
+ * @param ext_host_addr   external host address
+ * @param is_identity_nat 1 if indentity mapping
+ * @param out             if !=0 set to pointer of the mapping structure
  *
  * @returns 0 if match found otherwise 1.
  */
 int snat_static_mapping_match (snat_main_t * sm,
-                              snat_session_key_t match,
-                              snat_session_key_t * mapping,
+                              ip4_address_t match_addr,
+                              u16 match_port,
+                              u32 match_fib_index,
+                              nat_protocol_t match_protocol,
+                              ip4_address_t * mapping_addr,
+                              u16 * mapping_port,
+                              u32 * mapping_fib_index,
                               u8 by_external,
                               u8 * is_addr_only,
                               twice_nat_type_t * twice_nat,
                               lb_nat_type_t * lb,
                               ip4_address_t * ext_host_addr,
-                              u8 * is_identity_nat);
+                              u8 * is_identity_nat,
+                              snat_static_mapping_t ** out);
 
 /**
  * @brief Add/del NAT address to FIB.
@@ -1475,6 +1505,46 @@ int snat_static_mapping_match (snat_main_t * sm,
 void snat_add_del_addr_to_fib (ip4_address_t * addr,
                               u8 p_len, u32 sw_if_index, int is_add);
 
+void
+nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
+               ip4_address_t * out_addr, u16 out_port,
+               ip4_address_t * eh_addr, u16 eh_port,
+               ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
+               u32 fib_index, u16 flags, u32 thread_index);
+
+void
+nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
+               ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
+               u32 ti);
+
+void
+nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
+               ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
+               u32 total_pkts, u64 total_bytes, u32 thread_index);
+
+void
+nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
+                  ip4_address_t * out_addr, u16 out_port,
+                  ip4_address_t * eh_addr, u16 eh_port,
+                  ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
+                  u32 fib_index, u16 flags, u32 thread_index);
+
+void
+nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
+                  ip4_address_t * eh_addr, u16 eh_port, u8 proto,
+                  u32 fib_index, u32 ti);
+
+void
+nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
+                  ip4_address_t * eh_addr, u16 eh_port, u8 proto,
+                  u32 fib_index, u32 ti);
+
+void
+nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
+                  ip4_address_t * eh_addr, u16 eh_port, u8 proto,
+                  u32 fib_index, u32 total_pkts, u64 total_bytes,
+                  u32 thread_index);
+
 /*
  * Why is this here? Because we don't need to touch this layer to
  * simply reply to an icmp. We need to change id to a unique
@@ -1492,6 +1562,8 @@ typedef struct
   u16 src_port, dst_port;
 } tcp_udp_header_t;
 
+u8 *format_user_kvp (u8 * s, va_list * args);
+
 #endif /* __included_nat_h__ */
 /*
  * fd.io coding-style-patch-verification: ON