src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t nat44_ei_hairpin_dst_node;
  44 extern vlib_node_registration_t
  45   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  46 extern vlib_node_registration_t
  47   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  48
  49 #define skip_if_disabled()                                                    \
  50   do                                                                          \
  51     {                                                                         \
  52       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  53       if (PREDICT_FALSE (!nm->enabled))                                       \
  54         return;                                                               \
  55     }                                                                         \
  56   while (0)
  57
  58 #define fail_if_enabled()                                                     \
  59   do                                                                          \
  60     {                                                                         \
  61       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  62       if (PREDICT_FALSE (nm->enabled))                                        \
  63         {                                                                     \
  64           nat44_ei_log_err ("plugin enabled");                                \
  65           return 1;                                                           \
  66         }                                                                     \
  67     }                                                                         \
  68   while (0)
  69
  70 #define fail_if_disabled()                                                    \
  71   do                                                                          \
  72     {                                                                         \
  73       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  74       if (PREDICT_FALSE (!nm->enabled))                                       \
  75         {                                                                     \
  76           nat44_ei_log_err ("plugin disabled");                               \
  77           return 1;                                                           \
  78         }                                                                     \
  79     }                                                                         \
  80   while (0)
  81
  82 /* Hook up input features */
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ei-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  87                                "ip4-sv-reassembly-feature"),
  88 };
  89 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-ei-handoff-classify",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  93                                "ip4-sv-reassembly-feature"),
  94 };
  95 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  96   .arc_name = "ip4-unicast",
  97   .node_name = "nat44-ei-in2out",
  98   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  99                                "ip4-sv-reassembly-feature"),
 100 };
 101 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
 102   .arc_name = "ip4-unicast",
 103   .node_name = "nat44-ei-out2in",
 104   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 105                                "ip4-sv-reassembly-feature",
 106                                "ip4-dhcp-client-detect"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 109   .arc_name = "ip4-output",
 110   .node_name = "nat44-ei-in2out-output",
 111   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 112                                "ip4-sv-reassembly-output-feature"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_fast, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ei-in2out-fast",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 118                                "ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_fast, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ei-out2in-fast",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 124                                "ip4-sv-reassembly-feature",
 125                                "ip4-dhcp-client-detect"),
 126 };
 127 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_dst, static) = {
 128   .arc_name = "ip4-unicast",
 129   .node_name = "nat44-ei-hairpin-dst",
 130   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 131                                "ip4-sv-reassembly-feature"),
 132 };
 133 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_src, static) = {
 134   .arc_name = "ip4-output",
 135   .node_name = "nat44-ei-hairpin-src",
 136   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 137                                "ip4-sv-reassembly-output-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 140   .arc_name = "ip4-local",
 141   .node_name = "nat44-ei-hairpinning",
 142   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 143 };
 144 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 145   .arc_name = "ip4-unicast",
 146   .node_name = "nat44-ei-in2out-worker-handoff",
 147   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 148 };
 149 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 150   .arc_name = "ip4-unicast",
 151   .node_name = "nat44-ei-out2in-worker-handoff",
 152   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 153                                "ip4-dhcp-client-detect"),
 154 };
 155 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 156   .arc_name = "ip4-output",
 157   .node_name = "nat44-ei-in2out-output-worker-handoff",
 158   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 159                                "ip4-sv-reassembly-output-feature"),
 160 };
 161
 162 VLIB_PLUGIN_REGISTER () = {
 163   .version = VPP_BUILD_VER,
 164   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 165 };
 166
 167 #define foreach_nat44_ei_classify_error                                       \
 168   _ (NEXT_IN2OUT, "next in2out")                                              \
 169   _ (NEXT_OUT2IN, "next out2in")                                              \
 170   _ (FRAG_CACHED, "fragment cached")
 171
 172 typedef enum
 173 {
 174 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 175   foreach_nat44_ei_classify_error
 176 #undef _
 177     NAT44_EI_CLASSIFY_N_ERROR,
 178 } nat44_ei_classify_error_t;
 179
 180 static char *nat44_ei_classify_error_strings[] = {
 181 #define _(sym, string) string,
 182   foreach_nat44_ei_classify_error
 183 #undef _
 184 };
 185
 186 typedef enum
 187 {
 188   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 189   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 190   NAT44_EI_CLASSIFY_NEXT_DROP,
 191   NAT44_EI_CLASSIFY_N_NEXT,
 192 } nat44_ei_classify_next_t;
 193
 194 typedef struct
 195 {
 196   u8 next_in2out;
 197   u8 cached;
 198 } nat44_ei_classify_trace_t;
 199
 200 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 201                                    u32 sw_if_index, int is_add);
 202
 203 static u8 *
 204 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 205 {
 206   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 207   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 208   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 209   char *next;
 210
 211   if (t->cached)
 212     s = format (s, "nat44-ei-classify: fragment cached");
 213   else
 214     {
 215       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 216       s = format (s, "nat44-ei-classify: next %s", next);
 217     }
 218
 219   return s;
 220 }
 221
 222 static void nat44_ei_db_free ();
 223
 224 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 225                               u32 user_buckets);
 226
 227 static void nat44_ei_ip4_add_del_interface_address_cb (
 228   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 229   u32 address_length, u32 if_address_index, u32 is_delete);
 230
 231 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 232   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 233   u32 address_length, u32 if_address_index, u32 is_delete);
 234
 235 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 236                                          u32 sw_if_index, u32 new_fib_index,
 237                                          u32 old_fib_index);
 238
 239 void
 240 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 241 {
 242   vlib_node_t *node;
 243   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 244   nm->out2in_node_index = node->index;
 245   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 246   nm->in2out_node_index = node->index;
 247   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 248   nm->in2out_output_node_index = node->index;
 249 }
 250
 251 int
 252 nat44_ei_set_workers (uword *bitmap)
 253 {
 254   nat44_ei_main_t *nm = &nat44_ei_main;
 255   int i, j = 0;
 256
 257   if (nm->num_workers < 2)
 258     return VNET_API_ERROR_FEATURE_DISABLED;
 259
 260   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 261     return VNET_API_ERROR_INVALID_WORKER;
 262
 263   vec_free (nm->workers);
 264   clib_bitmap_foreach (i, bitmap)
 265     {
 266       vec_add1 (nm->workers, i);
 267       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 268       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 269       j++;
 270     }
 271
 272   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 273
 274   return 0;
 275 }
 276
 277 #define nat_validate_simple_counter(c, i)                                     \
 278   do                                                                          \
 279     {                                                                         \
 280       vlib_validate_simple_counter (&c, i);                                   \
 281       vlib_zero_simple_counter (&c, i);                                       \
 282     }                                                                         \
 283   while (0);
 284
 285 #define nat_init_simple_counter(c, n, sn)                                     \
 286   do                                                                          \
 287     {                                                                         \
 288       c.name = n;                                                             \
 289       c.stat_segment_name = sn;                                               \
 290       nat_validate_simple_counter (c, 0);                                     \
 291     }                                                                         \
 292   while (0);
 293
 294 static_always_inline void
 295 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 296 {
 297 #define _(x)                                                                  \
 298   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 299   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 300   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 301   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 302   foreach_nat_counter;
 303 #undef _
 304   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 305 }
 306
 307 static void
 308 nat44_ei_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 309 {
 310   nat44_ei_main_t *nm = &nat44_ei_main;
 311   nat44_ei_interface_t *i;
 312
 313   pool_foreach (i, nm->interfaces)
 314     {
 315       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 316         {
 317           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 318         }
 319     }
 320   pool_foreach (i, nm->output_feature_interfaces)
 321     {
 322       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 323         {
 324           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 325         }
 326     }
 327 }
 328
 329 static_always_inline void
 330 nat44_ei_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 331 {
 332   nat44_ei_main_t *nm = &nat44_ei_main;
 333   nat44_ei_address_t *ap;
 334
 335   vec_foreach (ap, nm->addresses)
 336     {
 337       nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 338     }
 339 }
 340
 341 static_always_inline void
 342 nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 343 {
 344   nat44_ei_main_t *nm = &nat44_ei_main;
 345   nat44_ei_static_mapping_t *m;
 346
 347   pool_foreach (m, nm->static_mappings)
 348     {
 349       if (is_sm_addr_only (m->flags) &&
 350           !(m->local_addr.as_u32 == m->external_addr.as_u32))
 351         {
 352           nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 353                                         is_add);
 354         }
 355     }
 356 }
 357
 358 static int
 359 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
 360 {
 361   nat44_ei_main_t *nm = &nat44_ei_main;
 362   nat44_ei_static_mapping_t *m;
 363   pool_foreach (m, nm->static_mappings)
 364     {
 365       if (is_sm_addr_only (m->flags) || is_sm_identity_nat (m->flags))
 366         {
 367           continue;
 368         }
 369       if (m->external_addr.as_u32 == addr.as_u32)
 370         {
 371           return 1;
 372         }
 373     }
 374   return 0;
 375 }
 376
 377 clib_error_t *
 378 nat44_ei_init (vlib_main_t *vm)
 379 {
 380   nat44_ei_main_t *nm = &nat44_ei_main;
 381   vlib_thread_main_t *tm = vlib_get_thread_main ();
 382   vlib_thread_registration_t *tr;
 383   ip4_add_del_interface_address_callback_t cbi = { 0 };
 384   ip4_table_bind_callback_t cbt = { 0 };
 385   u32 i, num_threads = 0;
 386   uword *p, *bitmap = 0;
 387
 388   clib_memset (nm, 0, sizeof (*nm));
 389
 390   // required
 391   nm->vnet_main = vnet_get_main ();
 392   // convenience
 393   nm->ip4_main = &ip4_main;
 394   nm->api_main = vlibapi_get_main ();
 395   nm->ip4_lookup_main = &ip4_main.lookup_main;
 396
 397   // handoff stuff
 398   nm->fq_out2in_index = ~0;
 399   nm->fq_in2out_index = ~0;
 400   nm->fq_in2out_output_index = ~0;
 401
 402   nm->log_level = NAT_LOG_ERROR;
 403
 404   nat44_ei_set_node_indexes (nm, vm);
 405   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 406
 407   nat_init_simple_counter (nm->total_users, "total-users",
 408                            "/nat44-ei/total-users");
 409   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 410                            "/nat44-ei/total-sessions");
 411   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 412                            "/nat44-ei/user-limit-reached");
 413
 414 #define _(x)                                                                  \
 415   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 416                            "/nat44-ei/in2out/fastpath/" #x);                  \
 417   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 418                            "/nat44-ei/out2in/fastpath/" #x);                  \
 419   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 420                            "/nat44-ei/in2out/slowpath/" #x);                  \
 421   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 422                            "/nat44-ei/out2in/slowpath/" #x);
 423   foreach_nat_counter;
 424 #undef _
 425   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 426                            "/nat44-ei/hairpinning");
 427
 428   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 429   if (p)
 430     {
 431       tr = (vlib_thread_registration_t *) p[0];
 432       if (tr)
 433         {
 434           nm->num_workers = tr->count;
 435           nm->first_worker_index = tr->first_index;
 436         }
 437     }
 438   num_threads = tm->n_vlib_mains - 1;
 439   nm->port_per_thread = 0xffff - 1024;
 440   vec_validate (nm->per_thread_data, num_threads);
 441
 442   /* Use all available workers by default */
 443   if (nm->num_workers > 1)
 444     {
 445       for (i = 0; i < nm->num_workers; i++)
 446         bitmap = clib_bitmap_set (bitmap, i, 1);
 447       nat44_ei_set_workers (bitmap);
 448       clib_bitmap_free (bitmap);
 449     }
 450   else
 451     {
 452       nm->per_thread_data[0].snat_thread_index = 0;
 453     }
 454
 455   /* callbacks to call when interface address changes. */
 456   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 457   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 458   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 459   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 460
 461   /* callbacks to call when interface to table biding changes */
 462   cbt.function = nat44_ei_update_outside_fib;
 463   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 464
 465   nm->fib_src_low = fib_source_allocate (
 466     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 467   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 468                                         FIB_SOURCE_BH_SIMPLE);
 469
 470   // used only by out2in-dpo feature
 471   nat_dpo_module_init ();
 472   nat_ha_init (vm, nm->num_workers, num_threads);
 473
 474   nm->hairpinning_fq_index =
 475     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 476   nm->hairpin_dst_fq_index =
 477     vlib_frame_queue_main_init (nat44_ei_hairpin_dst_node.index, 0);
 478   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 479     vlib_frame_queue_main_init (
 480       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 481   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 482     vlib_frame_queue_main_init (
 483       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 484   return nat44_ei_api_hookup (vm);
 485 }
 486
 487 VLIB_INIT_FUNCTION (nat44_ei_init);
 488
 489 int
 490 nat44_ei_plugin_enable (nat44_ei_config_t c)
 491 {
 492   nat44_ei_main_t *nm = &nat44_ei_main;
 493
 494   fail_if_enabled ();
 495
 496   if (!c.users)
 497     c.users = 1024;
 498
 499   if (!c.sessions)
 500     c.sessions = 10 * 1024;
 501
 502   if (!c.user_sessions)
 503     c.user_sessions = c.sessions;
 504
 505   nm->rconfig = c;
 506
 507   if (!nm->frame_queue_nelts)
 508     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 509
 510   nm->translations = c.sessions;
 511   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 512   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 513
 514   nm->pat = (!c.static_mapping_only ||
 515              (c.static_mapping_only && c.connection_tracking));
 516
 517   nm->static_mapping_only = c.static_mapping_only;
 518   nm->static_mapping_connection_tracking = c.connection_tracking;
 519   nm->out2in_dpo = c.out2in_dpo;
 520   nm->forwarding_enabled = 0;
 521   nm->mss_clamping = 0;
 522
 523   nm->max_users_per_thread = c.users;
 524   nm->max_translations_per_thread = c.sessions;
 525   nm->max_translations_per_user = c.user_sessions;
 526
 527   nm->inside_vrf_id = c.inside_vrf;
 528   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 529     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 530
 531   nm->outside_vrf_id = c.outside_vrf;
 532   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 533     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 534
 535   nat_reset_timeouts (&nm->timeouts);
 536   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 537                     nm->user_buckets);
 538   nat44_ei_set_alloc_default ();
 539
 540   // TODO: zero simple counter for all counters missing
 541
 542   vlib_zero_simple_counter (&nm->total_users, 0);
 543   vlib_zero_simple_counter (&nm->total_sessions, 0);
 544   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 545
 546   if (nm->num_workers > 1)
 547     {
 548       if (nm->fq_in2out_index == ~0)
 549         {
 550           nm->fq_in2out_index = vlib_frame_queue_main_init (
 551             nm->in2out_node_index, nm->frame_queue_nelts);
 552         }
 553       if (nm->fq_out2in_index == ~0)
 554         {
 555           nm->fq_out2in_index = vlib_frame_queue_main_init (
 556             nm->out2in_node_index, nm->frame_queue_nelts);
 557         }
 558       if (nm->fq_in2out_output_index == ~0)
 559         {
 560           nm->fq_in2out_output_index = vlib_frame_queue_main_init (
 561             nm->in2out_output_node_index, nm->frame_queue_nelts);
 562         }
 563     }
 564
 565   nat_ha_enable ();
 566   nm->enabled = 1;
 567
 568   return 0;
 569 }
 570
 571 void
 572 nat44_ei_addresses_free (nat44_ei_address_t **addresses)
 573 {
 574   nat44_ei_address_t *ap;
 575   vec_foreach (ap, *addresses)
 576     {
 577 #define _(N, i, n, s) vec_free (ap->busy_##n##_ports_per_thread);
 578       foreach_nat_protocol
 579 #undef _
 580     }
 581   vec_free (*addresses);
 582   *addresses = 0;
 583 }
 584
 585 static_always_inline nat44_ei_outside_fib_t *
 586 nat44_ei_get_outside_fib (nat44_ei_outside_fib_t *outside_fibs, u32 fib_index)
 587 {
 588   nat44_ei_outside_fib_t *f;
 589   vec_foreach (f, outside_fibs)
 590     {
 591       if (f->fib_index == fib_index)
 592         {
 593           return f;
 594         }
 595     }
 596   return 0;
 597 }
 598
 599 static_always_inline nat44_ei_interface_t *
 600 nat44_ei_get_interface (nat44_ei_interface_t *interfaces, u32 sw_if_index)
 601 {
 602   nat44_ei_interface_t *i;
 603   pool_foreach (i, interfaces)
 604     {
 605       if (i->sw_if_index == sw_if_index)
 606         {
 607           return i;
 608         }
 609     }
 610   return 0;
 611 }
 612
 613 int
 614 nat44_ei_add_interface (u32 sw_if_index, u8 is_inside)
 615 {
 616   const char *feature_name, *del_feature_name;
 617   nat44_ei_main_t *nm = &nat44_ei_main;
 618
 619   nat44_ei_outside_fib_t *outside_fib;
 620   nat44_ei_interface_t *i;
 621   u32 fib_index;
 622   int rv;
 623
 624   fail_if_disabled ();
 625
 626   if (nm->out2in_dpo && !is_inside)
 627     {
 628       nat44_ei_log_err ("error unsupported");
 629       return VNET_API_ERROR_UNSUPPORTED;
 630     }
 631
 632   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 633     {
 634       nat44_ei_log_err ("error interface already configured");
 635       return VNET_API_ERROR_VALUE_EXIST;
 636     }
 637
 638   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 639   if (i)
 640     {
 641       if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 642           (nat44_ei_interface_is_outside (i) && !is_inside))
 643         {
 644           return 0;
 645         }
 646       if (nm->num_workers > 1)
 647         {
 648           del_feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 649                                           "nat44-ei-out2in-worker-handoff";
 650           feature_name = "nat44-ei-handoff-classify";
 651         }
 652       else
 653         {
 654           del_feature_name =
 655             !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 656
 657           feature_name = "nat44-ei-classify";
 658         }
 659
 660       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 661       if (rv)
 662         {
 663           return rv;
 664         }
 665       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 666                                         sw_if_index, 0, 0, 0);
 667       if (rv)
 668         {
 669           return rv;
 670         }
 671       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 672                                         sw_if_index, 1, 0, 0);
 673       if (rv)
 674         {
 675           return rv;
 676         }
 677       if (!is_inside)
 678         {
 679           rv = vnet_feature_enable_disable (
 680             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 0, 0, 0);
 681           if (rv)
 682             {
 683               return rv;
 684             }
 685         }
 686     }
 687   else
 688     {
 689       if (nm->num_workers > 1)
 690         {
 691           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 692                                      "nat44-ei-out2in-worker-handoff";
 693         }
 694       else
 695         {
 696           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 697         }
 698       nat_validate_interface_counters (nm, sw_if_index);
 699
 700       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 701       if (rv)
 702         {
 703           return rv;
 704         }
 705       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 706                                         sw_if_index, 1, 0, 0);
 707       if (rv)
 708         {
 709           return rv;
 710         }
 711       if (is_inside && !nm->out2in_dpo)
 712         {
 713           rv = vnet_feature_enable_disable (
 714             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 1, 0, 0);
 715           if (rv)
 716             {
 717               return rv;
 718             }
 719         }
 720
 721       pool_get (nm->interfaces, i);
 722       i->sw_if_index = sw_if_index;
 723       i->flags = 0;
 724     }
 725
 726   fib_index =
 727     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 728
 729   if (!is_inside)
 730     {
 731       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 732
 733       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 734       if (outside_fib)
 735         {
 736           outside_fib->refcount++;
 737         }
 738       else
 739         {
 740           vec_add2 (nm->outside_fibs, outside_fib, 1);
 741           outside_fib->fib_index = fib_index;
 742           outside_fib->refcount = 1;
 743         }
 744
 745       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 746       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 747     }
 748   else
 749     {
 750       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 751     }
 752
 753   return 0;
 754 }
 755
 756 int
 757 nat44_ei_del_interface (u32 sw_if_index, u8 is_inside)
 758 {
 759   const char *feature_name, *del_feature_name;
 760   nat44_ei_main_t *nm = &nat44_ei_main;
 761
 762   nat44_ei_outside_fib_t *outside_fib;
 763   nat44_ei_interface_t *i;
 764   u32 fib_index;
 765   int rv;
 766
 767   fail_if_disabled ();
 768
 769   if (nm->out2in_dpo && !is_inside)
 770     {
 771       nat44_ei_log_err ("error unsupported");
 772       return VNET_API_ERROR_UNSUPPORTED;
 773     }
 774
 775   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 776   if (i == 0)
 777     {
 778       nat44_ei_log_err ("error interface couldn't be found");
 779       return VNET_API_ERROR_NO_SUCH_ENTRY;
 780     }
 781
 782   if (nat44_ei_interface_is_inside (i) && nat44_ei_interface_is_outside (i))
 783     {
 784       if (nm->num_workers > 1)
 785         {
 786           del_feature_name = "nat44-ei-handoff-classify";
 787           feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 788                                       "nat44-ei-out2in-worker-handoff";
 789         }
 790       else
 791         {
 792           del_feature_name = "nat44-ei-classify";
 793           feature_name = !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 794         }
 795
 796       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 797       if (rv)
 798         {
 799           return rv;
 800         }
 801       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 802                                         sw_if_index, 0, 0, 0);
 803       if (rv)
 804         {
 805           return rv;
 806         }
 807       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 808                                         sw_if_index, 1, 0, 0);
 809       if (rv)
 810         {
 811           return rv;
 812         }
 813       if (is_inside)
 814         {
 815           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 816         }
 817       else
 818         {
 819           rv = vnet_feature_enable_disable (
 820             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 1, 0, 0);
 821           if (rv)
 822             {
 823               return rv;
 824             }
 825           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 826         }
 827     }
 828   else
 829     {
 830       if (nm->num_workers > 1)
 831         {
 832           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 833                                      "nat44-ei-out2in-worker-handoff";
 834         }
 835       else
 836         {
 837           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 838         }
 839
 840       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 841       if (rv)
 842         {
 843           return rv;
 844         }
 845       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 846                                         sw_if_index, 0, 0, 0);
 847       if (rv)
 848         {
 849           return rv;
 850         }
 851       if (is_inside)
 852         {
 853           rv = vnet_feature_enable_disable (
 854             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 0, 0, 0);
 855           if (rv)
 856             {
 857               return rv;
 858             }
 859         }
 860
 861       // remove interface
 862       pool_put (nm->interfaces, i);
 863     }
 864
 865   if (!is_inside)
 866     {
 867       fib_index =
 868         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 869       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 870       if (outside_fib)
 871         {
 872           outside_fib->refcount--;
 873           if (!outside_fib->refcount)
 874             {
 875               vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
 876             }
 877         }
 878
 879       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
 880       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
 881     }
 882
 883   return 0;
 884 }
 885
 886 int
 887 nat44_ei_add_output_interface (u32 sw_if_index)
 888 {
 889   nat44_ei_main_t *nm = &nat44_ei_main;
 890
 891   nat44_ei_outside_fib_t *outside_fib;
 892   nat44_ei_interface_t *i;
 893   u32 fib_index;
 894   int rv;
 895
 896   fail_if_disabled ();
 897
 898   if (nat44_ei_get_interface (nm->interfaces, sw_if_index))
 899     {
 900       nat44_ei_log_err ("error interface already configured");
 901       return VNET_API_ERROR_VALUE_EXIST;
 902     }
 903
 904   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 905     {
 906       nat44_ei_log_err ("error interface already configured");
 907       return VNET_API_ERROR_VALUE_EXIST;
 908     }
 909
 910   if (nm->num_workers > 1)
 911     {
 912       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 913       if (rv)
 914         {
 915           return rv;
 916         }
 917       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 918       if (rv)
 919         {
 920           return rv;
 921         }
 922       rv = vnet_feature_enable_disable (
 923         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 1, 0, 0);
 924       if (rv)
 925         {
 926           return rv;
 927         }
 928       rv = vnet_feature_enable_disable (
 929         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 1,
 930         0, 0);
 931       if (rv)
 932         {
 933           return rv;
 934         }
 935     }
 936   else
 937     {
 938       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 939       if (rv)
 940         {
 941           return rv;
 942         }
 943       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 944       if (rv)
 945         {
 946           return rv;
 947         }
 948       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 949                                         sw_if_index, 1, 0, 0);
 950       if (rv)
 951         {
 952           return rv;
 953         }
 954       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 955                                         sw_if_index, 1, 0, 0);
 956       if (rv)
 957         {
 958           return rv;
 959         }
 960     }
 961
 962   nat_validate_interface_counters (nm, sw_if_index);
 963
 964   pool_get (nm->output_feature_interfaces, i);
 965   i->sw_if_index = sw_if_index;
 966   i->flags = 0;
 967   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 968   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 969
 970   fib_index =
 971     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 972   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 973   if (outside_fib)
 974     {
 975       outside_fib->refcount++;
 976     }
 977   else
 978     {
 979       vec_add2 (nm->outside_fibs, outside_fib, 1);
 980       outside_fib->fib_index = fib_index;
 981       outside_fib->refcount = 1;
 982     }
 983
 984   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 985   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 986
 987   return 0;
 988 }
 989
 990 int
 991 nat44_ei_del_output_interface (u32 sw_if_index)
 992 {
 993   nat44_ei_main_t *nm = &nat44_ei_main;
 994
 995   nat44_ei_outside_fib_t *outside_fib;
 996   nat44_ei_interface_t *i;
 997   u32 fib_index;
 998   int rv;
 999
1000   fail_if_disabled ();
1001
1002   i = nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index);
1003   if (!i)
1004     {
1005       nat44_ei_log_err ("error interface couldn't be found");
1006       return VNET_API_ERROR_NO_SUCH_ENTRY;
1007     }
1008
1009   if (nm->num_workers > 1)
1010     {
1011       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1012       if (rv)
1013         {
1014           return rv;
1015         }
1016       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1017       if (rv)
1018         {
1019           return rv;
1020         }
1021       rv = vnet_feature_enable_disable (
1022         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 0, 0, 0);
1023       if (rv)
1024         {
1025           return rv;
1026         }
1027       rv = vnet_feature_enable_disable (
1028         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 0,
1029         0, 0);
1030       if (rv)
1031         {
1032           return rv;
1033         }
1034     }
1035   else
1036     {
1037       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1038       if (rv)
1039         {
1040           return rv;
1041         }
1042       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1043       if (rv)
1044         {
1045           return rv;
1046         }
1047       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1048                                         sw_if_index, 0, 0, 0);
1049       if (rv)
1050         {
1051           return rv;
1052         }
1053       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1054                                         sw_if_index, 0, 0, 0);
1055       if (rv)
1056         {
1057           return rv;
1058         }
1059     }
1060
1061   pool_put (nm->output_feature_interfaces, i);
1062
1063   fib_index =
1064     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1065   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1066   if (outside_fib)
1067     {
1068       outside_fib->refcount--;
1069       if (!outside_fib->refcount)
1070         {
1071           vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
1072         }
1073     }
1074
1075   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1076   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1077
1078   return 0;
1079 }
1080
1081 int
1082 nat44_ei_plugin_disable ()
1083 {
1084   nat44_ei_main_t *nm = &nat44_ei_main;
1085   nat44_ei_interface_t *i, *pool;
1086   int error = 0;
1087
1088   // first unregister all nodes from interfaces
1089   pool = pool_dup (nm->interfaces);
1090   pool_foreach (i, pool)
1091     {
1092       if (nat44_ei_interface_is_inside (i))
1093         {
1094           error = nat44_ei_del_interface (i->sw_if_index, 1);
1095         }
1096       if (nat44_ei_interface_is_outside (i))
1097         {
1098           error = nat44_ei_del_interface (i->sw_if_index, 0);
1099         }
1100
1101       if (error)
1102         {
1103           nat44_ei_log_err ("error occurred while removing interface %u",
1104                             i->sw_if_index);
1105         }
1106     }
1107   pool_free (pool);
1108   pool_free (nm->interfaces);
1109
1110   pool = pool_dup (nm->output_feature_interfaces);
1111   pool_foreach (i, pool)
1112     {
1113       error = nat44_ei_del_output_interface (i->sw_if_index);
1114       if (error)
1115         {
1116           nat44_ei_log_err ("error occurred while removing interface %u",
1117                             i->sw_if_index);
1118         }
1119     }
1120   pool_free (pool);
1121   pool_free (nm->output_feature_interfaces);
1122
1123   nat_ha_disable ();
1124   nat44_ei_db_free ();
1125
1126   nat44_ei_addresses_free (&nm->addresses);
1127
1128   vec_free (nm->to_resolve);
1129   vec_free (nm->auto_add_sw_if_indices);
1130
1131   nm->to_resolve = 0;
1132   nm->auto_add_sw_if_indices = 0;
1133
1134   nm->forwarding_enabled = 0;
1135
1136   nm->enabled = 0;
1137   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
1138
1139   return 0;
1140 }
1141
1142 int
1143 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
1144                                        u32 thread_index, ip4_address_t addr,
1145                                        u16 port, nat_protocol_t protocol)
1146 {
1147   nat44_ei_main_t *nm = &nat44_ei_main;
1148   nat44_ei_address_t *a = 0;
1149   u32 address_index;
1150   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1151
1152   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1153     {
1154       if (addresses[address_index].addr.as_u32 != addr.as_u32)
1155         continue;
1156
1157       a = addresses + address_index;
1158       switch (protocol)
1159         {
1160 #define _(N, j, n, s)                                                         \
1161   case NAT_PROTOCOL_##N:                                                      \
1162     if (a->busy_##n##_port_refcounts[port_host_byte_order])                   \
1163       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
1164     ++a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1165     a->busy_##n##_ports_per_thread[thread_index]++;                           \
1166     a->busy_##n##_ports++;                                                    \
1167     return 0;
1168           foreach_nat_protocol
1169 #undef _
1170             default : nat_elog_info (nm, "unknown protocol");
1171           return 1;
1172         }
1173     }
1174
1175   return VNET_API_ERROR_NO_SUCH_ENTRY;
1176 }
1177
1178 void
1179 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
1180 {
1181   nat44_ei_main_t *nm = &nat44_ei_main;
1182   dpo_id_t dpo_v4 = DPO_INVALID;
1183   fib_prefix_t pfx = {
1184     .fp_proto = FIB_PROTOCOL_IP4,
1185     .fp_len = 32,
1186     .fp_addr.ip4.as_u32 = addr.as_u32,
1187   };
1188
1189   if (is_add)
1190     {
1191       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
1192       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
1193                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
1194       dpo_reset (&dpo_v4);
1195     }
1196   else
1197     {
1198       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
1199     }
1200 }
1201
1202 void
1203 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1204                                         u32 thread_index, ip4_address_t *addr,
1205                                         u16 port, nat_protocol_t protocol)
1206 {
1207   nat44_ei_main_t *nm = &nat44_ei_main;
1208   nat44_ei_address_t *a;
1209   u32 address_index;
1210   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1211
1212   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1213     {
1214       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1215         break;
1216     }
1217
1218   ASSERT (address_index < vec_len (addresses));
1219
1220   a = addresses + address_index;
1221
1222   switch (protocol)
1223     {
1224 #define _(N, i, n, s)                                                         \
1225   case NAT_PROTOCOL_##N:                                                      \
1226     ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1);         \
1227     --a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1228     a->busy_##n##_ports--;                                                    \
1229     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1230     break;
1231       foreach_nat_protocol
1232 #undef _
1233         default : nat_elog_info (nm, "unknown protocol");
1234       return;
1235     }
1236 }
1237
1238 void
1239 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1240                                u32 thread_index, u8 is_ha)
1241 {
1242   clib_bihash_kv_8_8_t kv;
1243
1244   /* session lookup tables */
1245   init_nat_i2o_k (&kv, s);
1246   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1247     nat_elog_warn (nm, "in2out key del failed");
1248   init_nat_o2i_k (&kv, s);
1249   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1250     nat_elog_warn (nm, "out2in key del failed");
1251
1252   if (!is_ha)
1253     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1254                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1255                              s->out2in.port, s->nat_proto);
1256
1257   if (nat44_ei_is_unk_proto_session (s))
1258     return;
1259
1260   if (!is_ha)
1261     {
1262       /* log NAT event */
1263       nat_ipfix_logging_nat44_ses_delete (
1264         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1265         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1266         s->in2out.fib_index);
1267
1268       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1269                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1270                    thread_index);
1271     }
1272
1273   if (nat44_ei_is_session_static (s))
1274     return;
1275
1276   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1277                                           &s->out2in.addr, s->out2in.port,
1278                                           s->nat_proto);
1279 }
1280
1281 nat44_ei_user_t *
1282 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1283                              u32 fib_index, u32 thread_index)
1284 {
1285   nat44_ei_user_t *u = 0;
1286   nat44_ei_user_key_t user_key;
1287   clib_bihash_kv_8_8_t kv, value;
1288   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1289   dlist_elt_t *per_user_list_head_elt;
1290
1291   user_key.addr.as_u32 = addr->as_u32;
1292   user_key.fib_index = fib_index;
1293   kv.key = user_key.as_u64;
1294
1295   /* Ever heard of the "user" = src ip4 address before? */
1296   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1297     {
1298       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1299         {
1300           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1301                                          0, 1);
1302           nat_elog_warn (nm, "maximum user limit reached");
1303           return NULL;
1304         }
1305       /* no, make a new one */
1306       pool_get (tnm->users, u);
1307       clib_memset (u, 0, sizeof (*u));
1308
1309       u->addr.as_u32 = addr->as_u32;
1310       u->fib_index = fib_index;
1311
1312       pool_get (tnm->list_pool, per_user_list_head_elt);
1313
1314       u->sessions_per_user_list_head_index =
1315         per_user_list_head_elt - tnm->list_pool;
1316
1317       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1318
1319       kv.value = u - tnm->users;
1320
1321       /* add user */
1322       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1323         {
1324           nat_elog_warn (nm, "user_hash key add failed");
1325           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1326           return NULL;
1327         }
1328
1329       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1330                                pool_elts (tnm->users));
1331     }
1332   else
1333     {
1334       u = pool_elt_at_index (tnm->users, value.value);
1335     }
1336
1337   return u;
1338 }
1339
1340 nat44_ei_session_t *
1341 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1342                                    u32 thread_index, f64 now)
1343 {
1344   nat44_ei_session_t *s;
1345   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1346   u32 oldest_per_user_translation_list_index, session_index;
1347   dlist_elt_t *oldest_per_user_translation_list_elt;
1348   dlist_elt_t *per_user_translation_list_elt;
1349
1350   /* Over quota? Recycle the least recently used translation */
1351   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1352     {
1353       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1354         tnm->list_pool, u->sessions_per_user_list_head_index);
1355
1356       ASSERT (oldest_per_user_translation_list_index != ~0);
1357
1358       /* Add it back to the end of the LRU list */
1359       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1360                           oldest_per_user_translation_list_index);
1361       /* Get the list element */
1362       oldest_per_user_translation_list_elt = pool_elt_at_index (
1363         tnm->list_pool, oldest_per_user_translation_list_index);
1364
1365       /* Get the session index from the list element */
1366       session_index = oldest_per_user_translation_list_elt->value;
1367
1368       /* Get the session */
1369       s = pool_elt_at_index (tnm->sessions, session_index);
1370
1371       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1372       if (nat44_ei_is_session_static (s))
1373         u->nstaticsessions--;
1374       else
1375         u->nsessions--;
1376       s->flags = 0;
1377       s->total_bytes = 0;
1378       s->total_pkts = 0;
1379       s->state = 0;
1380       s->ext_host_addr.as_u32 = 0;
1381       s->ext_host_port = 0;
1382       s->ext_host_nat_addr.as_u32 = 0;
1383       s->ext_host_nat_port = 0;
1384     }
1385   else
1386     {
1387       pool_get (tnm->sessions, s);
1388       clib_memset (s, 0, sizeof (*s));
1389
1390       /* Create list elts */
1391       pool_get (tnm->list_pool, per_user_translation_list_elt);
1392       clib_dlist_init (tnm->list_pool,
1393                        per_user_translation_list_elt - tnm->list_pool);
1394
1395       per_user_translation_list_elt->value = s - tnm->sessions;
1396       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1397       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1398
1399       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1400                           per_user_translation_list_elt - tnm->list_pool);
1401
1402       s->user_index = u - tnm->users;
1403       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1404                                pool_elts (tnm->sessions));
1405     }
1406
1407   s->ha_last_refreshed = now;
1408
1409   return s;
1410 }
1411
1412 void
1413 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1414                             u32 thread_index, u8 is_ha)
1415 {
1416   clib_bihash_kv_8_8_t kv;
1417
1418   init_nat_i2o_k (&kv, s);
1419   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1420     nat_elog_warn (nm, "in2out key del failed");
1421
1422   init_nat_o2i_k (&kv, s);
1423   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1424     nat_elog_warn (nm, "out2in key del failed");
1425
1426   if (!is_ha)
1427     {
1428       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1429                                &s->in2out.addr, s->in2out.port,
1430                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1431
1432       nat_ipfix_logging_nat44_ses_delete (
1433         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1434         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1435         s->in2out.fib_index);
1436
1437       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1438                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1439                    thread_index);
1440     }
1441
1442   if (nat44_ei_is_session_static (s))
1443     return;
1444
1445   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1446                                           &s->out2in.addr, s->out2in.port,
1447                                           s->nat_proto);
1448 }
1449
1450 static_always_inline void
1451 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1452 {
1453   dlist_elt_t *elt;
1454   nat44_ei_session_t *s;
1455
1456   nat44_ei_main_t *nm = &nat44_ei_main;
1457   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1458
1459   // get head
1460   elt =
1461     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1462   // get first element
1463   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1464
1465   while (elt->value != ~0)
1466     {
1467       s = pool_elt_at_index (tnm->sessions, elt->value);
1468       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1469
1470       nat44_ei_free_session_data (nm, s, thread_index, 0);
1471       nat44_ei_delete_session (nm, s, thread_index);
1472     }
1473 }
1474
1475 int
1476 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1477 {
1478   int rv = 1;
1479
1480   nat44_ei_main_t *nm = &nat44_ei_main;
1481   nat44_ei_main_per_thread_data_t *tnm;
1482
1483   nat44_ei_user_key_t user_key;
1484   clib_bihash_kv_8_8_t kv, value;
1485
1486   user_key.addr.as_u32 = addr->as_u32;
1487   user_key.fib_index = fib_index;
1488   kv.key = user_key.as_u64;
1489
1490   if (nm->num_workers > 1)
1491     {
1492       vec_foreach (tnm, nm->per_thread_data)
1493         {
1494           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1495             {
1496               nat44_ei_user_del_sessions (
1497                 pool_elt_at_index (tnm->users, value.value),
1498                 tnm->thread_index);
1499               rv = 0;
1500               break;
1501             }
1502         }
1503     }
1504   else
1505     {
1506       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1507       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1508         {
1509           nat44_ei_user_del_sessions (
1510             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1511           rv = 0;
1512         }
1513     }
1514   return rv;
1515 }
1516
1517 void
1518 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1519                                       nat44_ei_main_per_thread_data_t *tnm,
1520                                       nat44_ei_user_key_t u_key, int addr_only,
1521                                       ip4_address_t e_addr, u16 e_port)
1522 {
1523   clib_bihash_kv_8_8_t kv, value;
1524   kv.key = u_key.as_u64;
1525   u64 user_index;
1526   dlist_elt_t *head, *elt;
1527   nat44_ei_user_t *u;
1528   nat44_ei_session_t *s;
1529   u32 elt_index, head_index, ses_index;
1530
1531   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1532     {
1533       user_index = value.value;
1534       u = pool_elt_at_index (tnm->users, user_index);
1535       if (u->nstaticsessions)
1536         {
1537           head_index = u->sessions_per_user_list_head_index;
1538           head = pool_elt_at_index (tnm->list_pool, head_index);
1539           elt_index = head->next;
1540           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1541           ses_index = elt->value;
1542           while (ses_index != ~0)
1543             {
1544               s = pool_elt_at_index (tnm->sessions, ses_index);
1545               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1546               ses_index = elt->value;
1547
1548               if (!addr_only)
1549                 {
1550                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1551                       (s->out2in.port != e_port))
1552                     continue;
1553                 }
1554
1555               if (!nat44_ei_is_session_static (s))
1556                 continue;
1557
1558               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1559                                              0);
1560               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1561
1562               if (!addr_only)
1563                 break;
1564             }
1565         }
1566     }
1567 }
1568
1569 u32
1570 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1571                                   u8 is_output)
1572 {
1573   nat44_ei_main_t *nm = &nat44_ei_main;
1574   u32 next_worker_index = 0;
1575   u32 hash;
1576
1577   next_worker_index = nm->first_worker_index;
1578   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1579          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1580
1581   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1582     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1583   else
1584     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1585
1586   return next_worker_index;
1587 }
1588
1589 u32
1590 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1591                                   u32 rx_fib_index0, u8 is_output)
1592 {
1593   nat44_ei_main_t *nm = &nat44_ei_main;
1594   udp_header_t *udp;
1595   u16 port;
1596   clib_bihash_kv_8_8_t kv, value;
1597   nat44_ei_static_mapping_t *m;
1598   u32 proto;
1599   u32 next_worker_index = 0;
1600
1601   /* first try static mappings without port */
1602   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1603     {
1604       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1605       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1606                                    &value))
1607         {
1608           m = pool_elt_at_index (nm->static_mappings, value.value);
1609           return m->workers[0];
1610         }
1611     }
1612
1613   proto = ip_proto_to_nat_proto (ip0->protocol);
1614   udp = ip4_next_header (ip0);
1615   port = vnet_buffer (b)->ip.reass.l4_dst_port;
1616
1617   /* unknown protocol */
1618   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1619     {
1620       /* use current thread */
1621       return vlib_get_thread_index ();
1622     }
1623
1624   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1625     {
1626       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1627       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1628       if (!icmp_type_is_error_message (
1629             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1630         port = vnet_buffer (b)->ip.reass.l4_src_port;
1631       else
1632         {
1633           /* if error message, then it's not fragmented and we can access it */
1634           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1635           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1636           void *l4_header = ip4_next_header (inner_ip);
1637           switch (proto)
1638             {
1639             case NAT_PROTOCOL_ICMP:
1640               icmp = (icmp46_header_t *) l4_header;
1641               echo = (icmp_echo_header_t *) (icmp + 1);
1642               port = echo->identifier;
1643               break;
1644             case NAT_PROTOCOL_UDP:
1645             case NAT_PROTOCOL_TCP:
1646               port = ((tcp_udp_header_t *) l4_header)->src_port;
1647               break;
1648             default:
1649               return vlib_get_thread_index ();
1650             }
1651         }
1652     }
1653
1654   /* try static mappings with port */
1655   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1656     {
1657       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1658       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1659                                    &value))
1660         {
1661           m = pool_elt_at_index (nm->static_mappings, value.value);
1662           return m->workers[0];
1663         }
1664     }
1665
1666   /* worker by outside port */
1667   next_worker_index = nm->first_worker_index;
1668   next_worker_index +=
1669     nm->workers[(clib_net_to_host_u16 (port) - 1024) / nm->port_per_thread];
1670   return next_worker_index;
1671 }
1672
1673 static int
1674 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1675                            u32 thread_index, nat_protocol_t proto,
1676                            ip4_address_t s_addr, ip4_address_t *addr,
1677                            u16 *port, u16 port_per_thread,
1678                            u32 snat_thread_index)
1679 {
1680   nat44_ei_main_t *nm = &nat44_ei_main;
1681   nat44_ei_address_t *a, *ga = 0;
1682   u32 portnum;
1683   int i;
1684
1685   if (vec_len (addresses) > 0)
1686     {
1687
1688       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1689
1690       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1691         {
1692           a = addresses + i;
1693           switch (proto)
1694             {
1695 #define _(N, j, n, s)                                                         \
1696   case NAT_PROTOCOL_##N:                                                      \
1697     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1698       {                                                                       \
1699         if (a->fib_index == fib_index)                                        \
1700           {                                                                   \
1701             while (1)                                                         \
1702               {                                                               \
1703                 portnum = (port_per_thread * snat_thread_index) +             \
1704                           nat_random_port (&nm->random_seed, 0,               \
1705                                            port_per_thread - 1) +             \
1706                           1024;                                               \
1707                 if (a->busy_##n##_port_refcounts[portnum])                    \
1708                   continue;                                                   \
1709                 --a->busy_##n##_port_refcounts[portnum];                      \
1710                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1711                 a->busy_##n##_ports++;                                        \
1712                 *addr = a->addr;                                              \
1713                 *port = clib_host_to_net_u16 (portnum);                       \
1714                 return 0;                                                     \
1715               }                                                               \
1716           }                                                                   \
1717         else if (a->fib_index == ~0)                                          \
1718           {                                                                   \
1719             ga = a;                                                           \
1720           }                                                                   \
1721       }                                                                       \
1722     break;
1723               foreach_nat_protocol;
1724             default:
1725               nat_elog_info (nm, "unknown protocol");
1726               return 1;
1727             }
1728         }
1729
1730       for (i = 0; i < s_addr_offset; ++i)
1731         {
1732           a = addresses + i;
1733           switch (proto)
1734             {
1735               foreach_nat_protocol;
1736             default:
1737               nat_elog_info (nm, "unknown protocol");
1738               return 1;
1739             }
1740         }
1741   if (ga)
1742     {
1743       a = ga;
1744       // fake fib index to reuse macro
1745       fib_index = ~0;
1746       switch (proto)
1747         {
1748           foreach_nat_protocol;
1749             default : nat_elog_info (nm, "unknown protocol");
1750           return 1;
1751         }
1752     }
1753     }
1754
1755 #undef _
1756
1757   /* Totally out of translations to use... */
1758   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1759   return 1;
1760 }
1761
1762 static int
1763 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1764                          u32 thread_index, nat_protocol_t proto,
1765                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1766                          u16 port_per_thread, u32 snat_thread_index)
1767 {
1768   nat44_ei_main_t *nm = &nat44_ei_main;
1769   nat44_ei_address_t *a = addresses;
1770   u16 portnum, ports;
1771
1772   ports = nm->end_port - nm->start_port + 1;
1773
1774   if (!vec_len (addresses))
1775     goto exhausted;
1776
1777   switch (proto)
1778     {
1779 #define _(N, i, n, s)                                                         \
1780   case NAT_PROTOCOL_##N:                                                      \
1781     if (a->busy_##n##_ports < ports)                                          \
1782       {                                                                       \
1783         while (1)                                                             \
1784           {                                                                   \
1785             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1786                                        nm->end_port);                         \
1787             if (a->busy_##n##_port_refcounts[portnum])                        \
1788               continue;                                                       \
1789             ++a->busy_##n##_port_refcounts[portnum];                          \
1790             a->busy_##n##_ports++;                                            \
1791             *addr = a->addr;                                                  \
1792             *port = clib_host_to_net_u16 (portnum);                           \
1793             return 0;                                                         \
1794           }                                                                   \
1795       }                                                                       \
1796     break;
1797       foreach_nat_protocol
1798 #undef _
1799         default : nat_elog_info (nm, "unknown protocol");
1800       return 1;
1801     }
1802
1803 exhausted:
1804   /* Totally out of translations to use... */
1805   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1806   return 1;
1807 }
1808
1809 static int
1810 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1811                         u32 thread_index, nat_protocol_t proto,
1812                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1813                         u16 port_per_thread, u32 snat_thread_index)
1814 {
1815   nat44_ei_main_t *nm = &nat44_ei_main;
1816   nat44_ei_address_t *a = addresses;
1817   u16 m, ports, portnum, A, j;
1818   m = 16 - (nm->psid_offset + nm->psid_length);
1819   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1820
1821   if (!vec_len (addresses))
1822     goto exhausted;
1823
1824   switch (proto)
1825     {
1826 #define _(N, i, n, s)                                                         \
1827   case NAT_PROTOCOL_##N:                                                      \
1828     if (a->busy_##n##_ports < ports)                                          \
1829       {                                                                       \
1830         while (1)                                                             \
1831           {                                                                   \
1832             A = nat_random_port (&nm->random_seed, 1,                         \
1833                                  pow2_mask (nm->psid_offset));                \
1834             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
1835             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
1836             if (a->busy_##n##_port_refcounts[portnum])                        \
1837               continue;                                                       \
1838             ++a->busy_##n##_port_refcounts[portnum];                          \
1839             a->busy_##n##_ports++;                                            \
1840             *addr = a->addr;                                                  \
1841             *port = clib_host_to_net_u16 (portnum);                           \
1842             return 0;                                                         \
1843           }                                                                   \
1844       }                                                                       \
1845     break;
1846       foreach_nat_protocol
1847 #undef _
1848         default : nat_elog_info (nm, "unknown protocol");
1849       return 1;
1850     }
1851
1852 exhausted:
1853   /* Totally out of translations to use... */
1854   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1855   return 1;
1856 }
1857
1858 void
1859 nat44_ei_set_alloc_default ()
1860 {
1861   nat44_ei_main_t *nm = &nat44_ei_main;
1862
1863   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1864   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1865 }
1866
1867 void
1868 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1869 {
1870   nat44_ei_main_t *nm = &nat44_ei_main;
1871
1872   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1873   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1874   nm->start_port = start_port;
1875   nm->end_port = end_port;
1876 }
1877
1878 void
1879 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1880 {
1881   nat44_ei_main_t *nm = &nat44_ei_main;
1882
1883   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1884   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1885   nm->psid = psid;
1886   nm->psid_offset = psid_offset;
1887   nm->psid_length = psid_length;
1888 }
1889
1890 void
1891 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
1892                          u32 thread_index)
1893 {
1894   nat44_ei_main_per_thread_data_t *tnm =
1895     vec_elt_at_index (nm->per_thread_data, thread_index);
1896   clib_bihash_kv_8_8_t kv, value;
1897   nat44_ei_user_t *u;
1898   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
1899                                       .fib_index = ses->in2out.fib_index };
1900   const u8 u_static = nat44_ei_is_session_static (ses);
1901
1902   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
1903   pool_put_index (tnm->list_pool, ses->per_user_index);
1904
1905   pool_put (tnm->sessions, ses);
1906   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1907                            pool_elts (tnm->sessions));
1908
1909   kv.key = u_key.as_u64;
1910   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1911     {
1912       u = pool_elt_at_index (tnm->users, value.value);
1913       if (u_static)
1914         u->nstaticsessions--;
1915       else
1916         u->nsessions--;
1917
1918       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1919     }
1920 }
1921
1922 int
1923 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
1924                       nat_protocol_t proto, u32 vrf_id, int is_in)
1925 {
1926   nat44_ei_main_per_thread_data_t *tnm;
1927   clib_bihash_kv_8_8_t kv, value;
1928   u32 fib_index;
1929   nat44_ei_session_t *s;
1930   clib_bihash_8_8_t *t;
1931
1932   fail_if_disabled ();
1933
1934   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
1935   init_nat_k (&kv, *addr, port, fib_index, proto);
1936   t = is_in ? &nm->in2out : &nm->out2in;
1937   if (!clib_bihash_search_8_8 (t, &kv, &value))
1938     {
1939       // this is called from API/CLI, so the world is stopped here
1940       // it's safe to manipulate arbitrary per-thread data
1941       u32 thread_index = nat_value_get_thread_index (&value);
1942       tnm = vec_elt_at_index (nm->per_thread_data, thread_index);
1943       u32 session_index = nat_value_get_session_index (&value);
1944       if (pool_is_free_index (tnm->sessions, session_index))
1945         return VNET_API_ERROR_UNSPECIFIED;
1946
1947       s = pool_elt_at_index (tnm->sessions, session_index);
1948       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
1949       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1950       return 0;
1951     }
1952
1953   return VNET_API_ERROR_NO_SUCH_ENTRY;
1954 }
1955
1956 u32
1957 nat44_ei_get_thread_idx_by_port (u16 e_port)
1958 {
1959   nat44_ei_main_t *nm = &nat44_ei_main;
1960   u32 thread_idx = nm->num_workers;
1961   if (nm->num_workers > 1)
1962     {
1963       thread_idx = nm->first_worker_index +
1964                    nm->workers[(e_port - 1024) / nm->port_per_thread];
1965     }
1966   return thread_idx;
1967 }
1968
1969 void
1970 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
1971                               int is_add)
1972 {
1973   nat44_ei_main_t *nm = &nat44_ei_main;
1974   fib_prefix_t prefix = {
1975     .fp_len = p_len,
1976     .fp_proto = FIB_PROTOCOL_IP4,
1977     .fp_addr = {
1978                 .ip4.as_u32 = addr->as_u32,
1979                 },
1980   };
1981   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1982
1983   if (is_add)
1984     {
1985       fib_table_entry_update_one_path (fib_index, &prefix, nm->fib_src_low,
1986                                        (FIB_ENTRY_FLAG_CONNECTED |
1987                                         FIB_ENTRY_FLAG_LOCAL |
1988                                         FIB_ENTRY_FLAG_EXCLUSIVE),
1989                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
1990                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
1991     }
1992   else
1993     {
1994       fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
1995     }
1996 }
1997
1998 int
1999 nat44_ei_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2000 {
2001   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2002   nat44_ei_main_t *nm = &nat44_ei_main;
2003   nat44_ei_address_t *a = 0;
2004   int i;
2005
2006   for (i = 0; i < vec_len (nm->addresses); i++)
2007     {
2008       a = nm->addresses + i;
2009
2010       if (a->addr.as_u32 != addr.as_u32)
2011         continue;
2012
2013       switch (proto)
2014         {
2015 #define _(N, j, n, s)                                                         \
2016   case NAT_PROTOCOL_##N:                                                      \
2017     if (a->busy_##n##_port_refcounts[port])                                   \
2018       goto done;                                                              \
2019     ++a->busy_##n##_port_refcounts[port];                                     \
2020     if (port > 1024)                                                          \
2021       {                                                                       \
2022         a->busy_##n##_ports++;                                                \
2023         a->busy_##n##_ports_per_thread[ti]++;                                 \
2024       }                                                                       \
2025     break;
2026           foreach_nat_protocol
2027 #undef _
2028             default : nat_elog_info (nm, "unknown protocol");
2029           goto done;
2030         }
2031
2032       return 0;
2033     }
2034
2035 done:
2036   return 1;
2037 }
2038
2039 int
2040 nat44_ei_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2041 {
2042   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2043   nat44_ei_main_t *nm = &nat44_ei_main;
2044   nat44_ei_address_t *a = 0;
2045   int i;
2046
2047   for (i = 0; i < vec_len (nm->addresses); i++)
2048     {
2049       a = nm->addresses + i;
2050
2051       if (a->addr.as_u32 != addr.as_u32)
2052         continue;
2053
2054       switch (proto)
2055         {
2056 #define _(N, j, n, s)                                                         \
2057   case NAT_PROTOCOL_##N:                                                      \
2058     --a->busy_##n##_port_refcounts[port];                                     \
2059     if (port > 1024)                                                          \
2060       {                                                                       \
2061         a->busy_##n##_ports--;                                                \
2062         a->busy_##n##_ports_per_thread[ti]--;                                 \
2063       }                                                                       \
2064     break;
2065           foreach_nat_protocol
2066 #undef _
2067             default : nat_elog_info (nm, "unknown protocol");
2068           goto done;
2069         }
2070
2071       return 0;
2072     }
2073
2074 done:
2075   return 1;
2076 }
2077
2078 void
2079 nat44_ei_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2080                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2081                              u32 flags, ip4_address_t pool_addr, u8 *tag)
2082 {
2083   nat44_ei_static_map_resolve_t *rp;
2084   nat44_ei_main_t *nm = &nat44_ei_main;
2085
2086   vec_add2 (nm->to_resolve, rp, 1);
2087   rp->l_addr.as_u32 = l_addr.as_u32;
2088   rp->l_port = l_port;
2089   rp->e_port = e_port;
2090   rp->sw_if_index = sw_if_index;
2091   rp->vrf_id = vrf_id;
2092   rp->proto = proto;
2093   rp->flags = flags;
2094   rp->pool_addr = pool_addr;
2095   rp->tag = vec_dup (tag);
2096 }
2097
2098 int
2099 nat44_ei_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2100                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2101                              u32 flags, int *out)
2102 {
2103   nat44_ei_static_map_resolve_t *rp;
2104   nat44_ei_main_t *nm = &nat44_ei_main;
2105   int i;
2106
2107   for (i = 0; i < vec_len (nm->to_resolve); i++)
2108     {
2109       rp = nm->to_resolve + i;
2110
2111       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
2112         {
2113           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
2114             {
2115               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2116                 {
2117                   if (rp->e_port != e_port || rp->proto != proto)
2118                     {
2119                       continue;
2120                     }
2121                 }
2122             }
2123           else if (rp->l_addr.as_u32 == l_addr.as_u32)
2124             {
2125               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2126                 {
2127                   if (rp->l_port != l_port || rp->e_port != e_port ||
2128                       rp->proto != proto)
2129                     {
2130                       continue;
2131                     }
2132                 }
2133             }
2134           else
2135             {
2136               continue;
2137             }
2138           if (out)
2139             {
2140               *out = i;
2141             }
2142           return 0;
2143         }
2144     }
2145   return 1;
2146 }
2147
2148 int
2149 nat44_ei_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2150                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2151                              u32 flags)
2152 {
2153   nat44_ei_main_t *nm = &nat44_ei_main;
2154   int i;
2155   if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2156                                     sw_if_index, flags, &i))
2157     {
2158       vec_del1 (nm->to_resolve, i);
2159       return 0;
2160     }
2161   return 1;
2162 }
2163
2164 void
2165 delete_matching_dynamic_sessions (const nat44_ei_static_mapping_t *m,
2166                                   u32 worker_index)
2167 {
2168   nat44_ei_main_t *nm = &nat44_ei_main;
2169   clib_bihash_kv_8_8_t kv, value;
2170   nat44_ei_session_t *s;
2171   nat44_ei_user_key_t u_key;
2172   nat44_ei_user_t *u;
2173   nat44_ei_main_per_thread_data_t *tnm;
2174   dlist_elt_t *head, *elt;
2175   u32 elt_index, head_index;
2176   u32 ses_index;
2177   u64 user_index;
2178
2179   if (nm->static_mapping_only)
2180     return;
2181
2182   tnm = vec_elt_at_index (nm->per_thread_data, worker_index);
2183
2184   u_key.addr = m->local_addr;
2185   u_key.fib_index = m->fib_index;
2186   kv.key = u_key.as_u64;
2187   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2188     {
2189       user_index = value.value;
2190       u = pool_elt_at_index (tnm->users, user_index);
2191       if (u->nsessions)
2192         {
2193           head_index = u->sessions_per_user_list_head_index;
2194           head = pool_elt_at_index (tnm->list_pool, head_index);
2195           elt_index = head->next;
2196           elt = pool_elt_at_index (tnm->list_pool, elt_index);
2197           ses_index = elt->value;
2198           while (ses_index != ~0)
2199             {
2200               s = pool_elt_at_index (tnm->sessions, ses_index);
2201               elt = pool_elt_at_index (tnm->list_pool, elt->next);
2202               ses_index = elt->value;
2203
2204               if (nat44_ei_is_session_static (s))
2205                 continue;
2206
2207               if (!is_sm_addr_only (m->flags) &&
2208                   s->in2out.port != m->local_port)
2209                 continue;
2210
2211               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
2212                                              0);
2213               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2214
2215               if (!is_sm_addr_only (m->flags))
2216                 break;
2217             }
2218         }
2219     }
2220 }
2221
2222 int
2223 nat44_ei_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2224                              u16 l_port, u16 e_port, nat_protocol_t proto,
2225                              u32 vrf_id, u32 sw_if_index, u32 flags,
2226                              ip4_address_t pool_addr, u8 *tag)
2227 {
2228   nat44_ei_main_t *nm = &nat44_ei_main;
2229   clib_bihash_kv_8_8_t kv, value;
2230   nat44_ei_lb_addr_port_t *local;
2231   nat44_ei_static_mapping_t *m;
2232   u32 fib_index = ~0;
2233   u32 worker_index;
2234
2235   fail_if_disabled ();
2236
2237   if (is_sm_addr_only (flags))
2238     {
2239       e_port = l_port = proto = 0;
2240     }
2241
2242   if (sw_if_index != ~0)
2243     {
2244       // this mapping is interface bound
2245       ip4_address_t *first_int_addr;
2246
2247       // check if this record isn't registered for resolve
2248       if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2249                                         sw_if_index, flags, 0))
2250         {
2251           return VNET_API_ERROR_VALUE_EXIST;
2252         }
2253       // register record for resolve
2254       nat44_ei_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2255                                    sw_if_index, flags, pool_addr, tag);
2256
2257       first_int_addr =
2258         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2259       if (!first_int_addr)
2260         {
2261           // dhcp resolution required
2262           return 0;
2263         }
2264
2265       e_addr.as_u32 = first_int_addr->as_u32;
2266     }
2267
2268   if (is_sm_identity_nat (flags))
2269     {
2270       l_port = e_port;
2271       l_addr.as_u32 = e_addr.as_u32;
2272     }
2273
2274   // fib index 0
2275   init_nat_k (&kv, e_addr, e_port, 0, proto);
2276
2277   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2278     {
2279       m = pool_elt_at_index (nm->static_mappings, value.value);
2280       if (!is_sm_identity_nat (m->flags))
2281         {
2282           return VNET_API_ERROR_VALUE_EXIST;
2283         }
2284
2285       // case:
2286       // adding local identity nat record for different vrf table
2287       pool_foreach (local, m->locals)
2288         {
2289           if (local->vrf_id == vrf_id)
2290             {
2291               return VNET_API_ERROR_VALUE_EXIST;
2292             }
2293         }
2294
2295       pool_get (m->locals, local);
2296
2297       local->vrf_id = vrf_id;
2298       local->fib_index = fib_table_find_or_create_and_lock (
2299         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2300
2301       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
2302                    m->proto, 0, m - nm->static_mappings);
2303       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2304
2305       return 0;
2306     }
2307
2308   if (vrf_id != ~0)
2309     {
2310       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
2311                                                      nm->fib_src_low);
2312     }
2313   else
2314     {
2315       // fallback to default vrf
2316       vrf_id = nm->inside_vrf_id;
2317       fib_index = nm->inside_fib_index;
2318       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2319     }
2320
2321   if (!is_sm_identity_nat (flags))
2322     {
2323       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2324       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2325         {
2326           return VNET_API_ERROR_VALUE_EXIST;
2327         }
2328     }
2329
2330   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2331     {
2332       if (nat44_ei_reserve_port (e_addr, e_port, proto))
2333         {
2334           // remove resolve record
2335           if ((sw_if_index != ~0) && !is_sm_identity_nat (flags))
2336             {
2337               nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto,
2338                                            vrf_id, sw_if_index, flags);
2339             }
2340           return VNET_API_ERROR_NO_SUCH_ENTRY;
2341         }
2342     }
2343
2344   pool_get (nm->static_mappings, m);
2345   clib_memset (m, 0, sizeof (*m));
2346
2347   m->flags = flags;
2348   m->local_addr = l_addr;
2349   m->external_addr = e_addr;
2350
2351   m->tag = vec_dup (tag);
2352
2353   if (!is_sm_addr_only (flags))
2354     {
2355       m->local_port = l_port;
2356       m->external_port = e_port;
2357       m->proto = proto;
2358     }
2359
2360   if (is_sm_identity_nat (flags))
2361     {
2362       pool_get (m->locals, local);
2363
2364       local->vrf_id = vrf_id;
2365       local->fib_index = fib_index;
2366     }
2367   else
2368     {
2369       m->vrf_id = vrf_id;
2370       m->fib_index = fib_index;
2371     }
2372
2373   init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2374                m - nm->static_mappings);
2375   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2376
2377   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2378                m - nm->static_mappings);
2379   clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2380
2381   if (nm->num_workers > 1)
2382     {
2383       // store worker index for this record
2384       ip4_header_t ip = {
2385         .src_address = m->local_addr,
2386       };
2387       worker_index = nat44_ei_get_in2out_worker_index (&ip, m->fib_index, 0);
2388       vec_add1 (m->workers, worker_index);
2389     }
2390   else
2391     {
2392       worker_index = nm->num_workers;
2393     }
2394   delete_matching_dynamic_sessions (m, worker_index);
2395
2396   if (is_sm_addr_only (flags))
2397     {
2398       nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
2399     }
2400
2401   return 0;
2402 }
2403
2404 int
2405 nat44_ei_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2406                              u16 l_port, u16 e_port, nat_protocol_t proto,
2407                              u32 vrf_id, u32 sw_if_index, u32 flags)
2408 {
2409   nat44_ei_main_per_thread_data_t *tnm;
2410   nat44_ei_main_t *nm = &nat44_ei_main;
2411   clib_bihash_kv_8_8_t kv, value;
2412   nat44_ei_lb_addr_port_t *local;
2413   nat44_ei_static_mapping_t *m;
2414   u32 fib_index = ~0;
2415   nat44_ei_user_key_t u_key;
2416
2417   fail_if_disabled ();
2418
2419   if (is_sm_addr_only (flags))
2420     {
2421       e_port = l_port = proto = 0;
2422     }
2423
2424   if (sw_if_index != ~0)
2425     {
2426       // this mapping is interface bound
2427       ip4_address_t *first_int_addr;
2428
2429       // delete record registered for resolve
2430       if (nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2431                                        sw_if_index, flags))
2432         {
2433           return VNET_API_ERROR_NO_SUCH_ENTRY;
2434         }
2435
2436       first_int_addr =
2437         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2438       if (!first_int_addr)
2439         {
2440           // dhcp resolution required
2441           return 0;
2442         }
2443
2444       e_addr.as_u32 = first_int_addr->as_u32;
2445     }
2446
2447   if (is_sm_identity_nat (flags))
2448     {
2449       l_port = e_port;
2450       l_addr.as_u32 = e_addr.as_u32;
2451     }
2452
2453   // fib index 0
2454   init_nat_k (&kv, e_addr, e_port, 0, proto);
2455
2456   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2457     {
2458       if (sw_if_index != ~0)
2459         {
2460           return 0;
2461         }
2462       return VNET_API_ERROR_NO_SUCH_ENTRY;
2463     }
2464
2465   m = pool_elt_at_index (nm->static_mappings, value.value);
2466
2467   if (is_sm_identity_nat (flags))
2468     {
2469       u8 found = 0;
2470
2471       if (vrf_id == ~0)
2472         {
2473           vrf_id = nm->inside_vrf_id;
2474         }
2475
2476       pool_foreach (local, m->locals)
2477         {
2478           if (local->vrf_id == vrf_id)
2479             {
2480               local = pool_elt_at_index (m->locals, local - m->locals);
2481               fib_index = local->fib_index;
2482               pool_put (m->locals, local);
2483               found = 1;
2484             }
2485         }
2486       if (!found)
2487         {
2488           return VNET_API_ERROR_NO_SUCH_ENTRY;
2489         }
2490     }
2491   else
2492     {
2493       fib_index = m->fib_index;
2494     }
2495
2496   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2497     {
2498       if (nat44_ei_free_port (e_addr, e_port, proto))
2499         {
2500           return VNET_API_ERROR_INVALID_VALUE;
2501         }
2502     }
2503
2504   init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2505   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2506
2507   if (!nm->static_mapping_only || nm->static_mapping_connection_tracking)
2508     {
2509       // delete sessions for static mapping
2510       if (nm->num_workers > 1)
2511         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2512       else
2513         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2514
2515       u_key.addr = m->local_addr;
2516       u_key.fib_index = fib_index;
2517       kv.key = u_key.as_u64;
2518       nat44_ei_static_mapping_del_sessions (
2519         nm, tnm, u_key, is_sm_addr_only (flags), e_addr, e_port);
2520     }
2521
2522   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2523
2524   if (!pool_elts (m->locals))
2525     {
2526       // this is last record remove all required stuff
2527       // fib_index 0
2528       init_nat_k (&kv, e_addr, e_port, 0, proto);
2529       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2530
2531       vec_free (m->tag);
2532       vec_free (m->workers);
2533       pool_put (nm->static_mappings, m);
2534
2535       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
2536         {
2537           nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
2538         }
2539     }
2540
2541   return 0;
2542 }
2543
2544 int
2545 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2546                                u32 match_fib_index,
2547                                nat_protocol_t match_protocol,
2548                                ip4_address_t *mapping_addr, u16 *mapping_port,
2549                                u32 *mapping_fib_index, u8 by_external,
2550                                u8 *is_addr_only, u8 *is_identity_nat)
2551 {
2552   nat44_ei_main_t *nm = &nat44_ei_main;
2553   clib_bihash_kv_8_8_t kv, value;
2554   nat44_ei_static_mapping_t *m;
2555   u16 port;
2556
2557   if (by_external)
2558     {
2559       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2560       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2561                                   &value))
2562         {
2563           /* Try address only mapping */
2564           init_nat_k (&kv, match_addr, 0, 0, 0);
2565           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2566                                       &value))
2567             return 1;
2568         }
2569       m = pool_elt_at_index (nm->static_mappings, value.value);
2570
2571       *mapping_fib_index = m->fib_index;
2572       *mapping_addr = m->local_addr;
2573       port = m->local_port;
2574     }
2575   else
2576     {
2577       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2578                   match_protocol);
2579       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2580         {
2581           /* Try address only mapping */
2582           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2583           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2584                                       &value))
2585             return 1;
2586         }
2587       m = pool_elt_at_index (nm->static_mappings, value.value);
2588
2589       *mapping_fib_index = nm->outside_fib_index;
2590       *mapping_addr = m->external_addr;
2591       port = m->external_port;
2592     }
2593
2594   /* Address only mapping doesn't change port */
2595   if (is_sm_addr_only (m->flags))
2596     *mapping_port = match_port;
2597   else
2598     *mapping_port = port;
2599
2600   if (PREDICT_FALSE (is_addr_only != 0))
2601     *is_addr_only = is_sm_addr_only (m->flags);
2602
2603   if (PREDICT_FALSE (is_identity_nat != 0))
2604     *is_identity_nat = is_sm_identity_nat (m->flags);
2605
2606   return 0;
2607 }
2608
2609 static void
2610 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2611 {
2612   pool_free (tnm->list_pool);
2613   pool_free (tnm->lru_pool);
2614   pool_free (tnm->sessions);
2615   pool_free (tnm->users);
2616
2617   clib_bihash_free_8_8 (&tnm->user_hash);
2618 }
2619
2620 u8 *
2621 format_nat44_ei_key (u8 *s, va_list *args)
2622 {
2623   u64 key = va_arg (*args, u64);
2624
2625   ip4_address_t addr;
2626   u16 port;
2627   nat_protocol_t protocol;
2628   u32 fib_index;
2629
2630   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2631
2632   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2633               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2634               fib_index);
2635   return s;
2636 }
2637
2638 u8 *
2639 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2640 {
2641   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2642   nat44_ei_user_key_t k;
2643
2644   k.as_u64 = v->key;
2645
2646   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2647               k.fib_index, v->value);
2648
2649   return s;
2650 }
2651
2652 u8 *
2653 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2654 {
2655   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2656
2657   s = format (s, "%U thread-index %llu session-index %llu",
2658               format_nat44_ei_key, v->key, nat_value_get_thread_index (v),
2659               nat_value_get_session_index (v));
2660
2661   return s;
2662 }
2663
2664 u8 *
2665 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2666 {
2667   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2668
2669   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2670               v->value);
2671
2672   return s;
2673 }
2674
2675 static void
2676 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2677                          u32 translations, u32 translation_buckets,
2678                          u32 user_buckets)
2679 {
2680   dlist_elt_t *head;
2681
2682   pool_alloc (tnm->list_pool, translations);
2683   pool_alloc (tnm->lru_pool, translations);
2684   pool_alloc (tnm->sessions, translations);
2685
2686   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2687
2688   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2689                                      format_nat44_ei_user_kvp);
2690
2691   pool_get (tnm->lru_pool, head);
2692   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2693   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2694
2695   pool_get (tnm->lru_pool, head);
2696   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2697   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2698
2699   pool_get (tnm->lru_pool, head);
2700   tnm->udp_lru_head_index = head - tnm->lru_pool;
2701   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2702
2703   pool_get (tnm->lru_pool, head);
2704   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2705   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2706
2707   pool_get (tnm->lru_pool, head);
2708   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2709   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2710 }
2711
2712 static void
2713 nat44_ei_db_free ()
2714 {
2715   nat44_ei_main_t *nm = &nat44_ei_main;
2716   nat44_ei_main_per_thread_data_t *tnm;
2717
2718   pool_free (nm->static_mappings);
2719   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
2720   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
2721
2722   if (nm->pat)
2723     {
2724       clib_bihash_free_8_8 (&nm->in2out);
2725       clib_bihash_free_8_8 (&nm->out2in);
2726       vec_foreach (tnm, nm->per_thread_data)
2727         {
2728           nat44_ei_worker_db_free (tnm);
2729         }
2730     }
2731 }
2732
2733 static void
2734 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2735 {
2736   nat44_ei_main_t *nm = &nat44_ei_main;
2737   nat44_ei_main_per_thread_data_t *tnm;
2738
2739   u32 static_mapping_buckets = 1024;
2740   u32 static_mapping_memory_size = 64 << 20;
2741
2742   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2743                         "static_mapping_by_local", static_mapping_buckets,
2744                         static_mapping_memory_size);
2745   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2746                         "static_mapping_by_external", static_mapping_buckets,
2747                         static_mapping_memory_size);
2748   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2749                                      format_nat44_ei_static_mapping_kvp);
2750   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2751                                      format_nat44_ei_static_mapping_kvp);
2752
2753   if (nm->pat)
2754     {
2755       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2756       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2757       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2758                                          format_nat44_ei_session_kvp);
2759       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2760                                          format_nat44_ei_session_kvp);
2761       vec_foreach (tnm, nm->per_thread_data)
2762         {
2763           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2764                                    user_buckets);
2765         }
2766     }
2767 }
2768
2769 void
2770 nat44_ei_sessions_clear ()
2771 {
2772   nat44_ei_main_t *nm = &nat44_ei_main;
2773   nat44_ei_main_per_thread_data_t *tnm;
2774
2775   if (nm->pat)
2776     {
2777       clib_bihash_free_8_8 (&nm->in2out);
2778       clib_bihash_free_8_8 (&nm->out2in);
2779       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2780       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2781       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2782                                          format_nat44_ei_session_kvp);
2783       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2784                                          format_nat44_ei_session_kvp);
2785       vec_foreach (tnm, nm->per_thread_data)
2786         {
2787           nat44_ei_worker_db_free (tnm);
2788           nat44_ei_worker_db_init (tnm, nm->translations,
2789                                    nm->translation_buckets, nm->user_buckets);
2790         }
2791     }
2792
2793   vlib_zero_simple_counter (&nm->total_users, 0);
2794   vlib_zero_simple_counter (&nm->total_sessions, 0);
2795   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2796 }
2797
2798 static void
2799 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2800                              u32 new_fib_index, u32 old_fib_index)
2801 {
2802   nat44_ei_main_t *nm = &nat44_ei_main;
2803   nat44_ei_outside_fib_t *outside_fib;
2804   nat44_ei_interface_t *i;
2805   u8 is_add = 1;
2806   u8 match = 0;
2807
2808   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2809       (!vec_len (nm->outside_fibs)))
2810     {
2811       return;
2812     }
2813
2814   pool_foreach (i, nm->interfaces)
2815     {
2816       if (i->sw_if_index == sw_if_index)
2817         {
2818           if (!(nat44_ei_interface_is_outside (i)))
2819             return;
2820           match = 1;
2821         }
2822     }
2823
2824   pool_foreach (i, nm->output_feature_interfaces)
2825     {
2826       if (i->sw_if_index == sw_if_index)
2827         {
2828           if (!(nat44_ei_interface_is_outside (i)))
2829             return;
2830           match = 1;
2831         }
2832     }
2833
2834   if (!match)
2835     return;
2836
2837   vec_foreach (outside_fib, nm->outside_fibs)
2838     {
2839       if (outside_fib->fib_index == old_fib_index)
2840         {
2841           outside_fib->refcount--;
2842           if (!outside_fib->refcount)
2843             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2844           break;
2845         }
2846     }
2847
2848   vec_foreach (outside_fib, nm->outside_fibs)
2849     {
2850       if (outside_fib->fib_index == new_fib_index)
2851         {
2852           outside_fib->refcount++;
2853           is_add = 0;
2854           break;
2855         }
2856     }
2857
2858   if (is_add)
2859     {
2860       vec_add2 (nm->outside_fibs, outside_fib, 1);
2861       outside_fib->refcount = 1;
2862       outside_fib->fib_index = new_fib_index;
2863     }
2864 }
2865
2866 int
2867 nat44_ei_add_address (ip4_address_t *addr, u32 vrf_id)
2868 {
2869   nat44_ei_main_t *nm = &nat44_ei_main;
2870   vlib_thread_main_t *tm = vlib_get_thread_main ();
2871   nat44_ei_address_t *ap;
2872
2873   fail_if_disabled ();
2874
2875   /* Check if address already exists */
2876   vec_foreach (ap, nm->addresses)
2877     {
2878       if (ap->addr.as_u32 == addr->as_u32)
2879         {
2880           nat44_ei_log_err ("address exist");
2881           return VNET_API_ERROR_VALUE_EXIST;
2882         }
2883     }
2884
2885   vec_add2 (nm->addresses, ap, 1);
2886
2887   ap->fib_index = ~0;
2888   ap->addr = *addr;
2889
2890   if (vrf_id != ~0)
2891     {
2892       ap->fib_index = fib_table_find_or_create_and_lock (
2893         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2894     }
2895
2896 #define _(N, i, n, s)                                                         \
2897   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
2898                sizeof (ap->busy_##n##_port_refcounts));                       \
2899   ap->busy_##n##_ports = 0;                                                   \
2900   ap->busy_##n##_ports_per_thread = 0;                                        \
2901   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
2902                            tm->n_vlib_mains - 1, 0);
2903   foreach_nat_protocol
2904 #undef _
2905
2906     nat44_ei_add_del_addr_to_fib_foreach_out_if (addr, 1);
2907
2908   return 0;
2909 }
2910
2911 int
2912 nat44_ei_del_address (ip4_address_t addr, u8 delete_sm)
2913 {
2914   nat44_ei_main_t *nm = &nat44_ei_main;
2915   nat44_ei_address_t *a = 0;
2916   nat44_ei_session_t *ses;
2917   u32 *ses_to_be_removed = 0, *ses_index;
2918   nat44_ei_main_per_thread_data_t *tnm;
2919   nat44_ei_static_mapping_t *m;
2920   int j;
2921
2922   fail_if_disabled ();
2923
2924   /* Find SNAT address */
2925   for (j = 0; j < vec_len (nm->addresses); j++)
2926     {
2927       if (nm->addresses[j].addr.as_u32 == addr.as_u32)
2928         {
2929           a = nm->addresses + j;
2930           break;
2931         }
2932     }
2933   if (!a)
2934     {
2935       nat44_ei_log_err ("no such address");
2936       return VNET_API_ERROR_NO_SUCH_ENTRY;
2937     }
2938
2939   if (delete_sm)
2940     {
2941       pool_foreach (m, nm->static_mappings)
2942         {
2943           if (m->external_addr.as_u32 == addr.as_u32)
2944             nat44_ei_del_static_mapping (m->local_addr, m->external_addr,
2945                                          m->local_port, m->external_port,
2946                                          m->proto, m->vrf_id, ~0, m->flags);
2947         }
2948     }
2949   else
2950     {
2951       /* Check if address is used in some static mapping */
2952       if (nat44_ei_is_address_used_in_static_mapping (addr))
2953         {
2954           nat44_ei_log_err ("address used in static mapping");
2955           return VNET_API_ERROR_UNSPECIFIED;
2956         }
2957     }
2958
2959   if (a->fib_index != ~0)
2960     {
2961       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2962     }
2963
2964   /* Delete sessions using address */
2965   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
2966     {
2967       vec_foreach (tnm, nm->per_thread_data)
2968         {
2969           pool_foreach (ses, tnm->sessions)
2970             {
2971               if (ses->out2in.addr.as_u32 == addr.as_u32)
2972                 {
2973                   nat44_ei_free_session_data (nm, ses,
2974                                               tnm - nm->per_thread_data, 0);
2975                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
2976                 }
2977             }
2978           vec_foreach (ses_index, ses_to_be_removed)
2979             {
2980               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
2981               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
2982             }
2983           vec_free (ses_to_be_removed);
2984         }
2985     }
2986
2987 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
2988   foreach_nat_protocol
2989 #undef _
2990
2991     vec_del1 (nm->addresses, j);
2992
2993   nat44_ei_add_del_addr_to_fib_foreach_out_if (&addr, 0);
2994
2995   return 0;
2996 }
2997
2998 int
2999 nat44_ei_add_interface_address (u32 sw_if_index)
3000 {
3001   nat44_ei_main_t *nm = &nat44_ei_main;
3002   ip4_main_t *ip4_main = nm->ip4_main;
3003   ip4_address_t *first_int_addr;
3004   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3005   int i;
3006
3007   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3008     {
3009       if (auto_add_sw_if_indices[i] == sw_if_index)
3010         {
3011           return VNET_API_ERROR_VALUE_EXIST;
3012         }
3013     }
3014
3015   /* add to the auto-address list */
3016   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
3017
3018   // if the address is already bound - or static - add it now
3019   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3020   if (first_int_addr)
3021     {
3022       (void) nat44_ei_add_address (first_int_addr, ~0);
3023     }
3024
3025   return 0;
3026 }
3027
3028 int
3029 nat44_ei_del_interface_address (u32 sw_if_index)
3030 {
3031   nat44_ei_main_t *nm = &nat44_ei_main;
3032   ip4_main_t *ip4_main = nm->ip4_main;
3033   ip4_address_t *first_int_addr;
3034   nat44_ei_static_map_resolve_t *rp;
3035   u32 *indices_to_delete = 0;
3036   int i, j;
3037   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3038
3039   fail_if_disabled ();
3040
3041   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3042
3043   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3044     {
3045       if (auto_add_sw_if_indices[i] == sw_if_index)
3046         {
3047           first_int_addr =
3048             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3049           if (first_int_addr)
3050             {
3051               (void) nat44_ei_del_address (first_int_addr[0], 1);
3052             }
3053           else
3054             {
3055               for (j = 0; j < vec_len (nm->to_resolve); j++)
3056                 {
3057                   rp = nm->to_resolve + j;
3058                   if (rp->sw_if_index == sw_if_index)
3059                     {
3060                       vec_add1 (indices_to_delete, j);
3061                     }
3062                 }
3063               if (vec_len (indices_to_delete))
3064                 {
3065                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3066                     {
3067                       vec_del1 (nm->to_resolve, j);
3068                     }
3069                   vec_free (indices_to_delete);
3070                 }
3071             }
3072
3073           vec_del1 (nm->auto_add_sw_if_indices, i);
3074           return 0;
3075         }
3076     }
3077   return VNET_API_ERROR_NO_SUCH_ENTRY;
3078 }
3079
3080 static_always_inline int
3081 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3082 {
3083   u32 *i;
3084   vec_foreach (i, sw_if_indices)
3085     {
3086       if (*i == sw_if_index)
3087         {
3088           return 1;
3089         }
3090     }
3091   return 0;
3092 }
3093
3094 static void
3095 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3096                                            u32 sw_if_index,
3097                                            ip4_address_t *address,
3098                                            u32 address_length,
3099                                            u32 if_address_index, u32 is_delete)
3100 {
3101   nat44_ei_main_t *nm = &nat44_ei_main;
3102   nat44_ei_static_map_resolve_t *rp;
3103   nat44_ei_address_t *addresses = nm->addresses;
3104   int rv, i;
3105
3106   if (!nm->enabled)
3107     {
3108       return;
3109     }
3110
3111   if (!is_sw_if_index_reg_for_auto_resolve (nm->auto_add_sw_if_indices,
3112                                             sw_if_index))
3113     {
3114       return;
3115     }
3116
3117   if (!is_delete)
3118     {
3119       /* Don't trip over lease renewal, static config */
3120       for (i = 0; i < vec_len (addresses); i++)
3121         {
3122           if (addresses[i].addr.as_u32 == address->as_u32)
3123             {
3124               return;
3125             }
3126         }
3127
3128       (void) nat44_ei_add_address (address, ~0);
3129
3130       /* Scan static map resolution vector */
3131       for (i = 0; i < vec_len (nm->to_resolve); i++)
3132         {
3133           rp = nm->to_resolve + i;
3134           if (is_sm_addr_only (rp->flags))
3135             {
3136               continue;
3137             }
3138           /* On this interface? */
3139           if (rp->sw_if_index == sw_if_index)
3140             {
3141               rv = nat44_ei_add_static_mapping (
3142                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3143                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3144               if (rv)
3145                 {
3146                   nat_elog_notice_X1 (nm, "add_static_mapping returned %d",
3147                                       "i4", rv);
3148                 }
3149             }
3150         }
3151     }
3152   else
3153     {
3154       // remove all static mapping records
3155       (void) nat44_ei_del_address (address[0], 1);
3156     }
3157 }
3158
3159 int
3160 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
3161 {
3162   fail_if_enabled ();
3163   nat44_ei_main_t *nm = &nat44_ei_main;
3164   nm->frame_queue_nelts = frame_queue_nelts;
3165   return 0;
3166 }
3167
3168 static void
3169 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
3170                                       u32 sw_if_index, ip4_address_t *address,
3171                                       u32 address_length, u32 if_address_index,
3172                                       u32 is_delete)
3173 {
3174   nat44_ei_main_t *nm = &nat44_ei_main;
3175   nat44_ei_static_map_resolve_t *rp;
3176   nat44_ei_static_mapping_t *m;
3177   clib_bihash_kv_8_8_t kv, value;
3178   int i, rv = 0, match = 0;
3179
3180   if (!nm->enabled)
3181     {
3182       return;
3183     }
3184
3185   for (i = 0; i < vec_len (nm->to_resolve); i++)
3186     {
3187       rp = nm->to_resolve + i;
3188
3189       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3190         {
3191           match = 1;
3192           break;
3193         }
3194     }
3195
3196   if (!match)
3197     {
3198       return;
3199     }
3200
3201   init_nat_k (&kv, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port,
3202               nm->outside_fib_index,
3203               is_sm_addr_only (rp->flags) ? 0 : rp->proto);
3204   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
3205     m = 0;
3206   else
3207     m = pool_elt_at_index (nm->static_mappings, value.value);
3208
3209   if (is_delete)
3210     {
3211       if (!m)
3212         return;
3213       rv = nat44_ei_del_static_mapping (rp->l_addr, address[0], rp->l_port,
3214                                         rp->e_port, rp->proto, rp->vrf_id, ~0,
3215                                         rp->flags);
3216       if (rv)
3217         {
3218           nat_elog_notice_X1 (nm, "nat44_ei_del_static_mapping returned %d",
3219                               "i4", rv);
3220         }
3221     }
3222   else
3223     {
3224       if (m)
3225         return;
3226       rv = nat44_ei_add_static_mapping (rp->l_addr, address[0], rp->l_port,
3227                                         rp->e_port, rp->proto, rp->vrf_id, ~0,
3228                                         rp->flags, rp->pool_addr, rp->tag);
3229       if (rv)
3230         {
3231           nat_elog_notice_X1 (nm, "nat44_ei_add_static_mapping returned %d",
3232                               "i4", rv);
3233         }
3234     }
3235 }
3236
3237 static_always_inline uword
3238 nat44_ei_classify_inline_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
3239                              vlib_frame_t *frame)
3240 {
3241   u32 n_left_from, *from, *to_next;
3242   nat44_ei_classify_next_t next_index;
3243   nat44_ei_main_t *nm = &nat44_ei_main;
3244   nat44_ei_static_mapping_t *m;
3245   u32 next_in2out = 0, next_out2in = 0;
3246
3247   from = vlib_frame_vector_args (frame);
3248   n_left_from = frame->n_vectors;
3249   next_index = node->cached_next_index;
3250
3251   while (n_left_from > 0)
3252     {
3253       u32 n_left_to_next;
3254
3255       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3256
3257       while (n_left_from > 0 && n_left_to_next > 0)
3258         {
3259           u32 bi0;
3260           vlib_buffer_t *b0;
3261           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3262           ip4_header_t *ip0;
3263           nat44_ei_address_t *ap;
3264           clib_bihash_kv_8_8_t kv0, value0;
3265
3266           /* speculatively enqueue b0 to the current next frame */
3267           bi0 = from[0];
3268           to_next[0] = bi0;
3269           from += 1;
3270           to_next += 1;
3271           n_left_from -= 1;
3272           n_left_to_next -= 1;
3273
3274           b0 = vlib_get_buffer (vm, bi0);
3275           ip0 = vlib_buffer_get_current (b0);
3276
3277           vec_foreach (ap, nm->addresses)
3278             {
3279               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
3280                 {
3281                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3282                   goto enqueue0;
3283                 }
3284             }
3285
3286           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
3287             {
3288               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
3289               /* try to classify the fragment based on IP header alone */
3290               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3291                                            &kv0, &value0))
3292                 {
3293                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3294                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3295                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3296                   goto enqueue0;
3297                 }
3298               init_nat_k (&kv0, ip0->dst_address,
3299                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
3300                           ip_proto_to_nat_proto (ip0->protocol));
3301               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3302                                            &kv0, &value0))
3303                 {
3304                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3305                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3306                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3307                 }
3308             }
3309
3310         enqueue0:
3311           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
3312                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
3313             {
3314               nat44_ei_classify_trace_t *t =
3315                 vlib_add_trace (vm, node, b0, sizeof (*t));
3316               t->cached = 0;
3317               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
3318             }
3319
3320           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3321           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3322
3323           /* verify speculative enqueue, maybe switch current next frame */
3324           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3325                                            n_left_to_next, bi0, next0);
3326         }
3327
3328       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3329     }
3330
3331   vlib_node_increment_counter (
3332     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
3333   vlib_node_increment_counter (
3334     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3335   return frame->n_vectors;
3336 }
3337
3338 VLIB_NODE_FN (nat44_ei_classify_node)
3339 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3340 {
3341   return nat44_ei_classify_inline_fn (vm, node, frame);
3342 }
3343
3344 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3345   .name = "nat44-ei-classify",
3346   .vector_size = sizeof (u32),
3347   .format_trace = format_nat44_ei_classify_trace,
3348   .type = VLIB_NODE_TYPE_INTERNAL,
3349   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3350   .error_strings = nat44_ei_classify_error_strings,
3351   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3352   .next_nodes = {
3353     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3354     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3355     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3356   },
3357 };
3358
3359 VLIB_NODE_FN (nat44_ei_handoff_classify_node)
3360 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3361 {
3362   return nat44_ei_classify_inline_fn (vm, node, frame);
3363 }
3364
3365 VLIB_REGISTER_NODE (nat44_ei_handoff_classify_node) = {
3366   .name = "nat44-ei-handoff-classify",
3367   .vector_size = sizeof (u32),
3368   .format_trace = format_nat44_ei_classify_trace,
3369   .type = VLIB_NODE_TYPE_INTERNAL,
3370   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3371   .error_strings = nat44_ei_classify_error_strings,
3372   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3373   .next_nodes = {
3374     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out-worker-handoff",
3375     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in-worker-handoff",
3376     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3377   },
3378 };
3379
3380 /*
3381  * fd.io coding-style-patch-verification: ON
3382  *
3383  * Local Variables:
3384  * eval: (c-set-style "gnu")
3385  * End:
3386  */