src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t nat44_ei_hairpin_dst_node;
  44 extern vlib_node_registration_t
  45   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  46 extern vlib_node_registration_t
  47   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  48
  49 #define skip_if_disabled()                                                    \
  50   do                                                                          \
  51     {                                                                         \
  52       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  53       if (PREDICT_FALSE (!nm->enabled))                                       \
  54         return;                                                               \
  55     }                                                                         \
  56   while (0)
  57
  58 #define fail_if_enabled()                                                     \
  59   do                                                                          \
  60     {                                                                         \
  61       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  62       if (PREDICT_FALSE (nm->enabled))                                        \
  63         {                                                                     \
  64           nat44_ei_log_err ("plugin enabled");                                \
  65           return 1;                                                           \
  66         }                                                                     \
  67     }                                                                         \
  68   while (0)
  69
  70 #define fail_if_disabled()                                                    \
  71   do                                                                          \
  72     {                                                                         \
  73       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  74       if (PREDICT_FALSE (!nm->enabled))                                       \
  75         {                                                                     \
  76           nat44_ei_log_err ("plugin disabled");                               \
  77           return 1;                                                           \
  78         }                                                                     \
  79     }                                                                         \
  80   while (0)
  81
  82 /* Hook up input features */
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ei-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  87                                "ip4-sv-reassembly-feature"),
  88 };
  89 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-ei-handoff-classify",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  93                                "ip4-sv-reassembly-feature"),
  94 };
  95 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  96   .arc_name = "ip4-unicast",
  97   .node_name = "nat44-ei-in2out",
  98   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  99                                "ip4-sv-reassembly-feature"),
 100 };
 101 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
 102   .arc_name = "ip4-unicast",
 103   .node_name = "nat44-ei-out2in",
 104   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 105                                "ip4-sv-reassembly-feature",
 106                                "ip4-dhcp-client-detect"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 109   .arc_name = "ip4-output",
 110   .node_name = "nat44-ei-in2out-output",
 111   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 112                                "ip4-sv-reassembly-output-feature"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_fast, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ei-in2out-fast",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 118                                "ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_fast, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ei-out2in-fast",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 124                                "ip4-sv-reassembly-feature",
 125                                "ip4-dhcp-client-detect"),
 126 };
 127 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_dst, static) = {
 128   .arc_name = "ip4-unicast",
 129   .node_name = "nat44-ei-hairpin-dst",
 130   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 131                                "ip4-sv-reassembly-feature"),
 132 };
 133 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_src, static) = {
 134   .arc_name = "ip4-output",
 135   .node_name = "nat44-ei-hairpin-src",
 136   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 137                                "ip4-sv-reassembly-output-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 140   .arc_name = "ip4-local",
 141   .node_name = "nat44-ei-hairpinning",
 142   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 143 };
 144 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 145   .arc_name = "ip4-unicast",
 146   .node_name = "nat44-ei-in2out-worker-handoff",
 147   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 148 };
 149 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 150   .arc_name = "ip4-unicast",
 151   .node_name = "nat44-ei-out2in-worker-handoff",
 152   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 153                                "ip4-dhcp-client-detect"),
 154 };
 155 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 156   .arc_name = "ip4-output",
 157   .node_name = "nat44-ei-in2out-output-worker-handoff",
 158   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 159                                "ip4-sv-reassembly-output-feature"),
 160 };
 161
 162 VLIB_PLUGIN_REGISTER () = {
 163   .version = VPP_BUILD_VER,
 164   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 165 };
 166
 167 #define foreach_nat44_ei_classify_error                                       \
 168   _ (NEXT_IN2OUT, "next in2out")                                              \
 169   _ (NEXT_OUT2IN, "next out2in")                                              \
 170   _ (FRAG_CACHED, "fragment cached")
 171
 172 typedef enum
 173 {
 174 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 175   foreach_nat44_ei_classify_error
 176 #undef _
 177     NAT44_EI_CLASSIFY_N_ERROR,
 178 } nat44_ei_classify_error_t;
 179
 180 static char *nat44_ei_classify_error_strings[] = {
 181 #define _(sym, string) string,
 182   foreach_nat44_ei_classify_error
 183 #undef _
 184 };
 185
 186 typedef enum
 187 {
 188   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 189   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 190   NAT44_EI_CLASSIFY_NEXT_DROP,
 191   NAT44_EI_CLASSIFY_N_NEXT,
 192 } nat44_ei_classify_next_t;
 193
 194 typedef struct
 195 {
 196   u8 next_in2out;
 197   u8 cached;
 198 } nat44_ei_classify_trace_t;
 199
 200 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 201                                    u32 sw_if_index, int is_add);
 202
 203 static u8 *
 204 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 205 {
 206   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 207   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 208   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 209   char *next;
 210
 211   if (t->cached)
 212     s = format (s, "nat44-ei-classify: fragment cached");
 213   else
 214     {
 215       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 216       s = format (s, "nat44-ei-classify: next %s", next);
 217     }
 218
 219   return s;
 220 }
 221
 222 static void nat44_ei_db_free ();
 223
 224 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 225                               u32 user_buckets);
 226
 227 static void nat44_ei_ip4_add_del_interface_address_cb (
 228   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 229   u32 address_length, u32 if_address_index, u32 is_delete);
 230
 231 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 232   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 233   u32 address_length, u32 if_address_index, u32 is_delete);
 234
 235 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 236                                          u32 sw_if_index, u32 new_fib_index,
 237                                          u32 old_fib_index);
 238
 239 void
 240 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 241 {
 242   vlib_node_t *node;
 243   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 244   nm->out2in_node_index = node->index;
 245   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 246   nm->in2out_node_index = node->index;
 247   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 248   nm->in2out_output_node_index = node->index;
 249 }
 250
 251 int
 252 nat44_ei_set_workers (uword *bitmap)
 253 {
 254   nat44_ei_main_t *nm = &nat44_ei_main;
 255   int i, j = 0;
 256
 257   if (nm->num_workers < 2)
 258     return VNET_API_ERROR_FEATURE_DISABLED;
 259
 260   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 261     return VNET_API_ERROR_INVALID_WORKER;
 262
 263   vec_free (nm->workers);
 264   clib_bitmap_foreach (i, bitmap)
 265     {
 266       vec_add1 (nm->workers, i);
 267       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 268       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 269       j++;
 270     }
 271
 272   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 273
 274   return 0;
 275 }
 276
 277 #define nat_validate_simple_counter(c, i)                                     \
 278   do                                                                          \
 279     {                                                                         \
 280       vlib_validate_simple_counter (&c, i);                                   \
 281       vlib_zero_simple_counter (&c, i);                                       \
 282     }                                                                         \
 283   while (0);
 284
 285 #define nat_init_simple_counter(c, n, sn)                                     \
 286   do                                                                          \
 287     {                                                                         \
 288       c.name = n;                                                             \
 289       c.stat_segment_name = sn;                                               \
 290       nat_validate_simple_counter (c, 0);                                     \
 291     }                                                                         \
 292   while (0);
 293
 294 static_always_inline void
 295 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 296 {
 297 #define _(x)                                                                  \
 298   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 299   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 300   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 301   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 302   foreach_nat_counter;
 303 #undef _
 304   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 305 }
 306
 307 static void
 308 nat44_ei_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 309 {
 310   nat44_ei_main_t *nm = &nat44_ei_main;
 311   nat44_ei_interface_t *i;
 312
 313   pool_foreach (i, nm->interfaces)
 314     {
 315       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 316         {
 317           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 318         }
 319     }
 320   pool_foreach (i, nm->output_feature_interfaces)
 321     {
 322       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 323         {
 324           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 325         }
 326     }
 327 }
 328
 329 static_always_inline void
 330 nat44_ei_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 331 {
 332   nat44_ei_main_t *nm = &nat44_ei_main;
 333   nat44_ei_address_t *ap;
 334
 335   vec_foreach (ap, nm->addresses)
 336     {
 337       nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 338     }
 339 }
 340
 341 static_always_inline void
 342 nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 343 {
 344   nat44_ei_main_t *nm = &nat44_ei_main;
 345   nat44_ei_static_mapping_t *m;
 346
 347   pool_foreach (m, nm->static_mappings)
 348     {
 349       if (is_sm_addr_only (m->flags) &&
 350           !(m->local_addr.as_u32 == m->external_addr.as_u32))
 351         {
 352           nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 353                                         is_add);
 354         }
 355     }
 356 }
 357
 358 static int
 359 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
 360 {
 361   nat44_ei_main_t *nm = &nat44_ei_main;
 362   nat44_ei_static_mapping_t *m;
 363   pool_foreach (m, nm->static_mappings)
 364     {
 365       if (is_sm_addr_only (m->flags) || is_sm_identity_nat (m->flags))
 366         {
 367           continue;
 368         }
 369       if (m->external_addr.as_u32 == addr.as_u32)
 370         {
 371           return 1;
 372         }
 373     }
 374   return 0;
 375 }
 376
 377 clib_error_t *
 378 nat44_ei_init (vlib_main_t *vm)
 379 {
 380   nat44_ei_main_t *nm = &nat44_ei_main;
 381   vlib_thread_main_t *tm = vlib_get_thread_main ();
 382   vlib_thread_registration_t *tr;
 383   ip4_add_del_interface_address_callback_t cbi = { 0 };
 384   ip4_table_bind_callback_t cbt = { 0 };
 385   u32 i, num_threads = 0;
 386   uword *p, *bitmap = 0;
 387
 388   clib_memset (nm, 0, sizeof (*nm));
 389
 390   // required
 391   nm->vnet_main = vnet_get_main ();
 392   // convenience
 393   nm->ip4_main = &ip4_main;
 394   nm->api_main = vlibapi_get_main ();
 395   nm->ip4_lookup_main = &ip4_main.lookup_main;
 396
 397   // handoff stuff
 398   nm->fq_out2in_index = ~0;
 399   nm->fq_in2out_index = ~0;
 400   nm->fq_in2out_output_index = ~0;
 401
 402   nm->log_level = NAT_LOG_ERROR;
 403
 404   nat44_ei_set_node_indexes (nm, vm);
 405   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 406
 407   nat_init_simple_counter (nm->total_users, "total-users",
 408                            "/nat44-ei/total-users");
 409   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 410                            "/nat44-ei/total-sessions");
 411   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 412                            "/nat44-ei/user-limit-reached");
 413
 414 #define _(x)                                                                  \
 415   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 416                            "/nat44-ei/in2out/fastpath/" #x);                  \
 417   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 418                            "/nat44-ei/out2in/fastpath/" #x);                  \
 419   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 420                            "/nat44-ei/in2out/slowpath/" #x);                  \
 421   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 422                            "/nat44-ei/out2in/slowpath/" #x);
 423   foreach_nat_counter;
 424 #undef _
 425   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 426                            "/nat44-ei/hairpinning");
 427
 428   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 429   if (p)
 430     {
 431       tr = (vlib_thread_registration_t *) p[0];
 432       if (tr)
 433         {
 434           nm->num_workers = tr->count;
 435           nm->first_worker_index = tr->first_index;
 436         }
 437     }
 438   num_threads = tm->n_vlib_mains - 1;
 439   nm->port_per_thread = 0xffff - 1024;
 440   vec_validate (nm->per_thread_data, num_threads);
 441
 442   /* Use all available workers by default */
 443   if (nm->num_workers > 1)
 444     {
 445       for (i = 0; i < nm->num_workers; i++)
 446         bitmap = clib_bitmap_set (bitmap, i, 1);
 447       nat44_ei_set_workers (bitmap);
 448       clib_bitmap_free (bitmap);
 449     }
 450   else
 451     {
 452       nm->per_thread_data[0].snat_thread_index = 0;
 453     }
 454
 455   /* callbacks to call when interface address changes. */
 456   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 457   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 458   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 459   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 460
 461   /* callbacks to call when interface to table biding changes */
 462   cbt.function = nat44_ei_update_outside_fib;
 463   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 464
 465   nm->fib_src_low = fib_source_allocate (
 466     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 467   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 468                                         FIB_SOURCE_BH_SIMPLE);
 469
 470   // used only by out2in-dpo feature
 471   nat_dpo_module_init ();
 472   nat_ha_init (vm, nm->num_workers, num_threads);
 473
 474   nm->hairpinning_fq_index =
 475     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 476   nm->hairpin_dst_fq_index =
 477     vlib_frame_queue_main_init (nat44_ei_hairpin_dst_node.index, 0);
 478   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 479     vlib_frame_queue_main_init (
 480       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 481   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 482     vlib_frame_queue_main_init (
 483       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 484   return nat44_ei_api_hookup (vm);
 485 }
 486
 487 VLIB_INIT_FUNCTION (nat44_ei_init);
 488
 489 int
 490 nat44_ei_plugin_enable (nat44_ei_config_t c)
 491 {
 492   nat44_ei_main_t *nm = &nat44_ei_main;
 493
 494   fail_if_enabled ();
 495
 496   if (!c.users)
 497     c.users = 1024;
 498
 499   if (!c.sessions)
 500     c.sessions = 10 * 1024;
 501
 502   if (!c.user_sessions)
 503     c.user_sessions = c.sessions;
 504
 505   nm->rconfig = c;
 506
 507   if (!nm->frame_queue_nelts)
 508     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 509
 510   nm->translations = c.sessions;
 511   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 512   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 513
 514   nm->pat = (!c.static_mapping_only ||
 515              (c.static_mapping_only && c.connection_tracking));
 516
 517   nm->static_mapping_only = c.static_mapping_only;
 518   nm->static_mapping_connection_tracking = c.connection_tracking;
 519   nm->out2in_dpo = c.out2in_dpo;
 520   nm->forwarding_enabled = 0;
 521   nm->mss_clamping = 0;
 522
 523   nm->max_users_per_thread = c.users;
 524   nm->max_translations_per_thread = c.sessions;
 525   nm->max_translations_per_user = c.user_sessions;
 526
 527   nm->inside_vrf_id = c.inside_vrf;
 528   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 529     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 530
 531   nm->outside_vrf_id = c.outside_vrf;
 532   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 533     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 534
 535   nat_reset_timeouts (&nm->timeouts);
 536   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 537                     nm->user_buckets);
 538   nat44_ei_set_alloc_default ();
 539
 540   // TODO: zero simple counter for all counters missing
 541
 542   vlib_zero_simple_counter (&nm->total_users, 0);
 543   vlib_zero_simple_counter (&nm->total_sessions, 0);
 544   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 545
 546   if (nm->num_workers > 1)
 547     {
 548       if (nm->fq_in2out_index == ~0)
 549         {
 550           nm->fq_in2out_index = vlib_frame_queue_main_init (
 551             nm->in2out_node_index, nm->frame_queue_nelts);
 552         }
 553       if (nm->fq_out2in_index == ~0)
 554         {
 555           nm->fq_out2in_index = vlib_frame_queue_main_init (
 556             nm->out2in_node_index, nm->frame_queue_nelts);
 557         }
 558       if (nm->fq_in2out_output_index == ~0)
 559         {
 560           nm->fq_in2out_output_index = vlib_frame_queue_main_init (
 561             nm->in2out_output_node_index, nm->frame_queue_nelts);
 562         }
 563     }
 564
 565   nat_ha_enable ();
 566   nm->enabled = 1;
 567
 568   return 0;
 569 }
 570
 571 void
 572 nat44_ei_addresses_free (nat44_ei_address_t **addresses)
 573 {
 574   nat44_ei_address_t *ap;
 575   vec_foreach (ap, *addresses)
 576     {
 577 #define _(N, i, n, s) vec_free (ap->busy_##n##_ports_per_thread);
 578       foreach_nat_protocol
 579 #undef _
 580     }
 581   vec_free (*addresses);
 582   *addresses = 0;
 583 }
 584
 585 static_always_inline nat44_ei_outside_fib_t *
 586 nat44_ei_get_outside_fib (nat44_ei_outside_fib_t *outside_fibs, u32 fib_index)
 587 {
 588   nat44_ei_outside_fib_t *f;
 589   vec_foreach (f, outside_fibs)
 590     {
 591       if (f->fib_index == fib_index)
 592         {
 593           return f;
 594         }
 595     }
 596   return 0;
 597 }
 598
 599 static_always_inline nat44_ei_interface_t *
 600 nat44_ei_get_interface (nat44_ei_interface_t *interfaces, u32 sw_if_index)
 601 {
 602   nat44_ei_interface_t *i;
 603   pool_foreach (i, interfaces)
 604     {
 605       if (i->sw_if_index == sw_if_index)
 606         {
 607           return i;
 608         }
 609     }
 610   return 0;
 611 }
 612
 613 static_always_inline int
 614 nat44_ei_add_interface (u32 sw_if_index, u8 is_inside)
 615 {
 616   const char *feature_name, *del_feature_name;
 617   nat44_ei_main_t *nm = &nat44_ei_main;
 618
 619   nat44_ei_outside_fib_t *outside_fib;
 620   nat44_ei_interface_t *i;
 621   u32 fib_index;
 622   int rv;
 623
 624   fail_if_disabled ();
 625
 626   if (nm->out2in_dpo && !is_inside)
 627     {
 628       nat44_ei_log_err ("error unsupported");
 629       return VNET_API_ERROR_UNSUPPORTED;
 630     }
 631
 632   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 633     {
 634       nat44_ei_log_err ("error interface already configured");
 635       return VNET_API_ERROR_VALUE_EXIST;
 636     }
 637
 638   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 639   if (i)
 640     {
 641       if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 642           (nat44_ei_interface_is_outside (i) && !is_inside))
 643         {
 644           return 0;
 645         }
 646       if (nm->num_workers > 1)
 647         {
 648           del_feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 649                                           "nat44-ei-out2in-worker-handoff";
 650           feature_name = "nat44-ei-handoff-classify";
 651         }
 652       else
 653         {
 654           del_feature_name =
 655             !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 656
 657           feature_name = "nat44-ei-classify";
 658         }
 659
 660       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 661       if (rv)
 662         {
 663           return rv;
 664         }
 665       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 666                                         sw_if_index, 0, 0, 0);
 667       if (rv)
 668         {
 669           return rv;
 670         }
 671       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 672                                         sw_if_index, 1, 0, 0);
 673       if (rv)
 674         {
 675           return rv;
 676         }
 677       if (!is_inside)
 678         {
 679           rv = vnet_feature_enable_disable (
 680             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 0, 0, 0);
 681           if (rv)
 682             {
 683               return rv;
 684             }
 685         }
 686     }
 687   else
 688     {
 689       if (nm->num_workers > 1)
 690         {
 691           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 692                                      "nat44-ei-out2in-worker-handoff";
 693         }
 694       else
 695         {
 696           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 697         }
 698       nat_validate_interface_counters (nm, sw_if_index);
 699
 700       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 701       if (rv)
 702         {
 703           return rv;
 704         }
 705       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 706                                         sw_if_index, 1, 0, 0);
 707       if (rv)
 708         {
 709           return rv;
 710         }
 711       if (is_inside && !nm->out2in_dpo)
 712         {
 713           rv = vnet_feature_enable_disable (
 714             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 1, 0, 0);
 715           if (rv)
 716             {
 717               return rv;
 718             }
 719         }
 720
 721       pool_get (nm->interfaces, i);
 722       i->sw_if_index = sw_if_index;
 723       i->flags = 0;
 724     }
 725
 726   fib_index =
 727     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 728
 729   if (!is_inside)
 730     {
 731       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 732
 733       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 734       if (outside_fib)
 735         {
 736           outside_fib->refcount++;
 737         }
 738       else
 739         {
 740           vec_add2 (nm->outside_fibs, outside_fib, 1);
 741           outside_fib->fib_index = fib_index;
 742           outside_fib->refcount = 1;
 743         }
 744
 745       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 746       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 747     }
 748   else
 749     {
 750       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 751     }
 752
 753   return 0;
 754 }
 755
 756 static_always_inline int
 757 nat44_ei_del_interface (u32 sw_if_index, u8 is_inside)
 758 {
 759   const char *feature_name, *del_feature_name;
 760   nat44_ei_main_t *nm = &nat44_ei_main;
 761
 762   nat44_ei_outside_fib_t *outside_fib;
 763   nat44_ei_interface_t *i;
 764   u32 fib_index;
 765   int rv;
 766
 767   fail_if_disabled ();
 768
 769   if (nm->out2in_dpo && !is_inside)
 770     {
 771       nat44_ei_log_err ("error unsupported");
 772       return VNET_API_ERROR_UNSUPPORTED;
 773     }
 774
 775   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 776   if (i == 0)
 777     {
 778       nat44_ei_log_err ("error interface couldn't be found");
 779       return VNET_API_ERROR_NO_SUCH_ENTRY;
 780     }
 781
 782   if (nat44_ei_interface_is_inside (i) && nat44_ei_interface_is_outside (i))
 783     {
 784       if (nm->num_workers > 1)
 785         {
 786           del_feature_name = "nat44-ei-handoff-classify";
 787           feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 788                                       "nat44-ei-out2in-worker-handoff";
 789         }
 790       else
 791         {
 792           del_feature_name = "nat44-ei-classify";
 793           feature_name = !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 794         }
 795
 796       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 797       if (rv)
 798         {
 799           return rv;
 800         }
 801       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 802                                         sw_if_index, 0, 0, 0);
 803       if (rv)
 804         {
 805           return rv;
 806         }
 807       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 808                                         sw_if_index, 1, 0, 0);
 809       if (rv)
 810         {
 811           return rv;
 812         }
 813       if (is_inside)
 814         {
 815           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 816         }
 817       else
 818         {
 819           rv = vnet_feature_enable_disable (
 820             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 1, 0, 0);
 821           if (rv)
 822             {
 823               return rv;
 824             }
 825           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 826         }
 827     }
 828   else
 829     {
 830       if (nm->num_workers > 1)
 831         {
 832           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 833                                      "nat44-ei-out2in-worker-handoff";
 834         }
 835       else
 836         {
 837           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 838         }
 839
 840       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 841       if (rv)
 842         {
 843           return rv;
 844         }
 845       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 846                                         sw_if_index, 0, 0, 0);
 847       if (rv)
 848         {
 849           return rv;
 850         }
 851       if (is_inside)
 852         {
 853           rv = vnet_feature_enable_disable (
 854             "ip4-local", "nat44-ei-hairpinning", sw_if_index, 0, 0, 0);
 855           if (rv)
 856             {
 857               return rv;
 858             }
 859         }
 860
 861       // remove interface
 862       pool_put (nm->interfaces, i);
 863     }
 864
 865   if (!is_inside)
 866     {
 867       fib_index =
 868         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 869       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 870       if (outside_fib)
 871         {
 872           outside_fib->refcount--;
 873           if (!outside_fib->refcount)
 874             {
 875               vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
 876             }
 877         }
 878
 879       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
 880       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
 881     }
 882
 883   return 0;
 884 }
 885
 886 int
 887 nat44_ei_add_del_interface (u32 sw_if_index, u8 is_inside, int is_del)
 888 {
 889   if (is_del)
 890     {
 891       return nat44_ei_del_interface (sw_if_index, is_inside);
 892     }
 893   else
 894     {
 895       return nat44_ei_add_interface (sw_if_index, is_inside);
 896     }
 897 }
 898
 899 static_always_inline int
 900 nat44_ei_add_output_interface (u32 sw_if_index)
 901 {
 902   nat44_ei_main_t *nm = &nat44_ei_main;
 903
 904   nat44_ei_outside_fib_t *outside_fib;
 905   nat44_ei_interface_t *i;
 906   u32 fib_index;
 907   int rv;
 908
 909   fail_if_disabled ();
 910
 911   if (nat44_ei_get_interface (nm->interfaces, sw_if_index))
 912     {
 913       nat44_ei_log_err ("error interface already configured");
 914       return VNET_API_ERROR_VALUE_EXIST;
 915     }
 916
 917   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 918     {
 919       nat44_ei_log_err ("error interface already configured");
 920       return VNET_API_ERROR_VALUE_EXIST;
 921     }
 922
 923   if (nm->num_workers > 1)
 924     {
 925       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 926       if (rv)
 927         {
 928           return rv;
 929         }
 930       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 931       if (rv)
 932         {
 933           return rv;
 934         }
 935       rv = vnet_feature_enable_disable (
 936         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 1, 0, 0);
 937       if (rv)
 938         {
 939           return rv;
 940         }
 941       rv = vnet_feature_enable_disable (
 942         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 1,
 943         0, 0);
 944       if (rv)
 945         {
 946           return rv;
 947         }
 948     }
 949   else
 950     {
 951       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 952       if (rv)
 953         {
 954           return rv;
 955         }
 956       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 957       if (rv)
 958         {
 959           return rv;
 960         }
 961       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 962                                         sw_if_index, 1, 0, 0);
 963       if (rv)
 964         {
 965           return rv;
 966         }
 967       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 968                                         sw_if_index, 1, 0, 0);
 969       if (rv)
 970         {
 971           return rv;
 972         }
 973     }
 974
 975   nat_validate_interface_counters (nm, sw_if_index);
 976
 977   pool_get (nm->output_feature_interfaces, i);
 978   i->sw_if_index = sw_if_index;
 979   i->flags = 0;
 980   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 981   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 982
 983   fib_index =
 984     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 985   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 986   if (outside_fib)
 987     {
 988       outside_fib->refcount++;
 989     }
 990   else
 991     {
 992       vec_add2 (nm->outside_fibs, outside_fib, 1);
 993       outside_fib->fib_index = fib_index;
 994       outside_fib->refcount = 1;
 995     }
 996
 997   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 998   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 999
1000   return 0;
1001 }
1002
1003 static_always_inline int
1004 nat44_ei_del_output_interface (u32 sw_if_index)
1005 {
1006   nat44_ei_main_t *nm = &nat44_ei_main;
1007
1008   nat44_ei_outside_fib_t *outside_fib;
1009   nat44_ei_interface_t *i;
1010   u32 fib_index;
1011   int rv;
1012
1013   fail_if_disabled ();
1014
1015   i = nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index);
1016   if (!i)
1017     {
1018       nat44_ei_log_err ("error interface couldn't be found");
1019       return VNET_API_ERROR_NO_SUCH_ENTRY;
1020     }
1021
1022   if (nm->num_workers > 1)
1023     {
1024       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1025       if (rv)
1026         {
1027           return rv;
1028         }
1029       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1030       if (rv)
1031         {
1032           return rv;
1033         }
1034       rv = vnet_feature_enable_disable (
1035         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 0, 0, 0);
1036       if (rv)
1037         {
1038           return rv;
1039         }
1040       rv = vnet_feature_enable_disable (
1041         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 0,
1042         0, 0);
1043       if (rv)
1044         {
1045           return rv;
1046         }
1047     }
1048   else
1049     {
1050       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1051       if (rv)
1052         {
1053           return rv;
1054         }
1055       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1056       if (rv)
1057         {
1058           return rv;
1059         }
1060       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1061                                         sw_if_index, 0, 0, 0);
1062       if (rv)
1063         {
1064           return rv;
1065         }
1066       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1067                                         sw_if_index, 0, 0, 0);
1068       if (rv)
1069         {
1070           return rv;
1071         }
1072     }
1073
1074   pool_put (nm->output_feature_interfaces, i);
1075
1076   fib_index =
1077     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1078   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1079   if (outside_fib)
1080     {
1081       outside_fib->refcount--;
1082       if (!outside_fib->refcount)
1083         {
1084           vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
1085         }
1086     }
1087
1088   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1089   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1090
1091   return 0;
1092 }
1093
1094 int
1095 nat44_ei_add_del_output_interface (u32 sw_if_index, int is_del)
1096 {
1097   if (is_del)
1098     {
1099       return nat44_ei_del_output_interface (sw_if_index);
1100     }
1101   else
1102     {
1103       return nat44_ei_add_output_interface (sw_if_index);
1104     }
1105 }
1106
1107 int
1108 nat44_ei_plugin_disable ()
1109 {
1110   nat44_ei_main_t *nm = &nat44_ei_main;
1111   nat44_ei_interface_t *i, *pool;
1112   int error = 0;
1113
1114   // first unregister all nodes from interfaces
1115   pool = pool_dup (nm->interfaces);
1116   pool_foreach (i, pool)
1117     {
1118       if (nat44_ei_interface_is_inside (i))
1119         {
1120           error = nat44_ei_del_interface (i->sw_if_index, 1);
1121         }
1122       if (nat44_ei_interface_is_outside (i))
1123         {
1124           error = nat44_ei_del_interface (i->sw_if_index, 0);
1125         }
1126
1127       if (error)
1128         {
1129           nat44_ei_log_err ("error occurred while removing interface %u",
1130                             i->sw_if_index);
1131         }
1132     }
1133   pool_free (pool);
1134   pool_free (nm->interfaces);
1135
1136   pool = pool_dup (nm->output_feature_interfaces);
1137   pool_foreach (i, pool)
1138     {
1139       error = nat44_ei_del_output_interface (i->sw_if_index);
1140       if (error)
1141         {
1142           nat44_ei_log_err ("error occurred while removing interface %u",
1143                             i->sw_if_index);
1144         }
1145     }
1146   pool_free (pool);
1147   pool_free (nm->output_feature_interfaces);
1148
1149   nat_ha_disable ();
1150   nat44_ei_db_free ();
1151
1152   nat44_ei_addresses_free (&nm->addresses);
1153
1154   vec_free (nm->to_resolve);
1155   vec_free (nm->auto_add_sw_if_indices);
1156
1157   nm->to_resolve = 0;
1158   nm->auto_add_sw_if_indices = 0;
1159
1160   nm->forwarding_enabled = 0;
1161
1162   nm->enabled = 0;
1163   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
1164
1165   return 0;
1166 }
1167
1168 int
1169 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
1170                                        u32 thread_index, ip4_address_t addr,
1171                                        u16 port, nat_protocol_t protocol)
1172 {
1173   nat44_ei_main_t *nm = &nat44_ei_main;
1174   nat44_ei_address_t *a = 0;
1175   u32 address_index;
1176   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1177
1178   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1179     {
1180       if (addresses[address_index].addr.as_u32 != addr.as_u32)
1181         continue;
1182
1183       a = addresses + address_index;
1184       switch (protocol)
1185         {
1186 #define _(N, j, n, s)                                                         \
1187   case NAT_PROTOCOL_##N:                                                      \
1188     if (a->busy_##n##_port_refcounts[port_host_byte_order])                   \
1189       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
1190     ++a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1191     a->busy_##n##_ports_per_thread[thread_index]++;                           \
1192     a->busy_##n##_ports++;                                                    \
1193     return 0;
1194           foreach_nat_protocol
1195 #undef _
1196             default : nat_elog_info (nm, "unknown protocol");
1197           return 1;
1198         }
1199     }
1200
1201   return VNET_API_ERROR_NO_SUCH_ENTRY;
1202 }
1203
1204 void
1205 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
1206 {
1207   nat44_ei_main_t *nm = &nat44_ei_main;
1208   dpo_id_t dpo_v4 = DPO_INVALID;
1209   fib_prefix_t pfx = {
1210     .fp_proto = FIB_PROTOCOL_IP4,
1211     .fp_len = 32,
1212     .fp_addr.ip4.as_u32 = addr.as_u32,
1213   };
1214
1215   if (is_add)
1216     {
1217       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
1218       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
1219                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
1220       dpo_reset (&dpo_v4);
1221     }
1222   else
1223     {
1224       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
1225     }
1226 }
1227
1228 void
1229 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1230                                         u32 thread_index, ip4_address_t *addr,
1231                                         u16 port, nat_protocol_t protocol)
1232 {
1233   nat44_ei_main_t *nm = &nat44_ei_main;
1234   nat44_ei_address_t *a;
1235   u32 address_index;
1236   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1237
1238   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1239     {
1240       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1241         break;
1242     }
1243
1244   ASSERT (address_index < vec_len (addresses));
1245
1246   a = addresses + address_index;
1247
1248   switch (protocol)
1249     {
1250 #define _(N, i, n, s)                                                         \
1251   case NAT_PROTOCOL_##N:                                                      \
1252     ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1);         \
1253     --a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1254     a->busy_##n##_ports--;                                                    \
1255     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1256     break;
1257       foreach_nat_protocol
1258 #undef _
1259         default : nat_elog_info (nm, "unknown protocol");
1260       return;
1261     }
1262 }
1263
1264 void
1265 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1266                                u32 thread_index, u8 is_ha)
1267 {
1268   clib_bihash_kv_8_8_t kv;
1269
1270   /* session lookup tables */
1271   init_nat_i2o_k (&kv, s);
1272   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1273     nat_elog_warn (nm, "in2out key del failed");
1274   init_nat_o2i_k (&kv, s);
1275   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1276     nat_elog_warn (nm, "out2in key del failed");
1277
1278   if (!is_ha)
1279     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1280                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1281                              s->out2in.port, s->nat_proto);
1282
1283   if (nat44_ei_is_unk_proto_session (s))
1284     return;
1285
1286   if (!is_ha)
1287     {
1288       /* log NAT event */
1289       nat_ipfix_logging_nat44_ses_delete (
1290         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1291         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1292         s->in2out.fib_index);
1293
1294       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1295                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1296                    thread_index);
1297     }
1298
1299   if (nat44_ei_is_session_static (s))
1300     return;
1301
1302   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1303                                           &s->out2in.addr, s->out2in.port,
1304                                           s->nat_proto);
1305 }
1306
1307 nat44_ei_user_t *
1308 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1309                              u32 fib_index, u32 thread_index)
1310 {
1311   nat44_ei_user_t *u = 0;
1312   nat44_ei_user_key_t user_key;
1313   clib_bihash_kv_8_8_t kv, value;
1314   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1315   dlist_elt_t *per_user_list_head_elt;
1316
1317   user_key.addr.as_u32 = addr->as_u32;
1318   user_key.fib_index = fib_index;
1319   kv.key = user_key.as_u64;
1320
1321   /* Ever heard of the "user" = src ip4 address before? */
1322   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1323     {
1324       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1325         {
1326           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1327                                          0, 1);
1328           nat_elog_warn (nm, "maximum user limit reached");
1329           return NULL;
1330         }
1331       /* no, make a new one */
1332       pool_get (tnm->users, u);
1333       clib_memset (u, 0, sizeof (*u));
1334
1335       u->addr.as_u32 = addr->as_u32;
1336       u->fib_index = fib_index;
1337
1338       pool_get (tnm->list_pool, per_user_list_head_elt);
1339
1340       u->sessions_per_user_list_head_index =
1341         per_user_list_head_elt - tnm->list_pool;
1342
1343       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1344
1345       kv.value = u - tnm->users;
1346
1347       /* add user */
1348       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1349         {
1350           nat_elog_warn (nm, "user_hash key add failed");
1351           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1352           return NULL;
1353         }
1354
1355       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1356                                pool_elts (tnm->users));
1357     }
1358   else
1359     {
1360       u = pool_elt_at_index (tnm->users, value.value);
1361     }
1362
1363   return u;
1364 }
1365
1366 nat44_ei_session_t *
1367 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1368                                    u32 thread_index, f64 now)
1369 {
1370   nat44_ei_session_t *s;
1371   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1372   u32 oldest_per_user_translation_list_index, session_index;
1373   dlist_elt_t *oldest_per_user_translation_list_elt;
1374   dlist_elt_t *per_user_translation_list_elt;
1375
1376   /* Over quota? Recycle the least recently used translation */
1377   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1378     {
1379       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1380         tnm->list_pool, u->sessions_per_user_list_head_index);
1381
1382       ASSERT (oldest_per_user_translation_list_index != ~0);
1383
1384       /* Add it back to the end of the LRU list */
1385       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1386                           oldest_per_user_translation_list_index);
1387       /* Get the list element */
1388       oldest_per_user_translation_list_elt = pool_elt_at_index (
1389         tnm->list_pool, oldest_per_user_translation_list_index);
1390
1391       /* Get the session index from the list element */
1392       session_index = oldest_per_user_translation_list_elt->value;
1393
1394       /* Get the session */
1395       s = pool_elt_at_index (tnm->sessions, session_index);
1396
1397       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1398       if (nat44_ei_is_session_static (s))
1399         u->nstaticsessions--;
1400       else
1401         u->nsessions--;
1402       s->flags = 0;
1403       s->total_bytes = 0;
1404       s->total_pkts = 0;
1405       s->state = 0;
1406       s->ext_host_addr.as_u32 = 0;
1407       s->ext_host_port = 0;
1408       s->ext_host_nat_addr.as_u32 = 0;
1409       s->ext_host_nat_port = 0;
1410     }
1411   else
1412     {
1413       pool_get (tnm->sessions, s);
1414       clib_memset (s, 0, sizeof (*s));
1415
1416       /* Create list elts */
1417       pool_get (tnm->list_pool, per_user_translation_list_elt);
1418       clib_dlist_init (tnm->list_pool,
1419                        per_user_translation_list_elt - tnm->list_pool);
1420
1421       per_user_translation_list_elt->value = s - tnm->sessions;
1422       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1423       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1424
1425       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1426                           per_user_translation_list_elt - tnm->list_pool);
1427
1428       s->user_index = u - tnm->users;
1429       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1430                                pool_elts (tnm->sessions));
1431     }
1432
1433   s->ha_last_refreshed = now;
1434
1435   return s;
1436 }
1437
1438 void
1439 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1440                             u32 thread_index, u8 is_ha)
1441 {
1442   clib_bihash_kv_8_8_t kv;
1443
1444   init_nat_i2o_k (&kv, s);
1445   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1446     nat_elog_warn (nm, "in2out key del failed");
1447
1448   init_nat_o2i_k (&kv, s);
1449   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1450     nat_elog_warn (nm, "out2in key del failed");
1451
1452   if (!is_ha)
1453     {
1454       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1455                                &s->in2out.addr, s->in2out.port,
1456                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1457
1458       nat_ipfix_logging_nat44_ses_delete (
1459         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1460         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1461         s->in2out.fib_index);
1462
1463       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1464                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1465                    thread_index);
1466     }
1467
1468   if (nat44_ei_is_session_static (s))
1469     return;
1470
1471   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1472                                           &s->out2in.addr, s->out2in.port,
1473                                           s->nat_proto);
1474 }
1475
1476 static_always_inline void
1477 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1478 {
1479   dlist_elt_t *elt;
1480   nat44_ei_session_t *s;
1481
1482   nat44_ei_main_t *nm = &nat44_ei_main;
1483   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1484
1485   // get head
1486   elt =
1487     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1488   // get first element
1489   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1490
1491   while (elt->value != ~0)
1492     {
1493       s = pool_elt_at_index (tnm->sessions, elt->value);
1494       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1495
1496       nat44_ei_free_session_data (nm, s, thread_index, 0);
1497       nat44_ei_delete_session (nm, s, thread_index);
1498     }
1499 }
1500
1501 int
1502 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1503 {
1504   int rv = 1;
1505
1506   nat44_ei_main_t *nm = &nat44_ei_main;
1507   nat44_ei_main_per_thread_data_t *tnm;
1508
1509   nat44_ei_user_key_t user_key;
1510   clib_bihash_kv_8_8_t kv, value;
1511
1512   user_key.addr.as_u32 = addr->as_u32;
1513   user_key.fib_index = fib_index;
1514   kv.key = user_key.as_u64;
1515
1516   if (nm->num_workers > 1)
1517     {
1518       vec_foreach (tnm, nm->per_thread_data)
1519         {
1520           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1521             {
1522               nat44_ei_user_del_sessions (
1523                 pool_elt_at_index (tnm->users, value.value),
1524                 tnm->thread_index);
1525               rv = 0;
1526               break;
1527             }
1528         }
1529     }
1530   else
1531     {
1532       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1533       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1534         {
1535           nat44_ei_user_del_sessions (
1536             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1537           rv = 0;
1538         }
1539     }
1540   return rv;
1541 }
1542
1543 void
1544 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1545                                       nat44_ei_main_per_thread_data_t *tnm,
1546                                       nat44_ei_user_key_t u_key, int addr_only,
1547                                       ip4_address_t e_addr, u16 e_port)
1548 {
1549   clib_bihash_kv_8_8_t kv, value;
1550   kv.key = u_key.as_u64;
1551   u64 user_index;
1552   dlist_elt_t *head, *elt;
1553   nat44_ei_user_t *u;
1554   nat44_ei_session_t *s;
1555   u32 elt_index, head_index, ses_index;
1556
1557   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1558     {
1559       user_index = value.value;
1560       u = pool_elt_at_index (tnm->users, user_index);
1561       if (u->nstaticsessions)
1562         {
1563           head_index = u->sessions_per_user_list_head_index;
1564           head = pool_elt_at_index (tnm->list_pool, head_index);
1565           elt_index = head->next;
1566           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1567           ses_index = elt->value;
1568           while (ses_index != ~0)
1569             {
1570               s = pool_elt_at_index (tnm->sessions, ses_index);
1571               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1572               ses_index = elt->value;
1573
1574               if (!addr_only)
1575                 {
1576                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1577                       (s->out2in.port != e_port))
1578                     continue;
1579                 }
1580
1581               if (!nat44_ei_is_session_static (s))
1582                 continue;
1583
1584               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1585                                              0);
1586               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1587
1588               if (!addr_only)
1589                 break;
1590             }
1591         }
1592     }
1593 }
1594
1595 u32
1596 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1597                                   u8 is_output)
1598 {
1599   nat44_ei_main_t *nm = &nat44_ei_main;
1600   u32 next_worker_index = 0;
1601   u32 hash;
1602
1603   next_worker_index = nm->first_worker_index;
1604   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1605          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1606
1607   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1608     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1609   else
1610     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1611
1612   return next_worker_index;
1613 }
1614
1615 u32
1616 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1617                                   u32 rx_fib_index0, u8 is_output)
1618 {
1619   nat44_ei_main_t *nm = &nat44_ei_main;
1620   udp_header_t *udp;
1621   u16 port;
1622   clib_bihash_kv_8_8_t kv, value;
1623   nat44_ei_static_mapping_t *m;
1624   u32 proto;
1625   u32 next_worker_index = 0;
1626
1627   /* first try static mappings without port */
1628   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1629     {
1630       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1631       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1632                                    &value))
1633         {
1634           m = pool_elt_at_index (nm->static_mappings, value.value);
1635           return m->workers[0];
1636         }
1637     }
1638
1639   proto = ip_proto_to_nat_proto (ip0->protocol);
1640   udp = ip4_next_header (ip0);
1641   port = vnet_buffer (b)->ip.reass.l4_dst_port;
1642
1643   /* unknown protocol */
1644   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1645     {
1646       /* use current thread */
1647       return vlib_get_thread_index ();
1648     }
1649
1650   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1651     {
1652       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1653       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1654       if (!icmp_type_is_error_message (
1655             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1656         port = vnet_buffer (b)->ip.reass.l4_src_port;
1657       else
1658         {
1659           /* if error message, then it's not fragmented and we can access it */
1660           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1661           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1662           void *l4_header = ip4_next_header (inner_ip);
1663           switch (proto)
1664             {
1665             case NAT_PROTOCOL_ICMP:
1666               icmp = (icmp46_header_t *) l4_header;
1667               echo = (icmp_echo_header_t *) (icmp + 1);
1668               port = echo->identifier;
1669               break;
1670             case NAT_PROTOCOL_UDP:
1671             case NAT_PROTOCOL_TCP:
1672               port = ((tcp_udp_header_t *) l4_header)->src_port;
1673               break;
1674             default:
1675               return vlib_get_thread_index ();
1676             }
1677         }
1678     }
1679
1680   /* try static mappings with port */
1681   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1682     {
1683       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1684       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1685                                    &value))
1686         {
1687           m = pool_elt_at_index (nm->static_mappings, value.value);
1688           return m->workers[0];
1689         }
1690     }
1691
1692   /* worker by outside port */
1693   next_worker_index = nm->first_worker_index;
1694   next_worker_index +=
1695     nm->workers[(clib_net_to_host_u16 (port) - 1024) / nm->port_per_thread];
1696   return next_worker_index;
1697 }
1698
1699 static int
1700 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1701                            u32 thread_index, nat_protocol_t proto,
1702                            ip4_address_t s_addr, ip4_address_t *addr,
1703                            u16 *port, u16 port_per_thread,
1704                            u32 snat_thread_index)
1705 {
1706   nat44_ei_main_t *nm = &nat44_ei_main;
1707   nat44_ei_address_t *a, *ga = 0;
1708   u32 portnum;
1709   int i;
1710
1711   if (vec_len (addresses) > 0)
1712     {
1713
1714       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1715
1716       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1717         {
1718           a = addresses + i;
1719           switch (proto)
1720             {
1721 #define _(N, j, n, s)                                                         \
1722   case NAT_PROTOCOL_##N:                                                      \
1723     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1724       {                                                                       \
1725         if (a->fib_index == fib_index)                                        \
1726           {                                                                   \
1727             while (1)                                                         \
1728               {                                                               \
1729                 portnum = (port_per_thread * snat_thread_index) +             \
1730                           nat_random_port (&nm->random_seed, 0,               \
1731                                            port_per_thread - 1) +             \
1732                           1024;                                               \
1733                 if (a->busy_##n##_port_refcounts[portnum])                    \
1734                   continue;                                                   \
1735                 --a->busy_##n##_port_refcounts[portnum];                      \
1736                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1737                 a->busy_##n##_ports++;                                        \
1738                 *addr = a->addr;                                              \
1739                 *port = clib_host_to_net_u16 (portnum);                       \
1740                 return 0;                                                     \
1741               }                                                               \
1742           }                                                                   \
1743         else if (a->fib_index == ~0)                                          \
1744           {                                                                   \
1745             ga = a;                                                           \
1746           }                                                                   \
1747       }                                                                       \
1748     break;
1749               foreach_nat_protocol;
1750             default:
1751               nat_elog_info (nm, "unknown protocol");
1752               return 1;
1753             }
1754         }
1755
1756       for (i = 0; i < s_addr_offset; ++i)
1757         {
1758           a = addresses + i;
1759           switch (proto)
1760             {
1761               foreach_nat_protocol;
1762             default:
1763               nat_elog_info (nm, "unknown protocol");
1764               return 1;
1765             }
1766         }
1767   if (ga)
1768     {
1769       a = ga;
1770       // fake fib index to reuse macro
1771       fib_index = ~0;
1772       switch (proto)
1773         {
1774           foreach_nat_protocol;
1775             default : nat_elog_info (nm, "unknown protocol");
1776           return 1;
1777         }
1778     }
1779     }
1780
1781 #undef _
1782
1783   /* Totally out of translations to use... */
1784   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1785   return 1;
1786 }
1787
1788 static int
1789 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1790                          u32 thread_index, nat_protocol_t proto,
1791                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1792                          u16 port_per_thread, u32 snat_thread_index)
1793 {
1794   nat44_ei_main_t *nm = &nat44_ei_main;
1795   nat44_ei_address_t *a = addresses;
1796   u16 portnum, ports;
1797
1798   ports = nm->end_port - nm->start_port + 1;
1799
1800   if (!vec_len (addresses))
1801     goto exhausted;
1802
1803   switch (proto)
1804     {
1805 #define _(N, i, n, s)                                                         \
1806   case NAT_PROTOCOL_##N:                                                      \
1807     if (a->busy_##n##_ports < ports)                                          \
1808       {                                                                       \
1809         while (1)                                                             \
1810           {                                                                   \
1811             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1812                                        nm->end_port);                         \
1813             if (a->busy_##n##_port_refcounts[portnum])                        \
1814               continue;                                                       \
1815             ++a->busy_##n##_port_refcounts[portnum];                          \
1816             a->busy_##n##_ports++;                                            \
1817             *addr = a->addr;                                                  \
1818             *port = clib_host_to_net_u16 (portnum);                           \
1819             return 0;                                                         \
1820           }                                                                   \
1821       }                                                                       \
1822     break;
1823       foreach_nat_protocol
1824 #undef _
1825         default : nat_elog_info (nm, "unknown protocol");
1826       return 1;
1827     }
1828
1829 exhausted:
1830   /* Totally out of translations to use... */
1831   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1832   return 1;
1833 }
1834
1835 static int
1836 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1837                         u32 thread_index, nat_protocol_t proto,
1838                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1839                         u16 port_per_thread, u32 snat_thread_index)
1840 {
1841   nat44_ei_main_t *nm = &nat44_ei_main;
1842   nat44_ei_address_t *a = addresses;
1843   u16 m, ports, portnum, A, j;
1844   m = 16 - (nm->psid_offset + nm->psid_length);
1845   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1846
1847   if (!vec_len (addresses))
1848     goto exhausted;
1849
1850   switch (proto)
1851     {
1852 #define _(N, i, n, s)                                                         \
1853   case NAT_PROTOCOL_##N:                                                      \
1854     if (a->busy_##n##_ports < ports)                                          \
1855       {                                                                       \
1856         while (1)                                                             \
1857           {                                                                   \
1858             A = nat_random_port (&nm->random_seed, 1,                         \
1859                                  pow2_mask (nm->psid_offset));                \
1860             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
1861             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
1862             if (a->busy_##n##_port_refcounts[portnum])                        \
1863               continue;                                                       \
1864             ++a->busy_##n##_port_refcounts[portnum];                          \
1865             a->busy_##n##_ports++;                                            \
1866             *addr = a->addr;                                                  \
1867             *port = clib_host_to_net_u16 (portnum);                           \
1868             return 0;                                                         \
1869           }                                                                   \
1870       }                                                                       \
1871     break;
1872       foreach_nat_protocol
1873 #undef _
1874         default : nat_elog_info (nm, "unknown protocol");
1875       return 1;
1876     }
1877
1878 exhausted:
1879   /* Totally out of translations to use... */
1880   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1881   return 1;
1882 }
1883
1884 void
1885 nat44_ei_set_alloc_default ()
1886 {
1887   nat44_ei_main_t *nm = &nat44_ei_main;
1888
1889   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1890   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1891 }
1892
1893 void
1894 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1895 {
1896   nat44_ei_main_t *nm = &nat44_ei_main;
1897
1898   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1899   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1900   nm->start_port = start_port;
1901   nm->end_port = end_port;
1902 }
1903
1904 void
1905 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1906 {
1907   nat44_ei_main_t *nm = &nat44_ei_main;
1908
1909   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1910   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1911   nm->psid = psid;
1912   nm->psid_offset = psid_offset;
1913   nm->psid_length = psid_length;
1914 }
1915
1916 void
1917 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
1918                          u32 thread_index)
1919 {
1920   nat44_ei_main_per_thread_data_t *tnm =
1921     vec_elt_at_index (nm->per_thread_data, thread_index);
1922   clib_bihash_kv_8_8_t kv, value;
1923   nat44_ei_user_t *u;
1924   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
1925                                       .fib_index = ses->in2out.fib_index };
1926   const u8 u_static = nat44_ei_is_session_static (ses);
1927
1928   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
1929   pool_put_index (tnm->list_pool, ses->per_user_index);
1930
1931   pool_put (tnm->sessions, ses);
1932   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1933                            pool_elts (tnm->sessions));
1934
1935   kv.key = u_key.as_u64;
1936   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1937     {
1938       u = pool_elt_at_index (tnm->users, value.value);
1939       if (u_static)
1940         u->nstaticsessions--;
1941       else
1942         u->nsessions--;
1943
1944       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1945     }
1946 }
1947
1948 int
1949 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
1950                       nat_protocol_t proto, u32 vrf_id, int is_in)
1951 {
1952   nat44_ei_main_per_thread_data_t *tnm;
1953   clib_bihash_kv_8_8_t kv, value;
1954   u32 fib_index;
1955   nat44_ei_session_t *s;
1956   clib_bihash_8_8_t *t;
1957
1958   fail_if_disabled ();
1959
1960   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
1961   init_nat_k (&kv, *addr, port, fib_index, proto);
1962   t = is_in ? &nm->in2out : &nm->out2in;
1963   if (!clib_bihash_search_8_8 (t, &kv, &value))
1964     {
1965       // this is called from API/CLI, so the world is stopped here
1966       // it's safe to manipulate arbitrary per-thread data
1967       u32 thread_index = nat_value_get_thread_index (&value);
1968       tnm = vec_elt_at_index (nm->per_thread_data, thread_index);
1969       u32 session_index = nat_value_get_session_index (&value);
1970       if (pool_is_free_index (tnm->sessions, session_index))
1971         return VNET_API_ERROR_UNSPECIFIED;
1972
1973       s = pool_elt_at_index (tnm->sessions, session_index);
1974       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
1975       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1976       return 0;
1977     }
1978
1979   return VNET_API_ERROR_NO_SUCH_ENTRY;
1980 }
1981
1982 u32
1983 nat44_ei_get_thread_idx_by_port (u16 e_port)
1984 {
1985   nat44_ei_main_t *nm = &nat44_ei_main;
1986   u32 thread_idx = nm->num_workers;
1987   if (nm->num_workers > 1)
1988     {
1989       thread_idx = nm->first_worker_index +
1990                    nm->workers[(e_port - 1024) / nm->port_per_thread];
1991     }
1992   return thread_idx;
1993 }
1994
1995 void
1996 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
1997                               int is_add)
1998 {
1999   nat44_ei_main_t *nm = &nat44_ei_main;
2000   fib_prefix_t prefix = {
2001     .fp_len = p_len,
2002     .fp_proto = FIB_PROTOCOL_IP4,
2003     .fp_addr = {
2004                 .ip4.as_u32 = addr->as_u32,
2005                 },
2006   };
2007   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2008
2009   if (is_add)
2010     {
2011       fib_table_entry_update_one_path (fib_index, &prefix, nm->fib_src_low,
2012                                        (FIB_ENTRY_FLAG_CONNECTED |
2013                                         FIB_ENTRY_FLAG_LOCAL |
2014                                         FIB_ENTRY_FLAG_EXCLUSIVE),
2015                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
2016                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
2017     }
2018   else
2019     {
2020       fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
2021     }
2022 }
2023
2024 int
2025 nat44_ei_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2026 {
2027   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2028   nat44_ei_main_t *nm = &nat44_ei_main;
2029   nat44_ei_address_t *a = 0;
2030   int i;
2031
2032   for (i = 0; i < vec_len (nm->addresses); i++)
2033     {
2034       a = nm->addresses + i;
2035
2036       if (a->addr.as_u32 != addr.as_u32)
2037         continue;
2038
2039       switch (proto)
2040         {
2041 #define _(N, j, n, s)                                                         \
2042   case NAT_PROTOCOL_##N:                                                      \
2043     if (a->busy_##n##_port_refcounts[port])                                   \
2044       goto done;                                                              \
2045     ++a->busy_##n##_port_refcounts[port];                                     \
2046     if (port > 1024)                                                          \
2047       {                                                                       \
2048         a->busy_##n##_ports++;                                                \
2049         a->busy_##n##_ports_per_thread[ti]++;                                 \
2050       }                                                                       \
2051     break;
2052           foreach_nat_protocol
2053 #undef _
2054             default : nat_elog_info (nm, "unknown protocol");
2055           goto done;
2056         }
2057
2058       return 0;
2059     }
2060
2061 done:
2062   return 1;
2063 }
2064
2065 int
2066 nat44_ei_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2067 {
2068   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2069   nat44_ei_main_t *nm = &nat44_ei_main;
2070   nat44_ei_address_t *a = 0;
2071   int i;
2072
2073   for (i = 0; i < vec_len (nm->addresses); i++)
2074     {
2075       a = nm->addresses + i;
2076
2077       if (a->addr.as_u32 != addr.as_u32)
2078         continue;
2079
2080       switch (proto)
2081         {
2082 #define _(N, j, n, s)                                                         \
2083   case NAT_PROTOCOL_##N:                                                      \
2084     --a->busy_##n##_port_refcounts[port];                                     \
2085     if (port > 1024)                                                          \
2086       {                                                                       \
2087         a->busy_##n##_ports--;                                                \
2088         a->busy_##n##_ports_per_thread[ti]--;                                 \
2089       }                                                                       \
2090     break;
2091           foreach_nat_protocol
2092 #undef _
2093             default : nat_elog_info (nm, "unknown protocol");
2094           goto done;
2095         }
2096
2097       return 0;
2098     }
2099
2100 done:
2101   return 1;
2102 }
2103
2104 void
2105 nat44_ei_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2106                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2107                              u32 flags, ip4_address_t pool_addr, u8 *tag)
2108 {
2109   nat44_ei_static_map_resolve_t *rp;
2110   nat44_ei_main_t *nm = &nat44_ei_main;
2111
2112   vec_add2 (nm->to_resolve, rp, 1);
2113   rp->l_addr.as_u32 = l_addr.as_u32;
2114   rp->l_port = l_port;
2115   rp->e_port = e_port;
2116   rp->sw_if_index = sw_if_index;
2117   rp->vrf_id = vrf_id;
2118   rp->proto = proto;
2119   rp->flags = flags;
2120   rp->pool_addr = pool_addr;
2121   rp->tag = vec_dup (tag);
2122 }
2123
2124 int
2125 nat44_ei_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2126                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2127                              u32 flags, int *out)
2128 {
2129   nat44_ei_static_map_resolve_t *rp;
2130   nat44_ei_main_t *nm = &nat44_ei_main;
2131   int i;
2132
2133   for (i = 0; i < vec_len (nm->to_resolve); i++)
2134     {
2135       rp = nm->to_resolve + i;
2136
2137       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
2138         {
2139           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
2140             {
2141               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2142                 {
2143                   if (rp->e_port != e_port || rp->proto != proto)
2144                     {
2145                       continue;
2146                     }
2147                 }
2148             }
2149           else if (rp->l_addr.as_u32 == l_addr.as_u32)
2150             {
2151               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2152                 {
2153                   if (rp->l_port != l_port || rp->e_port != e_port ||
2154                       rp->proto != proto)
2155                     {
2156                       continue;
2157                     }
2158                 }
2159             }
2160           else
2161             {
2162               continue;
2163             }
2164           if (out)
2165             {
2166               *out = i;
2167             }
2168           return 0;
2169         }
2170     }
2171   return 1;
2172 }
2173
2174 int
2175 nat44_ei_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2176                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2177                              u32 flags)
2178 {
2179   nat44_ei_main_t *nm = &nat44_ei_main;
2180   int i;
2181   if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2182                                     sw_if_index, flags, &i))
2183     {
2184       vec_del1 (nm->to_resolve, i);
2185       return 0;
2186     }
2187   return 1;
2188 }
2189
2190 void
2191 delete_matching_dynamic_sessions (const nat44_ei_static_mapping_t *m,
2192                                   u32 worker_index)
2193 {
2194   nat44_ei_main_t *nm = &nat44_ei_main;
2195   clib_bihash_kv_8_8_t kv, value;
2196   nat44_ei_session_t *s;
2197   nat44_ei_user_key_t u_key;
2198   nat44_ei_user_t *u;
2199   nat44_ei_main_per_thread_data_t *tnm;
2200   dlist_elt_t *head, *elt;
2201   u32 elt_index, head_index;
2202   u32 ses_index;
2203   u64 user_index;
2204
2205   if (nm->static_mapping_only)
2206     return;
2207
2208   tnm = vec_elt_at_index (nm->per_thread_data, worker_index);
2209
2210   u_key.addr = m->local_addr;
2211   u_key.fib_index = m->fib_index;
2212   kv.key = u_key.as_u64;
2213   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2214     {
2215       user_index = value.value;
2216       u = pool_elt_at_index (tnm->users, user_index);
2217       if (u->nsessions)
2218         {
2219           head_index = u->sessions_per_user_list_head_index;
2220           head = pool_elt_at_index (tnm->list_pool, head_index);
2221           elt_index = head->next;
2222           elt = pool_elt_at_index (tnm->list_pool, elt_index);
2223           ses_index = elt->value;
2224           while (ses_index != ~0)
2225             {
2226               s = pool_elt_at_index (tnm->sessions, ses_index);
2227               elt = pool_elt_at_index (tnm->list_pool, elt->next);
2228               ses_index = elt->value;
2229
2230               if (nat44_ei_is_session_static (s))
2231                 continue;
2232
2233               if (!is_sm_addr_only (m->flags) &&
2234                   s->in2out.port != m->local_port)
2235                 continue;
2236
2237               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
2238                                              0);
2239               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2240
2241               if (!is_sm_addr_only (m->flags))
2242                 break;
2243             }
2244         }
2245     }
2246 }
2247
2248 int
2249 nat44_ei_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2250                              u16 l_port, u16 e_port, nat_protocol_t proto,
2251                              u32 vrf_id, u32 sw_if_index, u32 flags,
2252                              ip4_address_t pool_addr, u8 *tag)
2253 {
2254   nat44_ei_main_t *nm = &nat44_ei_main;
2255   clib_bihash_kv_8_8_t kv, value;
2256   nat44_ei_lb_addr_port_t *local;
2257   nat44_ei_static_mapping_t *m;
2258   u32 fib_index = ~0;
2259   u32 worker_index;
2260
2261   fail_if_disabled ();
2262
2263   if (is_sm_addr_only (flags))
2264     {
2265       e_port = l_port = proto = 0;
2266     }
2267
2268   if (sw_if_index != ~0)
2269     {
2270       // this mapping is interface bound
2271       ip4_address_t *first_int_addr;
2272
2273       // check if this record isn't registered for resolve
2274       if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2275                                         sw_if_index, flags, 0))
2276         {
2277           return VNET_API_ERROR_VALUE_EXIST;
2278         }
2279       // register record for resolve
2280       nat44_ei_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2281                                    sw_if_index, flags, pool_addr, tag);
2282
2283       first_int_addr =
2284         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2285       if (!first_int_addr)
2286         {
2287           // dhcp resolution required
2288           return 0;
2289         }
2290
2291       e_addr.as_u32 = first_int_addr->as_u32;
2292     }
2293
2294   if (is_sm_identity_nat (flags))
2295     {
2296       l_port = e_port;
2297       l_addr.as_u32 = e_addr.as_u32;
2298     }
2299
2300   // fib index 0
2301   init_nat_k (&kv, e_addr, e_port, 0, proto);
2302
2303   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2304     {
2305       m = pool_elt_at_index (nm->static_mappings, value.value);
2306       if (!is_sm_identity_nat (m->flags))
2307         {
2308           return VNET_API_ERROR_VALUE_EXIST;
2309         }
2310
2311       // case:
2312       // adding local identity nat record for different vrf table
2313       pool_foreach (local, m->locals)
2314         {
2315           if (local->vrf_id == vrf_id)
2316             {
2317               return VNET_API_ERROR_VALUE_EXIST;
2318             }
2319         }
2320
2321       pool_get (m->locals, local);
2322
2323       local->vrf_id = vrf_id;
2324       local->fib_index = fib_table_find_or_create_and_lock (
2325         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2326
2327       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
2328                    m->proto, 0, m - nm->static_mappings);
2329       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2330
2331       return 0;
2332     }
2333
2334   if (vrf_id != ~0)
2335     {
2336       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
2337                                                      nm->fib_src_low);
2338     }
2339   else
2340     {
2341       // fallback to default vrf
2342       vrf_id = nm->inside_vrf_id;
2343       fib_index = nm->inside_fib_index;
2344       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2345     }
2346
2347   if (!is_sm_identity_nat (flags))
2348     {
2349       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2350       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2351         {
2352           return VNET_API_ERROR_VALUE_EXIST;
2353         }
2354     }
2355
2356   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2357     {
2358       if (nat44_ei_reserve_port (e_addr, e_port, proto))
2359         {
2360           // remove resolve record
2361           if ((sw_if_index != ~0) && !is_sm_identity_nat (flags))
2362             {
2363               nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto,
2364                                            vrf_id, sw_if_index, flags);
2365             }
2366           return VNET_API_ERROR_NO_SUCH_ENTRY;
2367         }
2368     }
2369
2370   pool_get (nm->static_mappings, m);
2371   clib_memset (m, 0, sizeof (*m));
2372
2373   m->flags = flags;
2374   m->local_addr = l_addr;
2375   m->external_addr = e_addr;
2376
2377   m->tag = vec_dup (tag);
2378
2379   if (!is_sm_addr_only (flags))
2380     {
2381       m->local_port = l_port;
2382       m->external_port = e_port;
2383       m->proto = proto;
2384     }
2385
2386   if (is_sm_identity_nat (flags))
2387     {
2388       pool_get (m->locals, local);
2389
2390       local->vrf_id = vrf_id;
2391       local->fib_index = fib_index;
2392     }
2393   else
2394     {
2395       m->vrf_id = vrf_id;
2396       m->fib_index = fib_index;
2397     }
2398
2399   init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2400                m - nm->static_mappings);
2401   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2402
2403   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2404                m - nm->static_mappings);
2405   clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2406
2407   if (nm->num_workers > 1)
2408     {
2409       // store worker index for this record
2410       ip4_header_t ip = {
2411         .src_address = m->local_addr,
2412       };
2413       worker_index = nat44_ei_get_in2out_worker_index (&ip, m->fib_index, 0);
2414       vec_add1 (m->workers, worker_index);
2415     }
2416   else
2417     {
2418       worker_index = nm->num_workers;
2419     }
2420   delete_matching_dynamic_sessions (m, worker_index);
2421
2422   if (is_sm_addr_only (flags))
2423     {
2424       nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
2425     }
2426
2427   return 0;
2428 }
2429
2430 int
2431 nat44_ei_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2432                              u16 l_port, u16 e_port, nat_protocol_t proto,
2433                              u32 vrf_id, u32 sw_if_index, u32 flags)
2434 {
2435   nat44_ei_main_per_thread_data_t *tnm;
2436   nat44_ei_main_t *nm = &nat44_ei_main;
2437   clib_bihash_kv_8_8_t kv, value;
2438   nat44_ei_lb_addr_port_t *local;
2439   nat44_ei_static_mapping_t *m;
2440   u32 fib_index = ~0;
2441   nat44_ei_user_key_t u_key;
2442
2443   fail_if_disabled ();
2444
2445   if (is_sm_addr_only (flags))
2446     {
2447       e_port = l_port = proto = 0;
2448     }
2449
2450   if (sw_if_index != ~0)
2451     {
2452       // this mapping is interface bound
2453       ip4_address_t *first_int_addr;
2454
2455       // delete record registered for resolve
2456       if (nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2457                                        sw_if_index, flags))
2458         {
2459           return VNET_API_ERROR_NO_SUCH_ENTRY;
2460         }
2461
2462       first_int_addr =
2463         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2464       if (!first_int_addr)
2465         {
2466           // dhcp resolution required
2467           return 0;
2468         }
2469
2470       e_addr.as_u32 = first_int_addr->as_u32;
2471     }
2472
2473   if (is_sm_identity_nat (flags))
2474     {
2475       l_port = e_port;
2476       l_addr.as_u32 = e_addr.as_u32;
2477     }
2478
2479   // fib index 0
2480   init_nat_k (&kv, e_addr, e_port, 0, proto);
2481
2482   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2483     {
2484       if (sw_if_index != ~0)
2485         {
2486           return 0;
2487         }
2488       return VNET_API_ERROR_NO_SUCH_ENTRY;
2489     }
2490
2491   m = pool_elt_at_index (nm->static_mappings, value.value);
2492
2493   if (is_sm_identity_nat (flags))
2494     {
2495       u8 found = 0;
2496
2497       if (vrf_id == ~0)
2498         {
2499           vrf_id = nm->inside_vrf_id;
2500         }
2501
2502       pool_foreach (local, m->locals)
2503         {
2504           if (local->vrf_id == vrf_id)
2505             {
2506               local = pool_elt_at_index (m->locals, local - m->locals);
2507               fib_index = local->fib_index;
2508               pool_put (m->locals, local);
2509               found = 1;
2510             }
2511         }
2512       if (!found)
2513         {
2514           return VNET_API_ERROR_NO_SUCH_ENTRY;
2515         }
2516     }
2517   else
2518     {
2519       fib_index = m->fib_index;
2520     }
2521
2522   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2523     {
2524       if (nat44_ei_free_port (e_addr, e_port, proto))
2525         {
2526           return VNET_API_ERROR_INVALID_VALUE;
2527         }
2528     }
2529
2530   init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2531   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2532
2533   if (!nm->static_mapping_only || nm->static_mapping_connection_tracking)
2534     {
2535       // delete sessions for static mapping
2536       if (nm->num_workers > 1)
2537         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2538       else
2539         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2540
2541       u_key.addr = m->local_addr;
2542       u_key.fib_index = fib_index;
2543       kv.key = u_key.as_u64;
2544       nat44_ei_static_mapping_del_sessions (
2545         nm, tnm, u_key, is_sm_addr_only (flags), e_addr, e_port);
2546     }
2547
2548   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2549
2550   if (!pool_elts (m->locals))
2551     {
2552       // this is last record remove all required stuff
2553       // fib_index 0
2554       init_nat_k (&kv, e_addr, e_port, 0, proto);
2555       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2556
2557       vec_free (m->tag);
2558       vec_free (m->workers);
2559       pool_put (nm->static_mappings, m);
2560
2561       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
2562         {
2563           nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
2564         }
2565     }
2566
2567   return 0;
2568 }
2569
2570 int
2571 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2572                                u32 match_fib_index,
2573                                nat_protocol_t match_protocol,
2574                                ip4_address_t *mapping_addr, u16 *mapping_port,
2575                                u32 *mapping_fib_index, u8 by_external,
2576                                u8 *is_addr_only, u8 *is_identity_nat)
2577 {
2578   nat44_ei_main_t *nm = &nat44_ei_main;
2579   clib_bihash_kv_8_8_t kv, value;
2580   nat44_ei_static_mapping_t *m;
2581   u16 port;
2582
2583   if (by_external)
2584     {
2585       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2586       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2587                                   &value))
2588         {
2589           /* Try address only mapping */
2590           init_nat_k (&kv, match_addr, 0, 0, 0);
2591           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2592                                       &value))
2593             return 1;
2594         }
2595       m = pool_elt_at_index (nm->static_mappings, value.value);
2596
2597       *mapping_fib_index = m->fib_index;
2598       *mapping_addr = m->local_addr;
2599       port = m->local_port;
2600     }
2601   else
2602     {
2603       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2604                   match_protocol);
2605       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2606         {
2607           /* Try address only mapping */
2608           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2609           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2610                                       &value))
2611             return 1;
2612         }
2613       m = pool_elt_at_index (nm->static_mappings, value.value);
2614
2615       *mapping_fib_index = nm->outside_fib_index;
2616       *mapping_addr = m->external_addr;
2617       port = m->external_port;
2618     }
2619
2620   /* Address only mapping doesn't change port */
2621   if (is_sm_addr_only (m->flags))
2622     *mapping_port = match_port;
2623   else
2624     *mapping_port = port;
2625
2626   if (PREDICT_FALSE (is_addr_only != 0))
2627     *is_addr_only = is_sm_addr_only (m->flags);
2628
2629   if (PREDICT_FALSE (is_identity_nat != 0))
2630     *is_identity_nat = is_sm_identity_nat (m->flags);
2631
2632   return 0;
2633 }
2634
2635 static void
2636 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2637 {
2638   pool_free (tnm->list_pool);
2639   pool_free (tnm->lru_pool);
2640   pool_free (tnm->sessions);
2641   pool_free (tnm->users);
2642
2643   clib_bihash_free_8_8 (&tnm->user_hash);
2644 }
2645
2646 u8 *
2647 format_nat44_ei_key (u8 *s, va_list *args)
2648 {
2649   u64 key = va_arg (*args, u64);
2650
2651   ip4_address_t addr;
2652   u16 port;
2653   nat_protocol_t protocol;
2654   u32 fib_index;
2655
2656   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2657
2658   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2659               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2660               fib_index);
2661   return s;
2662 }
2663
2664 u8 *
2665 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2666 {
2667   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2668   nat44_ei_user_key_t k;
2669
2670   k.as_u64 = v->key;
2671
2672   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2673               k.fib_index, v->value);
2674
2675   return s;
2676 }
2677
2678 u8 *
2679 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2680 {
2681   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2682
2683   s = format (s, "%U thread-index %llu session-index %llu",
2684               format_nat44_ei_key, v->key, nat_value_get_thread_index (v),
2685               nat_value_get_session_index (v));
2686
2687   return s;
2688 }
2689
2690 u8 *
2691 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2692 {
2693   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2694
2695   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2696               v->value);
2697
2698   return s;
2699 }
2700
2701 static void
2702 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2703                          u32 translations, u32 translation_buckets,
2704                          u32 user_buckets)
2705 {
2706   dlist_elt_t *head;
2707
2708   pool_alloc (tnm->list_pool, translations);
2709   pool_alloc (tnm->lru_pool, translations);
2710   pool_alloc (tnm->sessions, translations);
2711
2712   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2713
2714   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2715                                      format_nat44_ei_user_kvp);
2716
2717   pool_get (tnm->lru_pool, head);
2718   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2719   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2720
2721   pool_get (tnm->lru_pool, head);
2722   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2723   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2724
2725   pool_get (tnm->lru_pool, head);
2726   tnm->udp_lru_head_index = head - tnm->lru_pool;
2727   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2728
2729   pool_get (tnm->lru_pool, head);
2730   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2731   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2732
2733   pool_get (tnm->lru_pool, head);
2734   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2735   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2736 }
2737
2738 static void
2739 nat44_ei_db_free ()
2740 {
2741   nat44_ei_main_t *nm = &nat44_ei_main;
2742   nat44_ei_main_per_thread_data_t *tnm;
2743
2744   pool_free (nm->static_mappings);
2745   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
2746   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
2747
2748   if (nm->pat)
2749     {
2750       clib_bihash_free_8_8 (&nm->in2out);
2751       clib_bihash_free_8_8 (&nm->out2in);
2752       vec_foreach (tnm, nm->per_thread_data)
2753         {
2754           nat44_ei_worker_db_free (tnm);
2755         }
2756     }
2757 }
2758
2759 static void
2760 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2761 {
2762   nat44_ei_main_t *nm = &nat44_ei_main;
2763   nat44_ei_main_per_thread_data_t *tnm;
2764
2765   u32 static_mapping_buckets = 1024;
2766   u32 static_mapping_memory_size = 64 << 20;
2767
2768   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2769                         "static_mapping_by_local", static_mapping_buckets,
2770                         static_mapping_memory_size);
2771   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2772                         "static_mapping_by_external", static_mapping_buckets,
2773                         static_mapping_memory_size);
2774   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2775                                      format_nat44_ei_static_mapping_kvp);
2776   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2777                                      format_nat44_ei_static_mapping_kvp);
2778
2779   if (nm->pat)
2780     {
2781       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2782       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2783       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2784                                          format_nat44_ei_session_kvp);
2785       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2786                                          format_nat44_ei_session_kvp);
2787       vec_foreach (tnm, nm->per_thread_data)
2788         {
2789           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2790                                    user_buckets);
2791         }
2792     }
2793 }
2794
2795 void
2796 nat44_ei_sessions_clear ()
2797 {
2798   nat44_ei_main_t *nm = &nat44_ei_main;
2799   nat44_ei_main_per_thread_data_t *tnm;
2800
2801   if (nm->pat)
2802     {
2803       clib_bihash_free_8_8 (&nm->in2out);
2804       clib_bihash_free_8_8 (&nm->out2in);
2805       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2806       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2807       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2808                                          format_nat44_ei_session_kvp);
2809       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2810                                          format_nat44_ei_session_kvp);
2811       vec_foreach (tnm, nm->per_thread_data)
2812         {
2813           nat44_ei_worker_db_free (tnm);
2814           nat44_ei_worker_db_init (tnm, nm->translations,
2815                                    nm->translation_buckets, nm->user_buckets);
2816         }
2817     }
2818
2819   vlib_zero_simple_counter (&nm->total_users, 0);
2820   vlib_zero_simple_counter (&nm->total_sessions, 0);
2821   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2822 }
2823
2824 static void
2825 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2826                              u32 new_fib_index, u32 old_fib_index)
2827 {
2828   nat44_ei_main_t *nm = &nat44_ei_main;
2829   nat44_ei_outside_fib_t *outside_fib;
2830   nat44_ei_interface_t *i;
2831   u8 is_add = 1;
2832   u8 match = 0;
2833
2834   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2835       (!vec_len (nm->outside_fibs)))
2836     {
2837       return;
2838     }
2839
2840   pool_foreach (i, nm->interfaces)
2841     {
2842       if (i->sw_if_index == sw_if_index)
2843         {
2844           if (!(nat44_ei_interface_is_outside (i)))
2845             return;
2846           match = 1;
2847         }
2848     }
2849
2850   pool_foreach (i, nm->output_feature_interfaces)
2851     {
2852       if (i->sw_if_index == sw_if_index)
2853         {
2854           if (!(nat44_ei_interface_is_outside (i)))
2855             return;
2856           match = 1;
2857         }
2858     }
2859
2860   if (!match)
2861     return;
2862
2863   vec_foreach (outside_fib, nm->outside_fibs)
2864     {
2865       if (outside_fib->fib_index == old_fib_index)
2866         {
2867           outside_fib->refcount--;
2868           if (!outside_fib->refcount)
2869             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2870           break;
2871         }
2872     }
2873
2874   vec_foreach (outside_fib, nm->outside_fibs)
2875     {
2876       if (outside_fib->fib_index == new_fib_index)
2877         {
2878           outside_fib->refcount++;
2879           is_add = 0;
2880           break;
2881         }
2882     }
2883
2884   if (is_add)
2885     {
2886       vec_add2 (nm->outside_fibs, outside_fib, 1);
2887       outside_fib->refcount = 1;
2888       outside_fib->fib_index = new_fib_index;
2889     }
2890 }
2891
2892 int
2893 nat44_ei_add_address (ip4_address_t *addr, u32 vrf_id)
2894 {
2895   nat44_ei_main_t *nm = &nat44_ei_main;
2896   vlib_thread_main_t *tm = vlib_get_thread_main ();
2897   nat44_ei_address_t *ap;
2898
2899   fail_if_disabled ();
2900
2901   /* Check if address already exists */
2902   vec_foreach (ap, nm->addresses)
2903     {
2904       if (ap->addr.as_u32 == addr->as_u32)
2905         {
2906           nat44_ei_log_err ("address exist");
2907           return VNET_API_ERROR_VALUE_EXIST;
2908         }
2909     }
2910
2911   vec_add2 (nm->addresses, ap, 1);
2912
2913   ap->fib_index = ~0;
2914   ap->addr = *addr;
2915
2916   if (vrf_id != ~0)
2917     {
2918       ap->fib_index = fib_table_find_or_create_and_lock (
2919         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2920     }
2921
2922 #define _(N, i, n, s)                                                         \
2923   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
2924                sizeof (ap->busy_##n##_port_refcounts));                       \
2925   ap->busy_##n##_ports = 0;                                                   \
2926   ap->busy_##n##_ports_per_thread = 0;                                        \
2927   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
2928                            tm->n_vlib_mains - 1, 0);
2929   foreach_nat_protocol
2930 #undef _
2931
2932     nat44_ei_add_del_addr_to_fib_foreach_out_if (addr, 1);
2933
2934   return 0;
2935 }
2936
2937 int
2938 nat44_ei_del_address (ip4_address_t addr, u8 delete_sm)
2939 {
2940   nat44_ei_main_t *nm = &nat44_ei_main;
2941   nat44_ei_address_t *a = 0;
2942   nat44_ei_session_t *ses;
2943   u32 *ses_to_be_removed = 0, *ses_index;
2944   nat44_ei_main_per_thread_data_t *tnm;
2945   nat44_ei_static_mapping_t *m;
2946   int j;
2947
2948   fail_if_disabled ();
2949
2950   /* Find SNAT address */
2951   for (j = 0; j < vec_len (nm->addresses); j++)
2952     {
2953       if (nm->addresses[j].addr.as_u32 == addr.as_u32)
2954         {
2955           a = nm->addresses + j;
2956           break;
2957         }
2958     }
2959   if (!a)
2960     {
2961       nat44_ei_log_err ("no such address");
2962       return VNET_API_ERROR_NO_SUCH_ENTRY;
2963     }
2964
2965   if (delete_sm)
2966     {
2967       pool_foreach (m, nm->static_mappings)
2968         {
2969           if (m->external_addr.as_u32 == addr.as_u32)
2970             nat44_ei_del_static_mapping (m->local_addr, m->external_addr,
2971                                          m->local_port, m->external_port,
2972                                          m->proto, m->vrf_id, ~0, m->flags);
2973         }
2974     }
2975   else
2976     {
2977       /* Check if address is used in some static mapping */
2978       if (nat44_ei_is_address_used_in_static_mapping (addr))
2979         {
2980           nat44_ei_log_err ("address used in static mapping");
2981           return VNET_API_ERROR_UNSPECIFIED;
2982         }
2983     }
2984
2985   if (a->fib_index != ~0)
2986     {
2987       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2988     }
2989
2990   /* Delete sessions using address */
2991   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
2992     {
2993       vec_foreach (tnm, nm->per_thread_data)
2994         {
2995           pool_foreach (ses, tnm->sessions)
2996             {
2997               if (ses->out2in.addr.as_u32 == addr.as_u32)
2998                 {
2999                   nat44_ei_free_session_data (nm, ses,
3000                                               tnm - nm->per_thread_data, 0);
3001                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
3002                 }
3003             }
3004           vec_foreach (ses_index, ses_to_be_removed)
3005             {
3006               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
3007               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
3008             }
3009           vec_free (ses_to_be_removed);
3010         }
3011     }
3012
3013 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
3014   foreach_nat_protocol
3015 #undef _
3016
3017     vec_del1 (nm->addresses, j);
3018
3019   nat44_ei_add_del_addr_to_fib_foreach_out_if (&addr, 0);
3020
3021   return 0;
3022 }
3023
3024 int
3025 nat44_ei_add_interface_address (u32 sw_if_index)
3026 {
3027   nat44_ei_main_t *nm = &nat44_ei_main;
3028   ip4_main_t *ip4_main = nm->ip4_main;
3029   ip4_address_t *first_int_addr;
3030   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3031   int i;
3032
3033   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3034     {
3035       if (auto_add_sw_if_indices[i] == sw_if_index)
3036         {
3037           return VNET_API_ERROR_VALUE_EXIST;
3038         }
3039     }
3040
3041   /* add to the auto-address list */
3042   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
3043
3044   // if the address is already bound - or static - add it now
3045   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3046   if (first_int_addr)
3047     {
3048       (void) nat44_ei_add_address (first_int_addr, ~0);
3049     }
3050
3051   return 0;
3052 }
3053
3054 int
3055 nat44_ei_del_interface_address (u32 sw_if_index)
3056 {
3057   nat44_ei_main_t *nm = &nat44_ei_main;
3058   ip4_main_t *ip4_main = nm->ip4_main;
3059   ip4_address_t *first_int_addr;
3060   nat44_ei_static_map_resolve_t *rp;
3061   u32 *indices_to_delete = 0;
3062   int i, j;
3063   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3064
3065   fail_if_disabled ();
3066
3067   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3068
3069   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3070     {
3071       if (auto_add_sw_if_indices[i] == sw_if_index)
3072         {
3073           first_int_addr =
3074             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3075           if (first_int_addr)
3076             {
3077               (void) nat44_ei_del_address (first_int_addr[0], 1);
3078             }
3079           else
3080             {
3081               for (j = 0; j < vec_len (nm->to_resolve); j++)
3082                 {
3083                   rp = nm->to_resolve + j;
3084                   if (rp->sw_if_index == sw_if_index)
3085                     {
3086                       vec_add1 (indices_to_delete, j);
3087                     }
3088                 }
3089               if (vec_len (indices_to_delete))
3090                 {
3091                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3092                     {
3093                       vec_del1 (nm->to_resolve, j);
3094                     }
3095                   vec_free (indices_to_delete);
3096                 }
3097             }
3098
3099           vec_del1 (nm->auto_add_sw_if_indices, i);
3100           return 0;
3101         }
3102     }
3103   return VNET_API_ERROR_NO_SUCH_ENTRY;
3104 }
3105
3106 static_always_inline int
3107 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3108 {
3109   u32 *i;
3110   vec_foreach (i, sw_if_indices)
3111     {
3112       if (*i == sw_if_index)
3113         {
3114           return 1;
3115         }
3116     }
3117   return 0;
3118 }
3119
3120 static void
3121 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3122                                            u32 sw_if_index,
3123                                            ip4_address_t *address,
3124                                            u32 address_length,
3125                                            u32 if_address_index, u32 is_delete)
3126 {
3127   nat44_ei_main_t *nm = &nat44_ei_main;
3128   nat44_ei_static_map_resolve_t *rp;
3129   nat44_ei_address_t *addresses = nm->addresses;
3130   int rv, i;
3131
3132   if (!nm->enabled)
3133     {
3134       return;
3135     }
3136
3137   if (!is_sw_if_index_reg_for_auto_resolve (nm->auto_add_sw_if_indices,
3138                                             sw_if_index))
3139     {
3140       return;
3141     }
3142
3143   if (!is_delete)
3144     {
3145       /* Don't trip over lease renewal, static config */
3146       for (i = 0; i < vec_len (addresses); i++)
3147         {
3148           if (addresses[i].addr.as_u32 == address->as_u32)
3149             {
3150               return;
3151             }
3152         }
3153
3154       (void) nat44_ei_add_address (address, ~0);
3155
3156       /* Scan static map resolution vector */
3157       for (i = 0; i < vec_len (nm->to_resolve); i++)
3158         {
3159           rp = nm->to_resolve + i;
3160           if (is_sm_addr_only (rp->flags))
3161             {
3162               continue;
3163             }
3164           /* On this interface? */
3165           if (rp->sw_if_index == sw_if_index)
3166             {
3167               rv = nat44_ei_add_static_mapping (
3168                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3169                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3170               if (rv)
3171                 {
3172                   nat_elog_notice_X1 (nm, "add_static_mapping returned %d",
3173                                       "i4", rv);
3174                 }
3175             }
3176         }
3177     }
3178   else
3179     {
3180       // remove all static mapping records
3181       (void) nat44_ei_del_address (address[0], 1);
3182     }
3183 }
3184
3185 int
3186 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
3187 {
3188   fail_if_enabled ();
3189   nat44_ei_main_t *nm = &nat44_ei_main;
3190   nm->frame_queue_nelts = frame_queue_nelts;
3191   return 0;
3192 }
3193
3194 static void
3195 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
3196                                       u32 sw_if_index, ip4_address_t *address,
3197                                       u32 address_length, u32 if_address_index,
3198                                       u32 is_delete)
3199 {
3200   nat44_ei_main_t *nm = &nat44_ei_main;
3201   nat44_ei_static_map_resolve_t *rp;
3202   nat44_ei_static_mapping_t *m;
3203   clib_bihash_kv_8_8_t kv, value;
3204   int i, rv = 0, match = 0;
3205
3206   if (!nm->enabled)
3207     {
3208       return;
3209     }
3210
3211   for (i = 0; i < vec_len (nm->to_resolve); i++)
3212     {
3213       rp = nm->to_resolve + i;
3214
3215       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3216         {
3217           match = 1;
3218           break;
3219         }
3220     }
3221
3222   if (!match)
3223     {
3224       return;
3225     }
3226
3227   init_nat_k (&kv, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port,
3228               nm->outside_fib_index,
3229               is_sm_addr_only (rp->flags) ? 0 : rp->proto);
3230   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
3231     m = 0;
3232   else
3233     m = pool_elt_at_index (nm->static_mappings, value.value);
3234
3235   if (is_delete)
3236     {
3237       if (!m)
3238         return;
3239       rv = nat44_ei_del_static_mapping (rp->l_addr, address[0], rp->l_port,
3240                                         rp->e_port, rp->proto, rp->vrf_id, ~0,
3241                                         rp->flags);
3242       if (rv)
3243         {
3244           nat_elog_notice_X1 (nm, "nat44_ei_del_static_mapping returned %d",
3245                               "i4", rv);
3246         }
3247     }
3248   else
3249     {
3250       if (m)
3251         return;
3252       rv = nat44_ei_add_static_mapping (rp->l_addr, address[0], rp->l_port,
3253                                         rp->e_port, rp->proto, rp->vrf_id, ~0,
3254                                         rp->flags, rp->pool_addr, rp->tag);
3255       if (rv)
3256         {
3257           nat_elog_notice_X1 (nm, "nat44_ei_add_static_mapping returned %d",
3258                               "i4", rv);
3259         }
3260     }
3261 }
3262
3263 static_always_inline uword
3264 nat44_ei_classify_inline_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
3265                              vlib_frame_t *frame)
3266 {
3267   u32 n_left_from, *from, *to_next;
3268   nat44_ei_classify_next_t next_index;
3269   nat44_ei_main_t *nm = &nat44_ei_main;
3270   nat44_ei_static_mapping_t *m;
3271   u32 next_in2out = 0, next_out2in = 0;
3272
3273   from = vlib_frame_vector_args (frame);
3274   n_left_from = frame->n_vectors;
3275   next_index = node->cached_next_index;
3276
3277   while (n_left_from > 0)
3278     {
3279       u32 n_left_to_next;
3280
3281       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3282
3283       while (n_left_from > 0 && n_left_to_next > 0)
3284         {
3285           u32 bi0;
3286           vlib_buffer_t *b0;
3287           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3288           ip4_header_t *ip0;
3289           nat44_ei_address_t *ap;
3290           clib_bihash_kv_8_8_t kv0, value0;
3291
3292           /* speculatively enqueue b0 to the current next frame */
3293           bi0 = from[0];
3294           to_next[0] = bi0;
3295           from += 1;
3296           to_next += 1;
3297           n_left_from -= 1;
3298           n_left_to_next -= 1;
3299
3300           b0 = vlib_get_buffer (vm, bi0);
3301           ip0 = vlib_buffer_get_current (b0);
3302
3303           vec_foreach (ap, nm->addresses)
3304             {
3305               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
3306                 {
3307                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3308                   goto enqueue0;
3309                 }
3310             }
3311
3312           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
3313             {
3314               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
3315               /* try to classify the fragment based on IP header alone */
3316               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3317                                            &kv0, &value0))
3318                 {
3319                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3320                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3321                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3322                   goto enqueue0;
3323                 }
3324               init_nat_k (&kv0, ip0->dst_address,
3325                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
3326                           ip_proto_to_nat_proto (ip0->protocol));
3327               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3328                                            &kv0, &value0))
3329                 {
3330                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3331                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3332                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3333                 }
3334             }
3335
3336         enqueue0:
3337           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
3338                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
3339             {
3340               nat44_ei_classify_trace_t *t =
3341                 vlib_add_trace (vm, node, b0, sizeof (*t));
3342               t->cached = 0;
3343               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
3344             }
3345
3346           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3347           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3348
3349           /* verify speculative enqueue, maybe switch current next frame */
3350           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3351                                            n_left_to_next, bi0, next0);
3352         }
3353
3354       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3355     }
3356
3357   vlib_node_increment_counter (
3358     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
3359   vlib_node_increment_counter (
3360     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3361   return frame->n_vectors;
3362 }
3363
3364 VLIB_NODE_FN (nat44_ei_classify_node)
3365 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3366 {
3367   return nat44_ei_classify_inline_fn (vm, node, frame);
3368 }
3369
3370 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3371   .name = "nat44-ei-classify",
3372   .vector_size = sizeof (u32),
3373   .format_trace = format_nat44_ei_classify_trace,
3374   .type = VLIB_NODE_TYPE_INTERNAL,
3375   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3376   .error_strings = nat44_ei_classify_error_strings,
3377   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3378   .next_nodes = {
3379     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3380     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3381     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3382   },
3383 };
3384
3385 VLIB_NODE_FN (nat44_ei_handoff_classify_node)
3386 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3387 {
3388   return nat44_ei_classify_inline_fn (vm, node, frame);
3389 }
3390
3391 VLIB_REGISTER_NODE (nat44_ei_handoff_classify_node) = {
3392   .name = "nat44-ei-handoff-classify",
3393   .vector_size = sizeof (u32),
3394   .format_trace = format_nat44_ei_classify_trace,
3395   .type = VLIB_NODE_TYPE_INTERNAL,
3396   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3397   .error_strings = nat44_ei_classify_error_strings,
3398   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3399   .next_nodes = {
3400     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out-worker-handoff",
3401     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in-worker-handoff",
3402     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3403   },
3404 };
3405
3406 /*
3407  * fd.io coding-style-patch-verification: ON
3408  *
3409  * Local Variables:
3410  * eval: (c-set-style "gnu")
3411  * End:
3412  */