src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t
  44   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  45 extern vlib_node_registration_t
  46   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  47
  48 #define skip_if_disabled()                                                    \
  49   do                                                                          \
  50     {                                                                         \
  51       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  52       if (PREDICT_FALSE (!nm->enabled))                                       \
  53         return;                                                               \
  54     }                                                                         \
  55   while (0)
  56
  57 #define fail_if_enabled()                                                     \
  58   do                                                                          \
  59     {                                                                         \
  60       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  61       if (PREDICT_FALSE (nm->enabled))                                        \
  62         {                                                                     \
  63           nat44_ei_log_err ("plugin enabled");                                \
  64           return 1;                                                           \
  65         }                                                                     \
  66     }                                                                         \
  67   while (0)
  68
  69 #define fail_if_disabled()                                                    \
  70   do                                                                          \
  71     {                                                                         \
  72       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  73       if (PREDICT_FALSE (!nm->enabled))                                       \
  74         {                                                                     \
  75           nat44_ei_log_err ("plugin disabled");                               \
  76           return 1;                                                           \
  77         }                                                                     \
  78     }                                                                         \
  79   while (0)
  80
  81 /* Hook up input features */
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-ei-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  86                                "ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-ei-handoff-classify",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  92                                "ip4-sv-reassembly-feature"),
  93 };
  94 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-ei-in2out",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-sv-reassembly-feature"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-ei-out2in",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 104                                "ip4-sv-reassembly-feature",
 105                                "ip4-dhcp-client-detect"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 108   .arc_name = "ip4-output",
 109   .node_name = "nat44-ei-in2out-output",
 110   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 111                                "ip4-sv-reassembly-output-feature"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 114   .arc_name = "ip4-local",
 115   .node_name = "nat44-ei-hairpinning",
 116   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-ei-in2out-worker-handoff",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ei-out2in-worker-handoff",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 127                                "ip4-dhcp-client-detect"),
 128 };
 129 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 130   .arc_name = "ip4-output",
 131   .node_name = "nat44-ei-in2out-output-worker-handoff",
 132   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 133                                "ip4-sv-reassembly-output-feature"),
 134 };
 135
 136 VLIB_PLUGIN_REGISTER () = {
 137   .version = VPP_BUILD_VER,
 138   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 139 };
 140
 141 #define foreach_nat44_ei_classify_error                                       \
 142   _ (NEXT_IN2OUT, "next in2out")                                              \
 143   _ (NEXT_OUT2IN, "next out2in")                                              \
 144   _ (FRAG_CACHED, "fragment cached")
 145
 146 typedef enum
 147 {
 148 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 149   foreach_nat44_ei_classify_error
 150 #undef _
 151     NAT44_EI_CLASSIFY_N_ERROR,
 152 } nat44_ei_classify_error_t;
 153
 154 static char *nat44_ei_classify_error_strings[] = {
 155 #define _(sym, string) string,
 156   foreach_nat44_ei_classify_error
 157 #undef _
 158 };
 159
 160 typedef enum
 161 {
 162   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 163   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 164   NAT44_EI_CLASSIFY_NEXT_DROP,
 165   NAT44_EI_CLASSIFY_N_NEXT,
 166 } nat44_ei_classify_next_t;
 167
 168 typedef struct
 169 {
 170   u8 next_in2out;
 171   u8 cached;
 172 } nat44_ei_classify_trace_t;
 173
 174 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 175                                    u32 sw_if_index, int is_add);
 176
 177 static void nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm);
 178
 179 static int nat44_ei_add_static_mapping_internal (
 180   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 181   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 182   ip4_address_t pool_addr, u8 *tag);
 183
 184 static int nat44_ei_del_static_mapping_internal (
 185   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 186   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags);
 187
 188 static u8 *
 189 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 190 {
 191   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 192   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 193   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 194   char *next;
 195
 196   if (t->cached)
 197     s = format (s, "nat44-ei-classify: fragment cached");
 198   else
 199     {
 200       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 201       s = format (s, "nat44-ei-classify: next %s", next);
 202     }
 203
 204   return s;
 205 }
 206
 207 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 208                               u32 user_buckets);
 209
 210 static void nat44_ei_ip4_add_del_interface_address_cb (
 211   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 212   u32 address_length, u32 if_address_index, u32 is_delete);
 213
 214 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 215   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 216   u32 address_length, u32 if_address_index, u32 is_delete);
 217
 218 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 219                                          u32 sw_if_index, u32 new_fib_index,
 220                                          u32 old_fib_index);
 221
 222 void
 223 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 224 {
 225   vlib_node_t *node;
 226   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 227   nm->out2in_node_index = node->index;
 228   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 229   nm->in2out_node_index = node->index;
 230   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 231   nm->in2out_output_node_index = node->index;
 232 }
 233
 234 int
 235 nat44_ei_set_workers (uword *bitmap)
 236 {
 237   nat44_ei_main_t *nm = &nat44_ei_main;
 238   int i, j = 0;
 239
 240   if (nm->num_workers < 2)
 241     return VNET_API_ERROR_FEATURE_DISABLED;
 242
 243   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 244     return VNET_API_ERROR_INVALID_WORKER;
 245
 246   vec_free (nm->workers);
 247   clib_bitmap_foreach (i, bitmap)
 248     {
 249       vec_add1 (nm->workers, i);
 250       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 251       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 252       j++;
 253     }
 254
 255   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 256
 257   return 0;
 258 }
 259
 260 #define nat_validate_simple_counter(c, i)                                     \
 261   do                                                                          \
 262     {                                                                         \
 263       vlib_validate_simple_counter (&c, i);                                   \
 264       vlib_zero_simple_counter (&c, i);                                       \
 265     }                                                                         \
 266   while (0);
 267
 268 #define nat_init_simple_counter(c, n, sn)                                     \
 269   do                                                                          \
 270     {                                                                         \
 271       c.name = n;                                                             \
 272       c.stat_segment_name = sn;                                               \
 273       nat_validate_simple_counter (c, 0);                                     \
 274     }                                                                         \
 275   while (0);
 276
 277 static_always_inline void
 278 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 279 {
 280 #define _(x)                                                                  \
 281   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 282   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 283   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 284   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 285   foreach_nat_counter;
 286 #undef _
 287   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 288 }
 289
 290 static void
 291 nat44_ei_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 292 {
 293   nat44_ei_main_t *nm = &nat44_ei_main;
 294   nat44_ei_interface_t *i;
 295
 296   pool_foreach (i, nm->interfaces)
 297     {
 298       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 299         {
 300           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 301         }
 302     }
 303   pool_foreach (i, nm->output_feature_interfaces)
 304     {
 305       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 306         {
 307           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 308         }
 309     }
 310 }
 311
 312 static_always_inline void
 313 nat44_ei_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 314 {
 315   nat44_ei_main_t *nm = &nat44_ei_main;
 316   nat44_ei_address_t *ap;
 317
 318   vec_foreach (ap, nm->addresses)
 319     {
 320       nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 321     }
 322 }
 323
 324 static_always_inline void
 325 nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 326 {
 327   nat44_ei_main_t *nm = &nat44_ei_main;
 328   nat44_ei_static_mapping_t *m;
 329
 330   pool_foreach (m, nm->static_mappings)
 331     {
 332       if (is_sm_addr_only (m->flags) &&
 333           !(m->local_addr.as_u32 == m->external_addr.as_u32))
 334         {
 335           nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 336                                         is_add);
 337         }
 338     }
 339 }
 340
 341 static int
 342 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
 343 {
 344   nat44_ei_main_t *nm = &nat44_ei_main;
 345   nat44_ei_static_mapping_t *m;
 346   pool_foreach (m, nm->static_mappings)
 347     {
 348       if (is_sm_addr_only (m->flags) || is_sm_identity_nat (m->flags))
 349         {
 350           continue;
 351         }
 352       if (m->external_addr.as_u32 == addr.as_u32)
 353         {
 354           return 1;
 355         }
 356     }
 357   return 0;
 358 }
 359
 360 clib_error_t *
 361 nat44_ei_init (vlib_main_t *vm)
 362 {
 363   nat44_ei_main_t *nm = &nat44_ei_main;
 364   vlib_thread_main_t *tm = vlib_get_thread_main ();
 365   vlib_thread_registration_t *tr;
 366   ip4_add_del_interface_address_callback_t cbi = { 0 };
 367   ip4_table_bind_callback_t cbt = { 0 };
 368   u32 i, num_threads = 0;
 369   uword *p, *bitmap = 0;
 370
 371   clib_memset (nm, 0, sizeof (*nm));
 372
 373   // required
 374   nm->vnet_main = vnet_get_main ();
 375   // convenience
 376   nm->ip4_main = &ip4_main;
 377   nm->api_main = vlibapi_get_main ();
 378   nm->ip4_lookup_main = &ip4_main.lookup_main;
 379
 380   // handoff stuff
 381   nm->fq_out2in_index = ~0;
 382   nm->fq_in2out_index = ~0;
 383   nm->fq_in2out_output_index = ~0;
 384
 385   nm->log_level = NAT_LOG_ERROR;
 386
 387   nat44_ei_set_node_indexes (nm, vm);
 388   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 389
 390   nat_init_simple_counter (nm->total_users, "total-users",
 391                            "/nat44-ei/total-users");
 392   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 393                            "/nat44-ei/total-sessions");
 394   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 395                            "/nat44-ei/user-limit-reached");
 396
 397 #define _(x)                                                                  \
 398   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 399                            "/nat44-ei/in2out/fastpath/" #x);                  \
 400   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 401                            "/nat44-ei/out2in/fastpath/" #x);                  \
 402   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 403                            "/nat44-ei/in2out/slowpath/" #x);                  \
 404   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 405                            "/nat44-ei/out2in/slowpath/" #x);
 406   foreach_nat_counter;
 407 #undef _
 408   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 409                            "/nat44-ei/hairpinning");
 410
 411   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 412   if (p)
 413     {
 414       tr = (vlib_thread_registration_t *) p[0];
 415       if (tr)
 416         {
 417           nm->num_workers = tr->count;
 418           nm->first_worker_index = tr->first_index;
 419         }
 420     }
 421   num_threads = tm->n_vlib_mains - 1;
 422   nm->port_per_thread = 0xffff - 1024;
 423   vec_validate (nm->per_thread_data, num_threads);
 424
 425   /* Use all available workers by default */
 426   if (nm->num_workers > 1)
 427     {
 428       for (i = 0; i < nm->num_workers; i++)
 429         bitmap = clib_bitmap_set (bitmap, i, 1);
 430       nat44_ei_set_workers (bitmap);
 431       clib_bitmap_free (bitmap);
 432     }
 433   else
 434     {
 435       nm->per_thread_data[0].snat_thread_index = 0;
 436     }
 437
 438   /* callbacks to call when interface address changes. */
 439   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 440   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 441   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 442   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 443
 444   /* callbacks to call when interface to table biding changes */
 445   cbt.function = nat44_ei_update_outside_fib;
 446   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 447
 448   nm->fib_src_low = fib_source_allocate (
 449     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 450   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 451                                         FIB_SOURCE_BH_SIMPLE);
 452
 453   // used only by out2in-dpo feature
 454   nat_dpo_module_init ();
 455   nat_ha_init (vm, nm->num_workers, num_threads);
 456
 457   nm->hairpinning_fq_index =
 458     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 459   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 460     vlib_frame_queue_main_init (
 461       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 462   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 463     vlib_frame_queue_main_init (
 464       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 465   return nat44_ei_api_hookup (vm);
 466 }
 467
 468 VLIB_INIT_FUNCTION (nat44_ei_init);
 469
 470 int
 471 nat44_ei_plugin_enable (nat44_ei_config_t c)
 472 {
 473   nat44_ei_main_t *nm = &nat44_ei_main;
 474
 475   fail_if_enabled ();
 476
 477   if (!c.users)
 478     c.users = 1024;
 479
 480   if (!c.sessions)
 481     c.sessions = 10 * 1024;
 482
 483   if (!c.user_sessions)
 484     c.user_sessions = c.sessions;
 485
 486   nm->rconfig = c;
 487
 488   if (!nm->frame_queue_nelts)
 489     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 490
 491   nm->translations = c.sessions;
 492   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 493   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 494
 495   nm->pat = (!c.static_mapping_only ||
 496              (c.static_mapping_only && c.connection_tracking));
 497
 498   nm->static_mapping_only = c.static_mapping_only;
 499   nm->static_mapping_connection_tracking = c.connection_tracking;
 500   nm->out2in_dpo = c.out2in_dpo;
 501   nm->forwarding_enabled = 0;
 502   nm->mss_clamping = 0;
 503
 504   nm->max_users_per_thread = c.users;
 505   nm->max_translations_per_thread = c.sessions;
 506   nm->max_translations_per_user = c.user_sessions;
 507
 508   nm->inside_vrf_id = c.inside_vrf;
 509   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 510     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 511
 512   nm->outside_vrf_id = c.outside_vrf;
 513   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 514     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 515
 516   nat_reset_timeouts (&nm->timeouts);
 517   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 518                     nm->user_buckets);
 519   nat44_ei_set_alloc_default ();
 520
 521   vlib_zero_simple_counter (&nm->total_users, 0);
 522   vlib_zero_simple_counter (&nm->total_sessions, 0);
 523   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 524
 525   if (nm->num_workers > 1)
 526     {
 527       if (nm->fq_in2out_index == ~0)
 528         {
 529           nm->fq_in2out_index = vlib_frame_queue_main_init (
 530             nm->in2out_node_index, nm->frame_queue_nelts);
 531         }
 532       if (nm->fq_out2in_index == ~0)
 533         {
 534           nm->fq_out2in_index = vlib_frame_queue_main_init (
 535             nm->out2in_node_index, nm->frame_queue_nelts);
 536         }
 537       if (nm->fq_in2out_output_index == ~0)
 538         {
 539           nm->fq_in2out_output_index = vlib_frame_queue_main_init (
 540             nm->in2out_output_node_index, nm->frame_queue_nelts);
 541         }
 542     }
 543
 544   nat_ha_enable ();
 545   nm->enabled = 1;
 546
 547   return 0;
 548 }
 549
 550 static_always_inline nat44_ei_outside_fib_t *
 551 nat44_ei_get_outside_fib (nat44_ei_outside_fib_t *outside_fibs, u32 fib_index)
 552 {
 553   nat44_ei_outside_fib_t *f;
 554   vec_foreach (f, outside_fibs)
 555     {
 556       if (f->fib_index == fib_index)
 557         {
 558           return f;
 559         }
 560     }
 561   return 0;
 562 }
 563
 564 static_always_inline nat44_ei_interface_t *
 565 nat44_ei_get_interface (nat44_ei_interface_t *interfaces, u32 sw_if_index)
 566 {
 567   nat44_ei_interface_t *i;
 568   pool_foreach (i, interfaces)
 569     {
 570       if (i->sw_if_index == sw_if_index)
 571         {
 572           return i;
 573         }
 574     }
 575   return 0;
 576 }
 577
 578 static_always_inline int
 579 nat44_ei_hairpinning_enable (u8 is_enable)
 580 {
 581   nat44_ei_main_t *nm = &nat44_ei_main;
 582   u32 sw_if_index = 0; // local0
 583
 584   if (is_enable)
 585     {
 586       nm->hairpin_reg += 1;
 587       if (1 == nm->hairpin_reg)
 588         {
 589           return vnet_feature_enable_disable (
 590             "ip4-local", "nat44-ei-hairpinning", sw_if_index, is_enable, 0, 0);
 591         }
 592     }
 593   else
 594     {
 595       if (0 == nm->hairpin_reg)
 596         return 1;
 597
 598       nm->hairpin_reg -= 1;
 599       if (0 == nm->hairpin_reg)
 600         {
 601           return vnet_feature_enable_disable (
 602             "ip4-local", "nat44-ei-hairpinning", sw_if_index, is_enable, 0, 0);
 603         }
 604     }
 605
 606   return 0;
 607 }
 608
 609 int
 610 nat44_ei_add_interface (u32 sw_if_index, u8 is_inside)
 611 {
 612   const char *feature_name, *del_feature_name;
 613   nat44_ei_main_t *nm = &nat44_ei_main;
 614
 615   nat44_ei_outside_fib_t *outside_fib;
 616   nat44_ei_interface_t *i;
 617   u32 fib_index;
 618   int rv;
 619
 620   fail_if_disabled ();
 621
 622   if (nm->out2in_dpo && !is_inside)
 623     {
 624       nat44_ei_log_err ("error unsupported");
 625       return VNET_API_ERROR_UNSUPPORTED;
 626     }
 627
 628   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 629     {
 630       nat44_ei_log_err ("error interface already configured");
 631       return VNET_API_ERROR_VALUE_EXIST;
 632     }
 633
 634   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 635   if (i)
 636     {
 637       if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 638           (nat44_ei_interface_is_outside (i) && !is_inside))
 639         {
 640           return 0;
 641         }
 642       if (nm->num_workers > 1)
 643         {
 644           del_feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 645                                           "nat44-ei-out2in-worker-handoff";
 646           feature_name = "nat44-ei-handoff-classify";
 647         }
 648       else
 649         {
 650           del_feature_name =
 651             !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 652
 653           feature_name = "nat44-ei-classify";
 654         }
 655
 656       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 657       if (rv)
 658         {
 659           return rv;
 660         }
 661       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 662                                         sw_if_index, 0, 0, 0);
 663       if (rv)
 664         {
 665           return rv;
 666         }
 667       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 668                                         sw_if_index, 1, 0, 0);
 669       if (rv)
 670         {
 671           return rv;
 672         }
 673       if (!is_inside)
 674         {
 675           rv = nat44_ei_hairpinning_enable (0);
 676           if (rv)
 677             {
 678               return rv;
 679             }
 680         }
 681     }
 682   else
 683     {
 684       if (nm->num_workers > 1)
 685         {
 686           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 687                                      "nat44-ei-out2in-worker-handoff";
 688         }
 689       else
 690         {
 691           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 692         }
 693       nat_validate_interface_counters (nm, sw_if_index);
 694
 695       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 696       if (rv)
 697         {
 698           return rv;
 699         }
 700       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 701                                         sw_if_index, 1, 0, 0);
 702       if (rv)
 703         {
 704           return rv;
 705         }
 706       if (is_inside && !nm->out2in_dpo)
 707         {
 708           rv = nat44_ei_hairpinning_enable (1);
 709           if (rv)
 710             {
 711               return rv;
 712             }
 713         }
 714
 715       pool_get (nm->interfaces, i);
 716       i->sw_if_index = sw_if_index;
 717       i->flags = 0;
 718     }
 719
 720   fib_index =
 721     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 722
 723   if (!is_inside)
 724     {
 725       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 726
 727       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 728       if (outside_fib)
 729         {
 730           outside_fib->refcount++;
 731         }
 732       else
 733         {
 734           vec_add2 (nm->outside_fibs, outside_fib, 1);
 735           outside_fib->fib_index = fib_index;
 736           outside_fib->refcount = 1;
 737         }
 738
 739       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 740       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 741     }
 742   else
 743     {
 744       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 745     }
 746
 747   return 0;
 748 }
 749
 750 int
 751 nat44_ei_del_interface (u32 sw_if_index, u8 is_inside)
 752 {
 753   const char *feature_name, *del_feature_name;
 754   nat44_ei_main_t *nm = &nat44_ei_main;
 755
 756   nat44_ei_outside_fib_t *outside_fib;
 757   nat44_ei_interface_t *i;
 758   u32 fib_index;
 759   int rv;
 760
 761   fail_if_disabled ();
 762
 763   if (nm->out2in_dpo && !is_inside)
 764     {
 765       nat44_ei_log_err ("error unsupported");
 766       return VNET_API_ERROR_UNSUPPORTED;
 767     }
 768
 769   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 770   if (i == 0)
 771     {
 772       nat44_ei_log_err ("error interface couldn't be found");
 773       return VNET_API_ERROR_NO_SUCH_ENTRY;
 774     }
 775
 776   if (nat44_ei_interface_is_inside (i) && nat44_ei_interface_is_outside (i))
 777     {
 778       if (nm->num_workers > 1)
 779         {
 780           del_feature_name = "nat44-ei-handoff-classify";
 781           feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 782                                       "nat44-ei-out2in-worker-handoff";
 783         }
 784       else
 785         {
 786           del_feature_name = "nat44-ei-classify";
 787           feature_name = !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 788         }
 789
 790       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 791       if (rv)
 792         {
 793           return rv;
 794         }
 795       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 796                                         sw_if_index, 0, 0, 0);
 797       if (rv)
 798         {
 799           return rv;
 800         }
 801       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 802                                         sw_if_index, 1, 0, 0);
 803       if (rv)
 804         {
 805           return rv;
 806         }
 807       if (is_inside)
 808         {
 809           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 810         }
 811       else
 812         {
 813           rv = nat44_ei_hairpinning_enable (1);
 814           if (rv)
 815             {
 816               return rv;
 817             }
 818           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 819         }
 820     }
 821   else
 822     {
 823       if (nm->num_workers > 1)
 824         {
 825           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 826                                      "nat44-ei-out2in-worker-handoff";
 827         }
 828       else
 829         {
 830           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 831         }
 832
 833       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 834       if (rv)
 835         {
 836           return rv;
 837         }
 838       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 839                                         sw_if_index, 0, 0, 0);
 840       if (rv)
 841         {
 842           return rv;
 843         }
 844       if (is_inside)
 845         {
 846           rv = nat44_ei_hairpinning_enable (0);
 847           if (rv)
 848             {
 849               return rv;
 850             }
 851         }
 852
 853       // remove interface
 854       pool_put (nm->interfaces, i);
 855     }
 856
 857   if (!is_inside)
 858     {
 859       fib_index =
 860         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 861       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 862       if (outside_fib)
 863         {
 864           outside_fib->refcount--;
 865           if (!outside_fib->refcount)
 866             {
 867               vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
 868             }
 869         }
 870
 871       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
 872       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
 873     }
 874
 875   return 0;
 876 }
 877
 878 int
 879 nat44_ei_add_output_interface (u32 sw_if_index)
 880 {
 881   nat44_ei_main_t *nm = &nat44_ei_main;
 882
 883   nat44_ei_outside_fib_t *outside_fib;
 884   nat44_ei_interface_t *i;
 885   u32 fib_index;
 886   int rv;
 887
 888   fail_if_disabled ();
 889
 890   if (nat44_ei_get_interface (nm->interfaces, sw_if_index))
 891     {
 892       nat44_ei_log_err ("error interface already configured");
 893       return VNET_API_ERROR_VALUE_EXIST;
 894     }
 895
 896   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 897     {
 898       nat44_ei_log_err ("error interface already configured");
 899       return VNET_API_ERROR_VALUE_EXIST;
 900     }
 901
 902   if (nm->num_workers > 1)
 903     {
 904       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 905       if (rv)
 906         {
 907           return rv;
 908         }
 909       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 910       if (rv)
 911         {
 912           return rv;
 913         }
 914       rv = vnet_feature_enable_disable (
 915         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 1, 0, 0);
 916       if (rv)
 917         {
 918           return rv;
 919         }
 920       rv = vnet_feature_enable_disable (
 921         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 1,
 922         0, 0);
 923       if (rv)
 924         {
 925           return rv;
 926         }
 927     }
 928   else
 929     {
 930       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 931       if (rv)
 932         {
 933           return rv;
 934         }
 935       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 936       if (rv)
 937         {
 938           return rv;
 939         }
 940       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 941                                         sw_if_index, 1, 0, 0);
 942       if (rv)
 943         {
 944           return rv;
 945         }
 946       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 947                                         sw_if_index, 1, 0, 0);
 948       if (rv)
 949         {
 950           return rv;
 951         }
 952     }
 953
 954   nat_validate_interface_counters (nm, sw_if_index);
 955
 956   pool_get (nm->output_feature_interfaces, i);
 957   i->sw_if_index = sw_if_index;
 958   i->flags = 0;
 959   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 960   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 961
 962   fib_index =
 963     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 964   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 965   if (outside_fib)
 966     {
 967       outside_fib->refcount++;
 968     }
 969   else
 970     {
 971       vec_add2 (nm->outside_fibs, outside_fib, 1);
 972       outside_fib->fib_index = fib_index;
 973       outside_fib->refcount = 1;
 974     }
 975
 976   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 977   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 978
 979   return 0;
 980 }
 981
 982 int
 983 nat44_ei_del_output_interface (u32 sw_if_index)
 984 {
 985   nat44_ei_main_t *nm = &nat44_ei_main;
 986
 987   nat44_ei_outside_fib_t *outside_fib;
 988   nat44_ei_interface_t *i;
 989   u32 fib_index;
 990   int rv;
 991
 992   fail_if_disabled ();
 993
 994   i = nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index);
 995   if (!i)
 996     {
 997       nat44_ei_log_err ("error interface couldn't be found");
 998       return VNET_API_ERROR_NO_SUCH_ENTRY;
 999     }
1000
1001   if (nm->num_workers > 1)
1002     {
1003       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1004       if (rv)
1005         {
1006           return rv;
1007         }
1008       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1009       if (rv)
1010         {
1011           return rv;
1012         }
1013       rv = vnet_feature_enable_disable (
1014         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 0, 0, 0);
1015       if (rv)
1016         {
1017           return rv;
1018         }
1019       rv = vnet_feature_enable_disable (
1020         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 0,
1021         0, 0);
1022       if (rv)
1023         {
1024           return rv;
1025         }
1026     }
1027   else
1028     {
1029       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1030       if (rv)
1031         {
1032           return rv;
1033         }
1034       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1035       if (rv)
1036         {
1037           return rv;
1038         }
1039       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1040                                         sw_if_index, 0, 0, 0);
1041       if (rv)
1042         {
1043           return rv;
1044         }
1045       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1046                                         sw_if_index, 0, 0, 0);
1047       if (rv)
1048         {
1049           return rv;
1050         }
1051     }
1052
1053   pool_put (nm->output_feature_interfaces, i);
1054
1055   fib_index =
1056     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1057   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1058   if (outside_fib)
1059     {
1060       outside_fib->refcount--;
1061       if (!outside_fib->refcount)
1062         {
1063           vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
1064         }
1065     }
1066
1067   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
1068   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
1069
1070   return 0;
1071 }
1072
1073 int
1074 nat44_ei_add_del_output_interface (u32 sw_if_index, int is_del)
1075 {
1076   if (is_del)
1077     {
1078       return nat44_ei_del_output_interface (sw_if_index);
1079     }
1080   else
1081     {
1082       return nat44_ei_add_output_interface (sw_if_index);
1083     }
1084 }
1085
1086 int
1087 nat44_ei_del_addresses ()
1088 {
1089   nat44_ei_main_t *nm = &nat44_ei_main;
1090   nat44_ei_address_t *a, *vec;
1091   int error = 0;
1092
1093   vec = vec_dup (nm->addresses);
1094   vec_foreach (a, vec)
1095     {
1096       error = nat44_ei_del_address (a->addr, 0);
1097
1098       if (error)
1099         {
1100           nat44_ei_log_err ("error occurred while removing adderess");
1101         }
1102     }
1103   vec_free (vec);
1104   vec_free (nm->addresses);
1105   nm->addresses = 0;
1106
1107   vec_free (nm->auto_add_sw_if_indices);
1108   nm->auto_add_sw_if_indices = 0;
1109   return error;
1110 }
1111
1112 int
1113 nat44_ei_del_interfaces ()
1114 {
1115   nat44_ei_main_t *nm = &nat44_ei_main;
1116   nat44_ei_interface_t *i, *pool;
1117   int error = 0;
1118
1119   pool = pool_dup (nm->interfaces);
1120   pool_foreach (i, pool)
1121     {
1122       if (nat44_ei_interface_is_inside (i))
1123         {
1124           error = nat44_ei_del_interface (i->sw_if_index, 1);
1125         }
1126       if (nat44_ei_interface_is_outside (i))
1127         {
1128           error = nat44_ei_del_interface (i->sw_if_index, 0);
1129         }
1130
1131       if (error)
1132         {
1133           nat44_ei_log_err ("error occurred while removing interface");
1134         }
1135     }
1136   pool_free (pool);
1137   pool_free (nm->interfaces);
1138   nm->interfaces = 0;
1139   return error;
1140 }
1141
1142 int
1143 nat44_ei_del_output_interfaces ()
1144 {
1145   nat44_ei_main_t *nm = &nat44_ei_main;
1146   nat44_ei_interface_t *i, *pool;
1147   int error = 0;
1148
1149   pool = pool_dup (nm->output_feature_interfaces);
1150   pool_foreach (i, pool)
1151     {
1152       error = nat44_ei_del_output_interface (i->sw_if_index);
1153       if (error)
1154         {
1155           nat44_ei_log_err ("error occurred while removing output interface");
1156         }
1157     }
1158   pool_free (pool);
1159   pool_free (nm->output_feature_interfaces);
1160   nm->output_feature_interfaces = 0;
1161   return error;
1162 }
1163
1164 int
1165 nat44_ei_del_static_mappings ()
1166 {
1167   nat44_ei_main_t *nm = &nat44_ei_main;
1168   nat44_ei_static_mapping_t *m, *pool;
1169   int error = 0;
1170
1171   pool = pool_dup (nm->static_mappings);
1172   pool_foreach (m, pool)
1173     {
1174       error = nat44_ei_del_static_mapping_internal (
1175         m->local_addr, m->external_addr, m->local_port, m->external_port,
1176         m->proto, m->vrf_id, ~0, m->flags);
1177       if (error)
1178         {
1179           nat44_ei_log_err ("error occurred while removing mapping");
1180         }
1181     }
1182   pool_free (pool);
1183   pool_free (nm->static_mappings);
1184   nm->static_mappings = 0;
1185
1186   vec_free (nm->to_resolve);
1187   nm->to_resolve = 0;
1188
1189   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
1190   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
1191
1192   return error;
1193 }
1194
1195 int
1196 nat44_ei_plugin_disable ()
1197 {
1198   nat44_ei_main_t *nm = &nat44_ei_main;
1199   nat44_ei_main_per_thread_data_t *tnm;
1200   int rc, error = 0;
1201
1202   nat_ha_disable ();
1203
1204   rc = nat44_ei_del_static_mappings ();
1205   if (rc)
1206     error = 1;
1207
1208   rc = nat44_ei_del_addresses ();
1209   if (rc)
1210     error = 1;
1211
1212   rc = nat44_ei_del_interfaces ();
1213   if (rc)
1214     error = 1;
1215
1216   rc = nat44_ei_del_output_interfaces ();
1217   if (rc)
1218     error = 1;
1219
1220   if (nm->pat)
1221     {
1222       clib_bihash_free_8_8 (&nm->in2out);
1223       clib_bihash_free_8_8 (&nm->out2in);
1224
1225       vec_foreach (tnm, nm->per_thread_data)
1226         {
1227           nat44_ei_worker_db_free (tnm);
1228         }
1229     }
1230
1231   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
1232
1233   nm->forwarding_enabled = 0;
1234   nm->enabled = 0;
1235
1236   return error;
1237 }
1238
1239 int
1240 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
1241                                        u32 thread_index, ip4_address_t addr,
1242                                        u16 port, nat_protocol_t protocol)
1243 {
1244   nat44_ei_main_t *nm = &nat44_ei_main;
1245   nat44_ei_address_t *a = 0;
1246   u32 address_index;
1247   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1248
1249   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1250     {
1251       if (addresses[address_index].addr.as_u32 != addr.as_u32)
1252         continue;
1253
1254       a = addresses + address_index;
1255       switch (protocol)
1256         {
1257 #define _(N, j, n, s)                                                         \
1258   case NAT_PROTOCOL_##N:                                                      \
1259     if (a->busy_##n##_port_refcounts[port_host_byte_order])                   \
1260       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
1261     ++a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1262     a->busy_##n##_ports_per_thread[thread_index]++;                           \
1263     a->busy_##n##_ports++;                                                    \
1264     return 0;
1265           foreach_nat_protocol
1266 #undef _
1267             default : nat_elog_info (nm, "unknown protocol");
1268           return 1;
1269         }
1270     }
1271
1272   return VNET_API_ERROR_NO_SUCH_ENTRY;
1273 }
1274
1275 void
1276 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
1277 {
1278   nat44_ei_main_t *nm = &nat44_ei_main;
1279   dpo_id_t dpo_v4 = DPO_INVALID;
1280   fib_prefix_t pfx = {
1281     .fp_proto = FIB_PROTOCOL_IP4,
1282     .fp_len = 32,
1283     .fp_addr.ip4.as_u32 = addr.as_u32,
1284   };
1285
1286   if (is_add)
1287     {
1288       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
1289       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
1290                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
1291       dpo_reset (&dpo_v4);
1292     }
1293   else
1294     {
1295       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
1296     }
1297 }
1298
1299 void
1300 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1301                                         u32 thread_index, ip4_address_t *addr,
1302                                         u16 port, nat_protocol_t protocol)
1303 {
1304   nat44_ei_main_t *nm = &nat44_ei_main;
1305   nat44_ei_address_t *a;
1306   u32 address_index;
1307   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1308
1309   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1310     {
1311       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1312         break;
1313     }
1314
1315   ASSERT (address_index < vec_len (addresses));
1316
1317   a = addresses + address_index;
1318
1319   switch (protocol)
1320     {
1321 #define _(N, i, n, s)                                                         \
1322   case NAT_PROTOCOL_##N:                                                      \
1323     ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1);         \
1324     --a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1325     a->busy_##n##_ports--;                                                    \
1326     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1327     break;
1328       foreach_nat_protocol
1329 #undef _
1330         default : nat_elog_info (nm, "unknown protocol");
1331       return;
1332     }
1333 }
1334
1335 void
1336 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1337                                u32 thread_index, u8 is_ha)
1338 {
1339   clib_bihash_kv_8_8_t kv;
1340
1341   /* session lookup tables */
1342   init_nat_i2o_k (&kv, s);
1343   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1344     nat_elog_warn (nm, "in2out key del failed");
1345   init_nat_o2i_k (&kv, s);
1346   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1347     nat_elog_warn (nm, "out2in key del failed");
1348
1349   if (!is_ha)
1350     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1351                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1352                              s->out2in.port, s->nat_proto);
1353
1354   if (nat44_ei_is_unk_proto_session (s))
1355     return;
1356
1357   if (!is_ha)
1358     {
1359       /* log NAT event */
1360       nat_ipfix_logging_nat44_ses_delete (
1361         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1362         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1363         s->in2out.fib_index);
1364
1365       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1366                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1367                    thread_index);
1368     }
1369
1370   if (nat44_ei_is_session_static (s))
1371     return;
1372
1373   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1374                                           &s->out2in.addr, s->out2in.port,
1375                                           s->nat_proto);
1376 }
1377
1378 nat44_ei_user_t *
1379 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1380                              u32 fib_index, u32 thread_index)
1381 {
1382   nat44_ei_user_t *u = 0;
1383   nat44_ei_user_key_t user_key;
1384   clib_bihash_kv_8_8_t kv, value;
1385   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1386   dlist_elt_t *per_user_list_head_elt;
1387
1388   user_key.addr.as_u32 = addr->as_u32;
1389   user_key.fib_index = fib_index;
1390   kv.key = user_key.as_u64;
1391
1392   /* Ever heard of the "user" = src ip4 address before? */
1393   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1394     {
1395       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1396         {
1397           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1398                                          0, 1);
1399           nat_elog_warn (nm, "maximum user limit reached");
1400           return NULL;
1401         }
1402       /* no, make a new one */
1403       pool_get (tnm->users, u);
1404       clib_memset (u, 0, sizeof (*u));
1405
1406       u->addr.as_u32 = addr->as_u32;
1407       u->fib_index = fib_index;
1408
1409       pool_get (tnm->list_pool, per_user_list_head_elt);
1410
1411       u->sessions_per_user_list_head_index =
1412         per_user_list_head_elt - tnm->list_pool;
1413
1414       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1415
1416       kv.value = u - tnm->users;
1417
1418       /* add user */
1419       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1420         {
1421           nat_elog_warn (nm, "user_hash key add failed");
1422           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1423           return NULL;
1424         }
1425
1426       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1427                                pool_elts (tnm->users));
1428     }
1429   else
1430     {
1431       u = pool_elt_at_index (tnm->users, value.value);
1432     }
1433
1434   return u;
1435 }
1436
1437 nat44_ei_session_t *
1438 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1439                                    u32 thread_index, f64 now)
1440 {
1441   nat44_ei_session_t *s;
1442   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1443   u32 oldest_per_user_translation_list_index, session_index;
1444   dlist_elt_t *oldest_per_user_translation_list_elt;
1445   dlist_elt_t *per_user_translation_list_elt;
1446
1447   /* Over quota? Recycle the least recently used translation */
1448   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1449     {
1450       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1451         tnm->list_pool, u->sessions_per_user_list_head_index);
1452
1453       ASSERT (oldest_per_user_translation_list_index != ~0);
1454
1455       /* Add it back to the end of the LRU list */
1456       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1457                           oldest_per_user_translation_list_index);
1458       /* Get the list element */
1459       oldest_per_user_translation_list_elt = pool_elt_at_index (
1460         tnm->list_pool, oldest_per_user_translation_list_index);
1461
1462       /* Get the session index from the list element */
1463       session_index = oldest_per_user_translation_list_elt->value;
1464
1465       /* Get the session */
1466       s = pool_elt_at_index (tnm->sessions, session_index);
1467
1468       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1469       if (nat44_ei_is_session_static (s))
1470         u->nstaticsessions--;
1471       else
1472         u->nsessions--;
1473       s->flags = 0;
1474       s->total_bytes = 0;
1475       s->total_pkts = 0;
1476       s->state = 0;
1477       s->ext_host_addr.as_u32 = 0;
1478       s->ext_host_port = 0;
1479       s->ext_host_nat_addr.as_u32 = 0;
1480       s->ext_host_nat_port = 0;
1481     }
1482   else
1483     {
1484       pool_get (tnm->sessions, s);
1485       clib_memset (s, 0, sizeof (*s));
1486
1487       /* Create list elts */
1488       pool_get (tnm->list_pool, per_user_translation_list_elt);
1489       clib_dlist_init (tnm->list_pool,
1490                        per_user_translation_list_elt - tnm->list_pool);
1491
1492       per_user_translation_list_elt->value = s - tnm->sessions;
1493       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1494       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1495
1496       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1497                           per_user_translation_list_elt - tnm->list_pool);
1498
1499       s->user_index = u - tnm->users;
1500       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1501                                pool_elts (tnm->sessions));
1502     }
1503
1504   s->ha_last_refreshed = now;
1505
1506   return s;
1507 }
1508
1509 void
1510 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1511                             u32 thread_index, u8 is_ha)
1512 {
1513   clib_bihash_kv_8_8_t kv;
1514
1515   init_nat_i2o_k (&kv, s);
1516   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1517     nat_elog_warn (nm, "in2out key del failed");
1518
1519   init_nat_o2i_k (&kv, s);
1520   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1521     nat_elog_warn (nm, "out2in key del failed");
1522
1523   if (!is_ha)
1524     {
1525       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1526                                &s->in2out.addr, s->in2out.port,
1527                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1528
1529       nat_ipfix_logging_nat44_ses_delete (
1530         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1531         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1532         s->in2out.fib_index);
1533
1534       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1535                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1536                    thread_index);
1537     }
1538
1539   if (nat44_ei_is_session_static (s))
1540     return;
1541
1542   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1543                                           &s->out2in.addr, s->out2in.port,
1544                                           s->nat_proto);
1545 }
1546
1547 static_always_inline void
1548 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1549 {
1550   dlist_elt_t *elt;
1551   nat44_ei_session_t *s;
1552
1553   nat44_ei_main_t *nm = &nat44_ei_main;
1554   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1555
1556   // get head
1557   elt =
1558     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1559   // get first element
1560   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1561
1562   while (elt->value != ~0)
1563     {
1564       s = pool_elt_at_index (tnm->sessions, elt->value);
1565       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1566
1567       nat44_ei_free_session_data (nm, s, thread_index, 0);
1568       nat44_ei_delete_session (nm, s, thread_index);
1569     }
1570 }
1571
1572 int
1573 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1574 {
1575   int rv = 1;
1576
1577   nat44_ei_main_t *nm = &nat44_ei_main;
1578   nat44_ei_main_per_thread_data_t *tnm;
1579
1580   nat44_ei_user_key_t user_key;
1581   clib_bihash_kv_8_8_t kv, value;
1582
1583   user_key.addr.as_u32 = addr->as_u32;
1584   user_key.fib_index = fib_index;
1585   kv.key = user_key.as_u64;
1586
1587   if (nm->num_workers > 1)
1588     {
1589       vec_foreach (tnm, nm->per_thread_data)
1590         {
1591           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1592             {
1593               nat44_ei_user_del_sessions (
1594                 pool_elt_at_index (tnm->users, value.value),
1595                 tnm->thread_index);
1596               rv = 0;
1597               break;
1598             }
1599         }
1600     }
1601   else
1602     {
1603       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1604       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1605         {
1606           nat44_ei_user_del_sessions (
1607             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1608           rv = 0;
1609         }
1610     }
1611   return rv;
1612 }
1613
1614 void
1615 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1616                                       nat44_ei_main_per_thread_data_t *tnm,
1617                                       nat44_ei_user_key_t u_key, int addr_only,
1618                                       ip4_address_t e_addr, u16 e_port)
1619 {
1620   clib_bihash_kv_8_8_t kv, value;
1621   kv.key = u_key.as_u64;
1622   u64 user_index;
1623   dlist_elt_t *head, *elt;
1624   nat44_ei_user_t *u;
1625   nat44_ei_session_t *s;
1626   u32 elt_index, head_index, ses_index;
1627
1628   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1629     {
1630       user_index = value.value;
1631       u = pool_elt_at_index (tnm->users, user_index);
1632       if (u->nstaticsessions)
1633         {
1634           head_index = u->sessions_per_user_list_head_index;
1635           head = pool_elt_at_index (tnm->list_pool, head_index);
1636           elt_index = head->next;
1637           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1638           ses_index = elt->value;
1639           while (ses_index != ~0)
1640             {
1641               s = pool_elt_at_index (tnm->sessions, ses_index);
1642               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1643               ses_index = elt->value;
1644
1645               if (!addr_only)
1646                 {
1647                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1648                       (s->out2in.port != e_port))
1649                     continue;
1650                 }
1651
1652               if (!nat44_ei_is_session_static (s))
1653                 continue;
1654
1655               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1656                                              0);
1657               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1658
1659               if (!addr_only)
1660                 break;
1661             }
1662         }
1663     }
1664 }
1665
1666 u32
1667 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1668                                   u8 is_output)
1669 {
1670   nat44_ei_main_t *nm = &nat44_ei_main;
1671   u32 next_worker_index = 0;
1672   u32 hash;
1673
1674   next_worker_index = nm->first_worker_index;
1675   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1676          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1677
1678   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1679     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1680   else
1681     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1682
1683   return next_worker_index;
1684 }
1685
1686 u32
1687 nat44_ei_get_thread_idx_by_port (u16 e_port)
1688 {
1689   nat44_ei_main_t *nm = &nat44_ei_main;
1690   u32 thread_idx = nm->num_workers;
1691   if (nm->num_workers > 1)
1692     {
1693       thread_idx = nm->first_worker_index +
1694                    nm->workers[(e_port - 1024) / nm->port_per_thread %
1695                                _vec_len (nm->workers)];
1696     }
1697   return thread_idx;
1698 }
1699
1700 u32
1701 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1702                                   u32 rx_fib_index0, u8 is_output)
1703 {
1704   nat44_ei_main_t *nm = &nat44_ei_main;
1705   udp_header_t *udp;
1706   u16 port;
1707   clib_bihash_kv_8_8_t kv, value;
1708   nat44_ei_static_mapping_t *m;
1709   u32 proto;
1710   u32 next_worker_index = 0;
1711
1712   /* first try static mappings without port */
1713   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1714     {
1715       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1716       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1717                                    &value))
1718         {
1719           m = pool_elt_at_index (nm->static_mappings, value.value);
1720           return m->workers[0];
1721         }
1722     }
1723
1724   proto = ip_proto_to_nat_proto (ip0->protocol);
1725   udp = ip4_next_header (ip0);
1726   port = vnet_buffer (b)->ip.reass.l4_dst_port;
1727
1728   /* unknown protocol */
1729   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1730     {
1731       /* use current thread */
1732       return vlib_get_thread_index ();
1733     }
1734
1735   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1736     {
1737       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1738       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1739       if (!icmp_type_is_error_message (
1740             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1741         port = vnet_buffer (b)->ip.reass.l4_src_port;
1742       else
1743         {
1744           /* if error message, then it's not fragmented and we can access it */
1745           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1746           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1747           void *l4_header = ip4_next_header (inner_ip);
1748           switch (proto)
1749             {
1750             case NAT_PROTOCOL_ICMP:
1751               icmp = (icmp46_header_t *) l4_header;
1752               echo = (icmp_echo_header_t *) (icmp + 1);
1753               port = echo->identifier;
1754               break;
1755             case NAT_PROTOCOL_UDP:
1756             case NAT_PROTOCOL_TCP:
1757               port = ((tcp_udp_header_t *) l4_header)->src_port;
1758               break;
1759             default:
1760               return vlib_get_thread_index ();
1761             }
1762         }
1763     }
1764
1765   /* try static mappings with port */
1766   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1767     {
1768       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1769       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1770                                    &value))
1771         {
1772           m = pool_elt_at_index (nm->static_mappings, value.value);
1773           return m->workers[0];
1774         }
1775     }
1776
1777   /* worker by outside port */
1778   next_worker_index =
1779     nat44_ei_get_thread_idx_by_port (clib_net_to_host_u16 (port));
1780   return next_worker_index;
1781 }
1782
1783 static int
1784 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1785                            u32 thread_index, nat_protocol_t proto,
1786                            ip4_address_t s_addr, ip4_address_t *addr,
1787                            u16 *port, u16 port_per_thread,
1788                            u32 snat_thread_index)
1789 {
1790   nat44_ei_main_t *nm = &nat44_ei_main;
1791   nat44_ei_address_t *a, *ga = 0;
1792   u32 portnum;
1793   int i;
1794
1795   if (vec_len (addresses) > 0)
1796     {
1797
1798       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1799
1800       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1801         {
1802           a = addresses + i;
1803           switch (proto)
1804             {
1805 #define _(N, j, n, s)                                                         \
1806   case NAT_PROTOCOL_##N:                                                      \
1807     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1808       {                                                                       \
1809         if (a->fib_index == fib_index)                                        \
1810           {                                                                   \
1811             while (1)                                                         \
1812               {                                                               \
1813                 portnum = (port_per_thread * snat_thread_index) +             \
1814                           nat_random_port (&nm->random_seed, 0,               \
1815                                            port_per_thread - 1) +             \
1816                           1024;                                               \
1817                 if (a->busy_##n##_port_refcounts[portnum])                    \
1818                   continue;                                                   \
1819                 --a->busy_##n##_port_refcounts[portnum];                      \
1820                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1821                 a->busy_##n##_ports++;                                        \
1822                 *addr = a->addr;                                              \
1823                 *port = clib_host_to_net_u16 (portnum);                       \
1824                 return 0;                                                     \
1825               }                                                               \
1826           }                                                                   \
1827         else if (a->fib_index == ~0)                                          \
1828           {                                                                   \
1829             ga = a;                                                           \
1830           }                                                                   \
1831       }                                                                       \
1832     break;
1833               foreach_nat_protocol;
1834             default:
1835               nat_elog_info (nm, "unknown protocol");
1836               return 1;
1837             }
1838         }
1839
1840       for (i = 0; i < s_addr_offset; ++i)
1841         {
1842           a = addresses + i;
1843           switch (proto)
1844             {
1845               foreach_nat_protocol;
1846             default:
1847               nat_elog_info (nm, "unknown protocol");
1848               return 1;
1849             }
1850         }
1851   if (ga)
1852     {
1853       a = ga;
1854       // fake fib index to reuse macro
1855       fib_index = ~0;
1856       switch (proto)
1857         {
1858           foreach_nat_protocol;
1859             default : nat_elog_info (nm, "unknown protocol");
1860           return 1;
1861         }
1862     }
1863     }
1864
1865 #undef _
1866
1867   /* Totally out of translations to use... */
1868   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1869   return 1;
1870 }
1871
1872 static int
1873 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1874                          u32 thread_index, nat_protocol_t proto,
1875                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1876                          u16 port_per_thread, u32 snat_thread_index)
1877 {
1878   nat44_ei_main_t *nm = &nat44_ei_main;
1879   nat44_ei_address_t *a = addresses;
1880   u16 portnum, ports;
1881
1882   ports = nm->end_port - nm->start_port + 1;
1883
1884   if (!vec_len (addresses))
1885     goto exhausted;
1886
1887   switch (proto)
1888     {
1889 #define _(N, i, n, s)                                                         \
1890   case NAT_PROTOCOL_##N:                                                      \
1891     if (a->busy_##n##_ports < ports)                                          \
1892       {                                                                       \
1893         while (1)                                                             \
1894           {                                                                   \
1895             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1896                                        nm->end_port);                         \
1897             if (a->busy_##n##_port_refcounts[portnum])                        \
1898               continue;                                                       \
1899             ++a->busy_##n##_port_refcounts[portnum];                          \
1900             a->busy_##n##_ports++;                                            \
1901             *addr = a->addr;                                                  \
1902             *port = clib_host_to_net_u16 (portnum);                           \
1903             return 0;                                                         \
1904           }                                                                   \
1905       }                                                                       \
1906     break;
1907       foreach_nat_protocol
1908 #undef _
1909         default : nat_elog_info (nm, "unknown protocol");
1910       return 1;
1911     }
1912
1913 exhausted:
1914   /* Totally out of translations to use... */
1915   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1916   return 1;
1917 }
1918
1919 static int
1920 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1921                         u32 thread_index, nat_protocol_t proto,
1922                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1923                         u16 port_per_thread, u32 snat_thread_index)
1924 {
1925   nat44_ei_main_t *nm = &nat44_ei_main;
1926   nat44_ei_address_t *a = addresses;
1927   u16 m, ports, portnum, A, j;
1928   m = 16 - (nm->psid_offset + nm->psid_length);
1929   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1930
1931   if (!vec_len (addresses))
1932     goto exhausted;
1933
1934   switch (proto)
1935     {
1936 #define _(N, i, n, s)                                                         \
1937   case NAT_PROTOCOL_##N:                                                      \
1938     if (a->busy_##n##_ports < ports)                                          \
1939       {                                                                       \
1940         while (1)                                                             \
1941           {                                                                   \
1942             A = nat_random_port (&nm->random_seed, 1,                         \
1943                                  pow2_mask (nm->psid_offset));                \
1944             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
1945             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
1946             if (a->busy_##n##_port_refcounts[portnum])                        \
1947               continue;                                                       \
1948             ++a->busy_##n##_port_refcounts[portnum];                          \
1949             a->busy_##n##_ports++;                                            \
1950             *addr = a->addr;                                                  \
1951             *port = clib_host_to_net_u16 (portnum);                           \
1952             return 0;                                                         \
1953           }                                                                   \
1954       }                                                                       \
1955     break;
1956       foreach_nat_protocol
1957 #undef _
1958         default : nat_elog_info (nm, "unknown protocol");
1959       return 1;
1960     }
1961
1962 exhausted:
1963   /* Totally out of translations to use... */
1964   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1965   return 1;
1966 }
1967
1968 void
1969 nat44_ei_set_alloc_default ()
1970 {
1971   nat44_ei_main_t *nm = &nat44_ei_main;
1972
1973   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1974   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1975 }
1976
1977 void
1978 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1979 {
1980   nat44_ei_main_t *nm = &nat44_ei_main;
1981
1982   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1983   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1984   nm->start_port = start_port;
1985   nm->end_port = end_port;
1986 }
1987
1988 void
1989 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1990 {
1991   nat44_ei_main_t *nm = &nat44_ei_main;
1992
1993   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1994   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1995   nm->psid = psid;
1996   nm->psid_offset = psid_offset;
1997   nm->psid_length = psid_length;
1998 }
1999
2000 void
2001 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
2002                          u32 thread_index)
2003 {
2004   nat44_ei_main_per_thread_data_t *tnm =
2005     vec_elt_at_index (nm->per_thread_data, thread_index);
2006   clib_bihash_kv_8_8_t kv, value;
2007   nat44_ei_user_t *u;
2008   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
2009                                       .fib_index = ses->in2out.fib_index };
2010   const u8 u_static = nat44_ei_is_session_static (ses);
2011
2012   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
2013   pool_put_index (tnm->list_pool, ses->per_user_index);
2014
2015   pool_put (tnm->sessions, ses);
2016   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
2017                            pool_elts (tnm->sessions));
2018
2019   kv.key = u_key.as_u64;
2020   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2021     {
2022       u = pool_elt_at_index (tnm->users, value.value);
2023       if (u_static)
2024         u->nstaticsessions--;
2025       else
2026         u->nsessions--;
2027
2028       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
2029     }
2030 }
2031
2032 int
2033 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
2034                       nat_protocol_t proto, u32 vrf_id, int is_in)
2035 {
2036   nat44_ei_main_per_thread_data_t *tnm;
2037   clib_bihash_kv_8_8_t kv, value;
2038   u32 fib_index;
2039   nat44_ei_session_t *s;
2040   clib_bihash_8_8_t *t;
2041
2042   fail_if_disabled ();
2043
2044   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
2045   init_nat_k (&kv, *addr, port, fib_index, proto);
2046   t = is_in ? &nm->in2out : &nm->out2in;
2047   if (!clib_bihash_search_8_8 (t, &kv, &value))
2048     {
2049       // this is called from API/CLI, so the world is stopped here
2050       // it's safe to manipulate arbitrary per-thread data
2051       u32 thread_index = nat_value_get_thread_index (&value);
2052       tnm = vec_elt_at_index (nm->per_thread_data, thread_index);
2053       u32 session_index = nat_value_get_session_index (&value);
2054       if (pool_is_free_index (tnm->sessions, session_index))
2055         return VNET_API_ERROR_UNSPECIFIED;
2056
2057       s = pool_elt_at_index (tnm->sessions, session_index);
2058       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
2059       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2060       return 0;
2061     }
2062
2063   return VNET_API_ERROR_NO_SUCH_ENTRY;
2064 }
2065
2066 void
2067 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
2068                               int is_add)
2069 {
2070   nat44_ei_main_t *nm = &nat44_ei_main;
2071   fib_prefix_t prefix = {
2072     .fp_len = p_len,
2073     .fp_proto = FIB_PROTOCOL_IP4,
2074     .fp_addr = {
2075                 .ip4.as_u32 = addr->as_u32,
2076                 },
2077   };
2078   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2079
2080   if (is_add)
2081     {
2082       fib_table_entry_update_one_path (fib_index, &prefix, nm->fib_src_low,
2083                                        (FIB_ENTRY_FLAG_CONNECTED |
2084                                         FIB_ENTRY_FLAG_LOCAL |
2085                                         FIB_ENTRY_FLAG_EXCLUSIVE),
2086                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
2087                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
2088     }
2089   else
2090     {
2091       fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
2092     }
2093 }
2094
2095 int
2096 nat44_ei_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2097 {
2098   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2099   nat44_ei_main_t *nm = &nat44_ei_main;
2100   nat44_ei_address_t *a = 0;
2101   int i;
2102
2103   for (i = 0; i < vec_len (nm->addresses); i++)
2104     {
2105       a = nm->addresses + i;
2106
2107       if (a->addr.as_u32 != addr.as_u32)
2108         continue;
2109
2110       switch (proto)
2111         {
2112 #define _(N, j, n, s)                                                         \
2113   case NAT_PROTOCOL_##N:                                                      \
2114     if (a->busy_##n##_port_refcounts[port])                                   \
2115       goto done;                                                              \
2116     ++a->busy_##n##_port_refcounts[port];                                     \
2117     if (port > 1024)                                                          \
2118       {                                                                       \
2119         a->busy_##n##_ports++;                                                \
2120         a->busy_##n##_ports_per_thread[ti]++;                                 \
2121       }                                                                       \
2122     break;
2123           foreach_nat_protocol
2124 #undef _
2125             default : nat_elog_info (nm, "unknown protocol");
2126           goto done;
2127         }
2128
2129       return 0;
2130     }
2131
2132 done:
2133   return 1;
2134 }
2135
2136 int
2137 nat44_ei_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2138 {
2139   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2140   nat44_ei_main_t *nm = &nat44_ei_main;
2141   nat44_ei_address_t *a = 0;
2142   int i;
2143
2144   for (i = 0; i < vec_len (nm->addresses); i++)
2145     {
2146       a = nm->addresses + i;
2147
2148       if (a->addr.as_u32 != addr.as_u32)
2149         continue;
2150
2151       switch (proto)
2152         {
2153 #define _(N, j, n, s)                                                         \
2154   case NAT_PROTOCOL_##N:                                                      \
2155     --a->busy_##n##_port_refcounts[port];                                     \
2156     if (port > 1024)                                                          \
2157       {                                                                       \
2158         a->busy_##n##_ports--;                                                \
2159         a->busy_##n##_ports_per_thread[ti]--;                                 \
2160       }                                                                       \
2161     break;
2162           foreach_nat_protocol
2163 #undef _
2164             default : nat_elog_info (nm, "unknown protocol");
2165           goto done;
2166         }
2167
2168       return 0;
2169     }
2170
2171 done:
2172   return 1;
2173 }
2174
2175 void
2176 nat44_ei_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2177                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2178                              u32 flags, ip4_address_t pool_addr, u8 *tag)
2179 {
2180   nat44_ei_static_map_resolve_t *rp;
2181   nat44_ei_main_t *nm = &nat44_ei_main;
2182
2183   vec_add2 (nm->to_resolve, rp, 1);
2184   rp->l_addr.as_u32 = l_addr.as_u32;
2185   rp->l_port = l_port;
2186   rp->e_port = e_port;
2187   rp->sw_if_index = sw_if_index;
2188   rp->vrf_id = vrf_id;
2189   rp->proto = proto;
2190   rp->flags = flags;
2191   rp->pool_addr = pool_addr;
2192   rp->tag = vec_dup (tag);
2193 }
2194
2195 int
2196 nat44_ei_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2197                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2198                              u32 flags, int *out)
2199 {
2200   nat44_ei_static_map_resolve_t *rp;
2201   nat44_ei_main_t *nm = &nat44_ei_main;
2202   int i;
2203
2204   for (i = 0; i < vec_len (nm->to_resolve); i++)
2205     {
2206       rp = nm->to_resolve + i;
2207
2208       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
2209         {
2210           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
2211             {
2212               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2213                 {
2214                   if (rp->e_port != e_port || rp->proto != proto)
2215                     {
2216                       continue;
2217                     }
2218                 }
2219             }
2220           else if (rp->l_addr.as_u32 == l_addr.as_u32)
2221             {
2222               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2223                 {
2224                   if (rp->l_port != l_port || rp->e_port != e_port ||
2225                       rp->proto != proto)
2226                     {
2227                       continue;
2228                     }
2229                 }
2230             }
2231           else
2232             {
2233               continue;
2234             }
2235           if (out)
2236             {
2237               *out = i;
2238             }
2239           return 0;
2240         }
2241     }
2242   return 1;
2243 }
2244
2245 int
2246 nat44_ei_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2247                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2248                              u32 flags)
2249 {
2250   nat44_ei_main_t *nm = &nat44_ei_main;
2251   int i;
2252   if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2253                                     sw_if_index, flags, &i))
2254     {
2255       vec_del1 (nm->to_resolve, i);
2256       return 0;
2257     }
2258   return 1;
2259 }
2260
2261 void
2262 delete_matching_dynamic_sessions (const nat44_ei_static_mapping_t *m,
2263                                   u32 worker_index)
2264 {
2265   nat44_ei_main_t *nm = &nat44_ei_main;
2266   clib_bihash_kv_8_8_t kv, value;
2267   nat44_ei_session_t *s;
2268   nat44_ei_user_key_t u_key;
2269   nat44_ei_user_t *u;
2270   nat44_ei_main_per_thread_data_t *tnm;
2271   dlist_elt_t *head, *elt;
2272   u32 elt_index, head_index;
2273   u32 ses_index;
2274   u64 user_index;
2275
2276   if (nm->static_mapping_only)
2277     return;
2278
2279   tnm = vec_elt_at_index (nm->per_thread_data, worker_index);
2280
2281   u_key.addr = m->local_addr;
2282   u_key.fib_index = m->fib_index;
2283   kv.key = u_key.as_u64;
2284   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2285     {
2286       user_index = value.value;
2287       u = pool_elt_at_index (tnm->users, user_index);
2288       if (u->nsessions)
2289         {
2290           head_index = u->sessions_per_user_list_head_index;
2291           head = pool_elt_at_index (tnm->list_pool, head_index);
2292           elt_index = head->next;
2293           elt = pool_elt_at_index (tnm->list_pool, elt_index);
2294           ses_index = elt->value;
2295           while (ses_index != ~0)
2296             {
2297               s = pool_elt_at_index (tnm->sessions, ses_index);
2298               elt = pool_elt_at_index (tnm->list_pool, elt->next);
2299               ses_index = elt->value;
2300
2301               if (nat44_ei_is_session_static (s))
2302                 continue;
2303
2304               if (!is_sm_addr_only (m->flags) &&
2305                   s->in2out.port != m->local_port)
2306                 continue;
2307
2308               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
2309                                              0);
2310               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2311
2312               if (!is_sm_addr_only (m->flags))
2313                 break;
2314             }
2315         }
2316     }
2317 }
2318
2319 int
2320 nat44_ei_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2321                              u16 l_port, u16 e_port, nat_protocol_t proto,
2322                              u32 vrf_id, u32 sw_if_index, u32 flags,
2323                              ip4_address_t pool_addr, u8 *tag)
2324
2325 {
2326   nat44_ei_main_t *nm = &nat44_ei_main;
2327
2328   if (is_sm_switch_address (flags))
2329     {
2330       if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2331                                         sw_if_index, flags, 0))
2332         {
2333           return VNET_API_ERROR_VALUE_EXIST;
2334         }
2335
2336       nat44_ei_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2337                                    sw_if_index, flags, pool_addr, tag);
2338
2339       ip4_address_t *first_int_addr =
2340         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2341       if (!first_int_addr)
2342         {
2343           // dhcp resolution required
2344           return 0;
2345         }
2346
2347       e_addr.as_u32 = first_int_addr->as_u32;
2348     }
2349
2350   return nat44_ei_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
2351                                                proto, vrf_id, sw_if_index,
2352                                                flags, pool_addr, tag);
2353 }
2354
2355 int
2356 nat44_ei_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2357                              u16 l_port, u16 e_port, nat_protocol_t proto,
2358                              u32 vrf_id, u32 sw_if_index, u32 flags)
2359 {
2360   nat44_ei_main_t *nm = &nat44_ei_main;
2361
2362   if (is_sm_switch_address (flags))
2363     {
2364
2365       if (nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2366                                        sw_if_index, flags))
2367         {
2368           return VNET_API_ERROR_NO_SUCH_ENTRY;
2369         }
2370
2371       ip4_address_t *first_int_addr =
2372         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2373       if (!first_int_addr)
2374         {
2375           // dhcp resolution required
2376           return 0;
2377         }
2378
2379       e_addr.as_u32 = first_int_addr->as_u32;
2380     }
2381
2382   return nat44_ei_del_static_mapping_internal (
2383     l_addr, e_addr, l_port, e_port, proto, vrf_id, sw_if_index, flags);
2384 }
2385
2386 static int
2387 nat44_ei_add_static_mapping_internal (ip4_address_t l_addr,
2388                                       ip4_address_t e_addr, u16 l_port,
2389                                       u16 e_port, nat_protocol_t proto,
2390                                       u32 vrf_id, u32 sw_if_index, u32 flags,
2391                                       ip4_address_t pool_addr, u8 *tag)
2392 {
2393   nat44_ei_main_t *nm = &nat44_ei_main;
2394   clib_bihash_kv_8_8_t kv, value;
2395   nat44_ei_lb_addr_port_t *local;
2396   nat44_ei_static_mapping_t *m;
2397   u32 fib_index = ~0;
2398   u32 worker_index;
2399
2400   fail_if_disabled ();
2401
2402   if (is_sm_addr_only (flags))
2403     {
2404       e_port = l_port = proto = 0;
2405     }
2406
2407   if (is_sm_identity_nat (flags))
2408     {
2409       l_port = e_port;
2410       l_addr.as_u32 = e_addr.as_u32;
2411     }
2412
2413   // fib index 0
2414   init_nat_k (&kv, e_addr, e_port, 0, proto);
2415
2416   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2417     {
2418       m = pool_elt_at_index (nm->static_mappings, value.value);
2419       if (!is_sm_identity_nat (m->flags))
2420         {
2421           return VNET_API_ERROR_VALUE_EXIST;
2422         }
2423
2424       // case:
2425       // adding local identity nat record for different vrf table
2426       pool_foreach (local, m->locals)
2427         {
2428           if (local->vrf_id == vrf_id)
2429             {
2430               return VNET_API_ERROR_VALUE_EXIST;
2431             }
2432         }
2433
2434       pool_get (m->locals, local);
2435
2436       local->vrf_id = vrf_id;
2437       local->fib_index = fib_table_find_or_create_and_lock (
2438         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2439
2440       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
2441                    m->proto, 0, m - nm->static_mappings);
2442       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2443
2444       return 0;
2445     }
2446
2447   if (vrf_id != ~0)
2448     {
2449       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
2450                                                      nm->fib_src_low);
2451     }
2452   else
2453     {
2454       // fallback to default vrf
2455       vrf_id = nm->inside_vrf_id;
2456       fib_index = nm->inside_fib_index;
2457       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2458     }
2459
2460   if (!is_sm_identity_nat (flags))
2461     {
2462       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2463       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2464         {
2465           return VNET_API_ERROR_VALUE_EXIST;
2466         }
2467     }
2468
2469   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2470     {
2471       if (nat44_ei_reserve_port (e_addr, e_port, proto))
2472         {
2473           // remove resolve record
2474           if ((is_sm_switch_address (flags)) && !is_sm_identity_nat (flags))
2475             {
2476               nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto,
2477                                            vrf_id, sw_if_index, flags);
2478             }
2479           return VNET_API_ERROR_NO_SUCH_ENTRY;
2480         }
2481     }
2482
2483   pool_get (nm->static_mappings, m);
2484   clib_memset (m, 0, sizeof (*m));
2485
2486   m->flags = flags;
2487   m->local_addr = l_addr;
2488   m->external_addr = e_addr;
2489
2490   m->tag = vec_dup (tag);
2491
2492   if (!is_sm_addr_only (flags))
2493     {
2494       m->local_port = l_port;
2495       m->external_port = e_port;
2496       m->proto = proto;
2497     }
2498
2499   if (is_sm_identity_nat (flags))
2500     {
2501       pool_get (m->locals, local);
2502
2503       local->vrf_id = vrf_id;
2504       local->fib_index = fib_index;
2505     }
2506   else
2507     {
2508       m->vrf_id = vrf_id;
2509       m->fib_index = fib_index;
2510     }
2511
2512   init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2513                m - nm->static_mappings);
2514   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2515
2516   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2517                m - nm->static_mappings);
2518   clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2519
2520   if (nm->num_workers > 1)
2521     {
2522       // store worker index for this record
2523       ip4_header_t ip = {
2524         .src_address = m->local_addr,
2525       };
2526       worker_index = nat44_ei_get_in2out_worker_index (&ip, m->fib_index, 0);
2527       vec_add1 (m->workers, worker_index);
2528     }
2529   else
2530     {
2531       worker_index = nm->num_workers;
2532     }
2533   delete_matching_dynamic_sessions (m, worker_index);
2534
2535   if (is_sm_addr_only (flags))
2536     {
2537       nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
2538     }
2539
2540   return 0;
2541 }
2542
2543 static int
2544 nat44_ei_del_static_mapping_internal (ip4_address_t l_addr,
2545                                       ip4_address_t e_addr, u16 l_port,
2546                                       u16 e_port, nat_protocol_t proto,
2547                                       u32 vrf_id, u32 sw_if_index, u32 flags)
2548 {
2549   nat44_ei_main_per_thread_data_t *tnm;
2550   nat44_ei_main_t *nm = &nat44_ei_main;
2551   clib_bihash_kv_8_8_t kv, value;
2552   nat44_ei_lb_addr_port_t *local;
2553   nat44_ei_static_mapping_t *m;
2554   u32 fib_index = ~0;
2555   nat44_ei_user_key_t u_key;
2556
2557   fail_if_disabled ();
2558
2559   if (is_sm_addr_only (flags))
2560     {
2561       e_port = l_port = proto = 0;
2562     }
2563
2564   if (is_sm_identity_nat (flags))
2565     {
2566       l_port = e_port;
2567       l_addr.as_u32 = e_addr.as_u32;
2568     }
2569
2570   // fib index 0
2571   init_nat_k (&kv, e_addr, e_port, 0, proto);
2572
2573   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2574     {
2575       if (is_sm_switch_address (flags))
2576         {
2577           return 0;
2578         }
2579       return VNET_API_ERROR_NO_SUCH_ENTRY;
2580     }
2581
2582   m = pool_elt_at_index (nm->static_mappings, value.value);
2583
2584   if (is_sm_identity_nat (flags))
2585     {
2586       u8 found = 0;
2587
2588       if (vrf_id == ~0)
2589         {
2590           vrf_id = nm->inside_vrf_id;
2591         }
2592
2593       pool_foreach (local, m->locals)
2594         {
2595           if (local->vrf_id == vrf_id)
2596             {
2597               local = pool_elt_at_index (m->locals, local - m->locals);
2598               fib_index = local->fib_index;
2599               pool_put (m->locals, local);
2600               found = 1;
2601             }
2602         }
2603       if (!found)
2604         {
2605           return VNET_API_ERROR_NO_SUCH_ENTRY;
2606         }
2607     }
2608   else
2609     {
2610       fib_index = m->fib_index;
2611     }
2612
2613   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2614     {
2615       if (nat44_ei_free_port (e_addr, e_port, proto))
2616         {
2617           return VNET_API_ERROR_INVALID_VALUE;
2618         }
2619     }
2620
2621   init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2622   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2623
2624   if (!nm->static_mapping_only || nm->static_mapping_connection_tracking)
2625     {
2626       // delete sessions for static mapping
2627       if (nm->num_workers > 1)
2628         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2629       else
2630         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2631
2632       u_key.addr = m->local_addr;
2633       u_key.fib_index = fib_index;
2634       kv.key = u_key.as_u64;
2635       nat44_ei_static_mapping_del_sessions (
2636         nm, tnm, u_key, is_sm_addr_only (flags), e_addr, e_port);
2637     }
2638
2639   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2640
2641   if (!pool_elts (m->locals))
2642     {
2643       // this is last record remove all required stuff
2644       // fib_index 0
2645       init_nat_k (&kv, e_addr, e_port, 0, proto);
2646       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2647
2648       vec_free (m->tag);
2649       vec_free (m->workers);
2650       pool_put (nm->static_mappings, m);
2651
2652       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
2653         {
2654           nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
2655         }
2656     }
2657
2658   return 0;
2659 }
2660
2661 int
2662 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2663                                u32 match_fib_index,
2664                                nat_protocol_t match_protocol,
2665                                ip4_address_t *mapping_addr, u16 *mapping_port,
2666                                u32 *mapping_fib_index, u8 by_external,
2667                                u8 *is_addr_only, u8 *is_identity_nat)
2668 {
2669   nat44_ei_main_t *nm = &nat44_ei_main;
2670   clib_bihash_kv_8_8_t kv, value;
2671   nat44_ei_static_mapping_t *m;
2672   u16 port;
2673
2674   if (by_external)
2675     {
2676       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2677       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2678                                   &value))
2679         {
2680           /* Try address only mapping */
2681           init_nat_k (&kv, match_addr, 0, 0, 0);
2682           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2683                                       &value))
2684             return 1;
2685         }
2686       m = pool_elt_at_index (nm->static_mappings, value.value);
2687
2688       *mapping_fib_index = m->fib_index;
2689       *mapping_addr = m->local_addr;
2690       port = m->local_port;
2691     }
2692   else
2693     {
2694       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2695                   match_protocol);
2696       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2697         {
2698           /* Try address only mapping */
2699           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2700           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2701                                       &value))
2702             return 1;
2703         }
2704       m = pool_elt_at_index (nm->static_mappings, value.value);
2705
2706       *mapping_fib_index = nm->outside_fib_index;
2707       *mapping_addr = m->external_addr;
2708       port = m->external_port;
2709     }
2710
2711   /* Address only mapping doesn't change port */
2712   if (is_sm_addr_only (m->flags))
2713     *mapping_port = match_port;
2714   else
2715     *mapping_port = port;
2716
2717   if (PREDICT_FALSE (is_addr_only != 0))
2718     *is_addr_only = is_sm_addr_only (m->flags);
2719
2720   if (PREDICT_FALSE (is_identity_nat != 0))
2721     *is_identity_nat = is_sm_identity_nat (m->flags);
2722
2723   return 0;
2724 }
2725
2726 static void
2727 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2728 {
2729   pool_free (tnm->list_pool);
2730   pool_free (tnm->lru_pool);
2731   pool_free (tnm->sessions);
2732   pool_free (tnm->users);
2733
2734   clib_bihash_free_8_8 (&tnm->user_hash);
2735 }
2736
2737 u8 *
2738 format_nat44_ei_key (u8 *s, va_list *args)
2739 {
2740   u64 key = va_arg (*args, u64);
2741
2742   ip4_address_t addr;
2743   u16 port;
2744   nat_protocol_t protocol;
2745   u32 fib_index;
2746
2747   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2748
2749   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2750               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2751               fib_index);
2752   return s;
2753 }
2754
2755 u8 *
2756 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2757 {
2758   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2759   nat44_ei_user_key_t k;
2760
2761   k.as_u64 = v->key;
2762
2763   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2764               k.fib_index, v->value);
2765
2766   return s;
2767 }
2768
2769 u8 *
2770 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2771 {
2772   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2773
2774   s = format (s, "%U thread-index %llu session-index %llu",
2775               format_nat44_ei_key, v->key, nat_value_get_thread_index (v),
2776               nat_value_get_session_index (v));
2777
2778   return s;
2779 }
2780
2781 u8 *
2782 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2783 {
2784   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2785
2786   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2787               v->value);
2788
2789   return s;
2790 }
2791
2792 static void
2793 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2794                          u32 translations, u32 translation_buckets,
2795                          u32 user_buckets)
2796 {
2797   dlist_elt_t *head;
2798
2799   pool_alloc (tnm->list_pool, translations);
2800   pool_alloc (tnm->lru_pool, translations);
2801   pool_alloc (tnm->sessions, translations);
2802
2803   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2804
2805   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2806                                      format_nat44_ei_user_kvp);
2807
2808   pool_get (tnm->lru_pool, head);
2809   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2810   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2811
2812   pool_get (tnm->lru_pool, head);
2813   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2814   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2815
2816   pool_get (tnm->lru_pool, head);
2817   tnm->udp_lru_head_index = head - tnm->lru_pool;
2818   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2819
2820   pool_get (tnm->lru_pool, head);
2821   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2822   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2823
2824   pool_get (tnm->lru_pool, head);
2825   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2826   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2827 }
2828
2829 static void
2830 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2831 {
2832   nat44_ei_main_t *nm = &nat44_ei_main;
2833   nat44_ei_main_per_thread_data_t *tnm;
2834
2835   u32 static_mapping_buckets = 1024;
2836   u32 static_mapping_memory_size = 64 << 20;
2837
2838   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2839                         "static_mapping_by_local", static_mapping_buckets,
2840                         static_mapping_memory_size);
2841   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2842                         "static_mapping_by_external", static_mapping_buckets,
2843                         static_mapping_memory_size);
2844   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2845                                      format_nat44_ei_static_mapping_kvp);
2846   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2847                                      format_nat44_ei_static_mapping_kvp);
2848
2849   if (nm->pat)
2850     {
2851       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2852       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2853       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2854                                          format_nat44_ei_session_kvp);
2855       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2856                                          format_nat44_ei_session_kvp);
2857       vec_foreach (tnm, nm->per_thread_data)
2858         {
2859           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2860                                    user_buckets);
2861         }
2862     }
2863 }
2864
2865 void
2866 nat44_ei_sessions_clear ()
2867 {
2868   nat44_ei_main_t *nm = &nat44_ei_main;
2869   nat44_ei_main_per_thread_data_t *tnm;
2870
2871   if (nm->pat)
2872     {
2873       clib_bihash_free_8_8 (&nm->in2out);
2874       clib_bihash_free_8_8 (&nm->out2in);
2875       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2876       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2877       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2878                                          format_nat44_ei_session_kvp);
2879       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2880                                          format_nat44_ei_session_kvp);
2881       vec_foreach (tnm, nm->per_thread_data)
2882         {
2883           nat44_ei_worker_db_free (tnm);
2884           nat44_ei_worker_db_init (tnm, nm->translations,
2885                                    nm->translation_buckets, nm->user_buckets);
2886         }
2887     }
2888
2889   vlib_zero_simple_counter (&nm->total_users, 0);
2890   vlib_zero_simple_counter (&nm->total_sessions, 0);
2891   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2892 }
2893
2894 static void
2895 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2896                              u32 new_fib_index, u32 old_fib_index)
2897 {
2898   nat44_ei_main_t *nm = &nat44_ei_main;
2899   nat44_ei_outside_fib_t *outside_fib;
2900   nat44_ei_interface_t *i;
2901   u8 is_add = 1;
2902   u8 match = 0;
2903
2904   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2905       (!vec_len (nm->outside_fibs)))
2906     {
2907       return;
2908     }
2909
2910   pool_foreach (i, nm->interfaces)
2911     {
2912       if (i->sw_if_index == sw_if_index)
2913         {
2914           if (!(nat44_ei_interface_is_outside (i)))
2915             return;
2916           match = 1;
2917         }
2918     }
2919
2920   pool_foreach (i, nm->output_feature_interfaces)
2921     {
2922       if (i->sw_if_index == sw_if_index)
2923         {
2924           if (!(nat44_ei_interface_is_outside (i)))
2925             return;
2926           match = 1;
2927         }
2928     }
2929
2930   if (!match)
2931     return;
2932
2933   vec_foreach (outside_fib, nm->outside_fibs)
2934     {
2935       if (outside_fib->fib_index == old_fib_index)
2936         {
2937           outside_fib->refcount--;
2938           if (!outside_fib->refcount)
2939             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2940           break;
2941         }
2942     }
2943
2944   vec_foreach (outside_fib, nm->outside_fibs)
2945     {
2946       if (outside_fib->fib_index == new_fib_index)
2947         {
2948           outside_fib->refcount++;
2949           is_add = 0;
2950           break;
2951         }
2952     }
2953
2954   if (is_add)
2955     {
2956       vec_add2 (nm->outside_fibs, outside_fib, 1);
2957       outside_fib->refcount = 1;
2958       outside_fib->fib_index = new_fib_index;
2959     }
2960 }
2961
2962 int
2963 nat44_ei_add_address (ip4_address_t *addr, u32 vrf_id)
2964 {
2965   nat44_ei_main_t *nm = &nat44_ei_main;
2966   vlib_thread_main_t *tm = vlib_get_thread_main ();
2967   nat44_ei_address_t *ap;
2968
2969   fail_if_disabled ();
2970
2971   /* Check if address already exists */
2972   vec_foreach (ap, nm->addresses)
2973     {
2974       if (ap->addr.as_u32 == addr->as_u32)
2975         {
2976           nat44_ei_log_err ("address exist");
2977           return VNET_API_ERROR_VALUE_EXIST;
2978         }
2979     }
2980
2981   vec_add2 (nm->addresses, ap, 1);
2982
2983   ap->fib_index = ~0;
2984   ap->addr = *addr;
2985
2986   if (vrf_id != ~0)
2987     {
2988       ap->fib_index = fib_table_find_or_create_and_lock (
2989         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2990     }
2991
2992 #define _(N, i, n, s)                                                         \
2993   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
2994                sizeof (ap->busy_##n##_port_refcounts));                       \
2995   ap->busy_##n##_ports = 0;                                                   \
2996   ap->busy_##n##_ports_per_thread = 0;                                        \
2997   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
2998                            tm->n_vlib_mains - 1, 0);
2999   foreach_nat_protocol
3000 #undef _
3001
3002     nat44_ei_add_del_addr_to_fib_foreach_out_if (addr, 1);
3003
3004   return 0;
3005 }
3006
3007 int
3008 nat44_ei_del_address (ip4_address_t addr, u8 delete_sm)
3009 {
3010   nat44_ei_main_t *nm = &nat44_ei_main;
3011   nat44_ei_address_t *a = 0;
3012   nat44_ei_session_t *ses;
3013   u32 *ses_to_be_removed = 0, *ses_index;
3014   nat44_ei_main_per_thread_data_t *tnm;
3015   nat44_ei_static_mapping_t *m;
3016   int j;
3017
3018   fail_if_disabled ();
3019
3020   /* Find SNAT address */
3021   for (j = 0; j < vec_len (nm->addresses); j++)
3022     {
3023       if (nm->addresses[j].addr.as_u32 == addr.as_u32)
3024         {
3025           a = nm->addresses + j;
3026           break;
3027         }
3028     }
3029   if (!a)
3030     {
3031       nat44_ei_log_err ("no such address");
3032       return VNET_API_ERROR_NO_SUCH_ENTRY;
3033     }
3034
3035   if (delete_sm)
3036     {
3037       pool_foreach (m, nm->static_mappings)
3038         {
3039           if (m->external_addr.as_u32 == addr.as_u32)
3040             nat44_ei_del_static_mapping_internal (
3041               m->local_addr, m->external_addr, m->local_port, m->external_port,
3042               m->proto, m->vrf_id, ~0, m->flags);
3043         }
3044     }
3045   else
3046     {
3047       /* Check if address is used in some static mapping */
3048       if (nat44_ei_is_address_used_in_static_mapping (addr))
3049         {
3050           nat44_ei_log_err ("address used in static mapping");
3051           return VNET_API_ERROR_UNSPECIFIED;
3052         }
3053     }
3054
3055   /* Delete sessions using address */
3056   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
3057     {
3058       vec_foreach (tnm, nm->per_thread_data)
3059         {
3060           pool_foreach (ses, tnm->sessions)
3061             {
3062               if (ses->out2in.addr.as_u32 == addr.as_u32)
3063                 {
3064                   nat44_ei_free_session_data (nm, ses,
3065                                               tnm - nm->per_thread_data, 0);
3066                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
3067                 }
3068             }
3069           vec_foreach (ses_index, ses_to_be_removed)
3070             {
3071               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
3072               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
3073             }
3074           vec_free (ses_to_be_removed);
3075         }
3076     }
3077
3078   nat44_ei_add_del_addr_to_fib_foreach_out_if (&addr, 0);
3079
3080   if (a->fib_index != ~0)
3081     {
3082       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
3083     }
3084
3085 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
3086   foreach_nat_protocol
3087 #undef _
3088
3089     vec_del1 (nm->addresses, j);
3090   return 0;
3091 }
3092
3093 int
3094 nat44_ei_add_interface_address (u32 sw_if_index)
3095 {
3096   nat44_ei_main_t *nm = &nat44_ei_main;
3097   ip4_main_t *ip4_main = nm->ip4_main;
3098   ip4_address_t *first_int_addr;
3099   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3100   int i;
3101
3102   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3103     {
3104       if (auto_add_sw_if_indices[i] == sw_if_index)
3105         {
3106           return VNET_API_ERROR_VALUE_EXIST;
3107         }
3108     }
3109
3110   /* add to the auto-address list */
3111   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
3112
3113   // if the address is already bound - or static - add it now
3114   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3115   if (first_int_addr)
3116     {
3117       (void) nat44_ei_add_address (first_int_addr, ~0);
3118     }
3119
3120   return 0;
3121 }
3122
3123 int
3124 nat44_ei_del_interface_address (u32 sw_if_index)
3125 {
3126   nat44_ei_main_t *nm = &nat44_ei_main;
3127   ip4_main_t *ip4_main = nm->ip4_main;
3128   ip4_address_t *first_int_addr;
3129   nat44_ei_static_map_resolve_t *rp;
3130   u32 *indices_to_delete = 0;
3131   int i, j;
3132   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3133
3134   fail_if_disabled ();
3135
3136   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3137
3138   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3139     {
3140       if (auto_add_sw_if_indices[i] == sw_if_index)
3141         {
3142           first_int_addr =
3143             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3144           if (first_int_addr)
3145             {
3146               (void) nat44_ei_del_address (first_int_addr[0], 1);
3147             }
3148           else
3149             {
3150               for (j = 0; j < vec_len (nm->to_resolve); j++)
3151                 {
3152                   rp = nm->to_resolve + j;
3153                   if (rp->sw_if_index == sw_if_index)
3154                     {
3155                       vec_add1 (indices_to_delete, j);
3156                     }
3157                 }
3158               if (vec_len (indices_to_delete))
3159                 {
3160                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3161                     {
3162                       vec_del1 (nm->to_resolve, j);
3163                     }
3164                   vec_free (indices_to_delete);
3165                 }
3166             }
3167
3168           vec_del1 (nm->auto_add_sw_if_indices, i);
3169           return 0;
3170         }
3171     }
3172   return VNET_API_ERROR_NO_SUCH_ENTRY;
3173 }
3174
3175 static_always_inline int
3176 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3177 {
3178   u32 *i;
3179   vec_foreach (i, sw_if_indices)
3180     {
3181       if (*i == sw_if_index)
3182         {
3183           return 1;
3184         }
3185     }
3186   return 0;
3187 }
3188
3189 static void
3190 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3191                                            u32 sw_if_index,
3192                                            ip4_address_t *address,
3193                                            u32 address_length,
3194                                            u32 if_address_index, u32 is_delete)
3195 {
3196   nat44_ei_main_t *nm = &nat44_ei_main;
3197   nat44_ei_static_map_resolve_t *rp;
3198   nat44_ei_address_t *addresses = nm->addresses;
3199   int rv, i;
3200
3201   if (!nm->enabled)
3202     {
3203       return;
3204     }
3205
3206   if (!is_sw_if_index_reg_for_auto_resolve (nm->auto_add_sw_if_indices,
3207                                             sw_if_index))
3208     {
3209       return;
3210     }
3211
3212   if (!is_delete)
3213     {
3214       /* Don't trip over lease renewal, static config */
3215       for (i = 0; i < vec_len (addresses); i++)
3216         {
3217           if (addresses[i].addr.as_u32 == address->as_u32)
3218             {
3219               return;
3220             }
3221         }
3222
3223       (void) nat44_ei_add_address (address, ~0);
3224
3225       /* Scan static map resolution vector */
3226       for (i = 0; i < vec_len (nm->to_resolve); i++)
3227         {
3228           rp = nm->to_resolve + i;
3229           if (is_sm_addr_only (rp->flags))
3230             {
3231               continue;
3232             }
3233           /* On this interface? */
3234           if (rp->sw_if_index == sw_if_index)
3235             {
3236               rv = nat44_ei_add_static_mapping_internal (
3237                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3238                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3239               if (rv)
3240                 {
3241                   nat_elog_notice_X1 (
3242                     nm, "add_static_mapping_internal returned %d", "i4", rv);
3243                 }
3244             }
3245         }
3246     }
3247   else
3248     {
3249       // remove all static mapping records
3250       (void) nat44_ei_del_address (address[0], 1);
3251     }
3252 }
3253
3254 int
3255 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
3256 {
3257   fail_if_enabled ();
3258   nat44_ei_main_t *nm = &nat44_ei_main;
3259   nm->frame_queue_nelts = frame_queue_nelts;
3260   return 0;
3261 }
3262
3263 static void
3264 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
3265                                       u32 sw_if_index, ip4_address_t *address,
3266                                       u32 address_length, u32 if_address_index,
3267                                       u32 is_delete)
3268 {
3269   nat44_ei_main_t *nm = &nat44_ei_main;
3270   nat44_ei_static_map_resolve_t *rp;
3271   nat44_ei_static_mapping_t *m;
3272   clib_bihash_kv_8_8_t kv, value;
3273   int i, rv = 0, match = 0;
3274
3275   if (!nm->enabled)
3276     {
3277       return;
3278     }
3279
3280   for (i = 0; i < vec_len (nm->to_resolve); i++)
3281     {
3282       rp = nm->to_resolve + i;
3283
3284       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3285         {
3286           match = 1;
3287           break;
3288         }
3289     }
3290
3291   if (!match)
3292     {
3293       return;
3294     }
3295
3296   init_nat_k (&kv, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port,
3297               nm->outside_fib_index,
3298               is_sm_addr_only (rp->flags) ? 0 : rp->proto);
3299   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
3300     m = 0;
3301   else
3302     m = pool_elt_at_index (nm->static_mappings, value.value);
3303
3304   if (is_delete)
3305     {
3306       if (!m)
3307         return;
3308       rv = nat44_ei_del_static_mapping_internal (
3309         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3310         ~0, rp->flags);
3311       if (rv)
3312         {
3313           nat_elog_notice_X1 (nm, "nat44_ei_del_static_mapping returned %d",
3314                               "i4", rv);
3315         }
3316     }
3317   else
3318     {
3319       if (m)
3320         return;
3321       rv = nat44_ei_add_static_mapping_internal (
3322         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3323         ~0, rp->flags, rp->pool_addr, rp->tag);
3324
3325       if (rv)
3326         {
3327           nat_elog_notice_X1 (nm, "nat44_ei_add_static_mapping returned %d",
3328                               "i4", rv);
3329         }
3330     }
3331 }
3332
3333 static_always_inline uword
3334 nat44_ei_classify_inline_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
3335                              vlib_frame_t *frame)
3336 {
3337   u32 n_left_from, *from, *to_next;
3338   nat44_ei_classify_next_t next_index;
3339   nat44_ei_main_t *nm = &nat44_ei_main;
3340   nat44_ei_static_mapping_t *m;
3341   u32 next_in2out = 0, next_out2in = 0;
3342
3343   from = vlib_frame_vector_args (frame);
3344   n_left_from = frame->n_vectors;
3345   next_index = node->cached_next_index;
3346
3347   while (n_left_from > 0)
3348     {
3349       u32 n_left_to_next;
3350
3351       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3352
3353       while (n_left_from > 0 && n_left_to_next > 0)
3354         {
3355           u32 bi0;
3356           vlib_buffer_t *b0;
3357           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3358           ip4_header_t *ip0;
3359           nat44_ei_address_t *ap;
3360           clib_bihash_kv_8_8_t kv0, value0;
3361
3362           /* speculatively enqueue b0 to the current next frame */
3363           bi0 = from[0];
3364           to_next[0] = bi0;
3365           from += 1;
3366           to_next += 1;
3367           n_left_from -= 1;
3368           n_left_to_next -= 1;
3369
3370           b0 = vlib_get_buffer (vm, bi0);
3371           ip0 = vlib_buffer_get_current (b0);
3372
3373           vec_foreach (ap, nm->addresses)
3374             {
3375               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
3376                 {
3377                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3378                   goto enqueue0;
3379                 }
3380             }
3381
3382           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
3383             {
3384               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
3385               /* try to classify the fragment based on IP header alone */
3386               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3387                                            &kv0, &value0))
3388                 {
3389                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3390                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3391                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3392                   goto enqueue0;
3393                 }
3394               init_nat_k (&kv0, ip0->dst_address,
3395                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
3396                           ip_proto_to_nat_proto (ip0->protocol));
3397               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3398                                            &kv0, &value0))
3399                 {
3400                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3401                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3402                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3403                 }
3404             }
3405
3406         enqueue0:
3407           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
3408                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
3409             {
3410               nat44_ei_classify_trace_t *t =
3411                 vlib_add_trace (vm, node, b0, sizeof (*t));
3412               t->cached = 0;
3413               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
3414             }
3415
3416           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3417           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3418
3419           /* verify speculative enqueue, maybe switch current next frame */
3420           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3421                                            n_left_to_next, bi0, next0);
3422         }
3423
3424       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3425     }
3426
3427   vlib_node_increment_counter (
3428     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
3429   vlib_node_increment_counter (
3430     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3431   return frame->n_vectors;
3432 }
3433
3434 VLIB_NODE_FN (nat44_ei_classify_node)
3435 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3436 {
3437   return nat44_ei_classify_inline_fn (vm, node, frame);
3438 }
3439
3440 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3441   .name = "nat44-ei-classify",
3442   .vector_size = sizeof (u32),
3443   .format_trace = format_nat44_ei_classify_trace,
3444   .type = VLIB_NODE_TYPE_INTERNAL,
3445   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3446   .error_strings = nat44_ei_classify_error_strings,
3447   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3448   .next_nodes = {
3449     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3450     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3451     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3452   },
3453 };
3454
3455 VLIB_NODE_FN (nat44_ei_handoff_classify_node)
3456 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3457 {
3458   return nat44_ei_classify_inline_fn (vm, node, frame);
3459 }
3460
3461 VLIB_REGISTER_NODE (nat44_ei_handoff_classify_node) = {
3462   .name = "nat44-ei-handoff-classify",
3463   .vector_size = sizeof (u32),
3464   .format_trace = format_nat44_ei_classify_trace,
3465   .type = VLIB_NODE_TYPE_INTERNAL,
3466   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3467   .error_strings = nat44_ei_classify_error_strings,
3468   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3469   .next_nodes = {
3470     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out-worker-handoff",
3471     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in-worker-handoff",
3472     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3473   },
3474 };
3475
3476 /*
3477  * fd.io coding-style-patch-verification: ON
3478  *
3479  * Local Variables:
3480  * eval: (c-set-style "gnu")
3481  * End:
3482  */