src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t
  44   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  45 extern vlib_node_registration_t
  46   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  47
  48 #define skip_if_disabled()                                                    \
  49   do                                                                          \
  50     {                                                                         \
  51       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  52       if (PREDICT_FALSE (!nm->enabled))                                       \
  53         return;                                                               \
  54     }                                                                         \
  55   while (0)
  56
  57 #define fail_if_enabled()                                                     \
  58   do                                                                          \
  59     {                                                                         \
  60       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  61       if (PREDICT_FALSE (nm->enabled))                                        \
  62         {                                                                     \
  63           nat44_ei_log_err ("plugin enabled");                                \
  64           return 1;                                                           \
  65         }                                                                     \
  66     }                                                                         \
  67   while (0)
  68
  69 #define fail_if_disabled()                                                    \
  70   do                                                                          \
  71     {                                                                         \
  72       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  73       if (PREDICT_FALSE (!nm->enabled))                                       \
  74         {                                                                     \
  75           nat44_ei_log_err ("plugin disabled");                               \
  76           return 1;                                                           \
  77         }                                                                     \
  78     }                                                                         \
  79   while (0)
  80
  81 /* Hook up input features */
  82 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat44-ei-classify",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  86                                "ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-ei-handoff-classify",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  92                                "ip4-sv-reassembly-feature"),
  93 };
  94 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-ei-in2out",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-sv-reassembly-feature"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-ei-out2in",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 104                                "ip4-sv-reassembly-feature",
 105                                "ip4-dhcp-client-detect"),
 106 };
 107 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 108   .arc_name = "ip4-output",
 109   .node_name = "nat44-ei-in2out-output",
 110   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 111                                "ip4-sv-reassembly-output-feature"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 114   .arc_name = "ip4-local",
 115   .node_name = "nat44-ei-hairpinning",
 116   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-ei-in2out-worker-handoff",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 122 };
 123 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 124   .arc_name = "ip4-unicast",
 125   .node_name = "nat44-ei-out2in-worker-handoff",
 126   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 127                                "ip4-dhcp-client-detect"),
 128 };
 129 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 130   .arc_name = "ip4-output",
 131   .node_name = "nat44-ei-in2out-output-worker-handoff",
 132   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 133                                "ip4-sv-reassembly-output-feature"),
 134 };
 135
 136 VLIB_PLUGIN_REGISTER () = {
 137   .version = VPP_BUILD_VER,
 138   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 139 };
 140
 141 #define foreach_nat44_ei_classify_error                                       \
 142   _ (NEXT_IN2OUT, "next in2out")                                              \
 143   _ (NEXT_OUT2IN, "next out2in")                                              \
 144   _ (FRAG_CACHED, "fragment cached")
 145
 146 typedef enum
 147 {
 148 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 149   foreach_nat44_ei_classify_error
 150 #undef _
 151     NAT44_EI_CLASSIFY_N_ERROR,
 152 } nat44_ei_classify_error_t;
 153
 154 static char *nat44_ei_classify_error_strings[] = {
 155 #define _(sym, string) string,
 156   foreach_nat44_ei_classify_error
 157 #undef _
 158 };
 159
 160 typedef enum
 161 {
 162   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 163   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 164   NAT44_EI_CLASSIFY_NEXT_DROP,
 165   NAT44_EI_CLASSIFY_N_NEXT,
 166 } nat44_ei_classify_next_t;
 167
 168 typedef struct
 169 {
 170   u8 next_in2out;
 171   u8 cached;
 172 } nat44_ei_classify_trace_t;
 173
 174 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 175                                    u32 sw_if_index, int is_add);
 176
 177 static void nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm);
 178
 179 static int nat44_ei_add_static_mapping_internal (
 180   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 181   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 182   ip4_address_t pool_addr, u8 *tag);
 183
 184 static int nat44_ei_del_static_mapping_internal (
 185   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 186   nat_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags);
 187
 188 static u8 *
 189 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 190 {
 191   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 192   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 193   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 194   char *next;
 195
 196   if (t->cached)
 197     s = format (s, "nat44-ei-classify: fragment cached");
 198   else
 199     {
 200       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 201       s = format (s, "nat44-ei-classify: next %s", next);
 202     }
 203
 204   return s;
 205 }
 206
 207 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 208                               u32 user_buckets);
 209
 210 static void nat44_ei_ip4_add_del_interface_address_cb (
 211   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 212   u32 address_length, u32 if_address_index, u32 is_delete);
 213
 214 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 215   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 216   u32 address_length, u32 if_address_index, u32 is_delete);
 217
 218 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 219                                          u32 sw_if_index, u32 new_fib_index,
 220                                          u32 old_fib_index);
 221
 222 void
 223 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 224 {
 225   vlib_node_t *node;
 226   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 227   nm->out2in_node_index = node->index;
 228   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 229   nm->in2out_node_index = node->index;
 230   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 231   nm->in2out_output_node_index = node->index;
 232 }
 233
 234 int
 235 nat44_ei_set_workers (uword *bitmap)
 236 {
 237   nat44_ei_main_t *nm = &nat44_ei_main;
 238   int i, j = 0;
 239
 240   if (nm->num_workers < 2)
 241     return VNET_API_ERROR_FEATURE_DISABLED;
 242
 243   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 244     return VNET_API_ERROR_INVALID_WORKER;
 245
 246   vec_free (nm->workers);
 247   clib_bitmap_foreach (i, bitmap)
 248     {
 249       vec_add1 (nm->workers, i);
 250       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 251       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 252       j++;
 253     }
 254
 255   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 256
 257   return 0;
 258 }
 259
 260 #define nat_validate_simple_counter(c, i)                                     \
 261   do                                                                          \
 262     {                                                                         \
 263       vlib_validate_simple_counter (&c, i);                                   \
 264       vlib_zero_simple_counter (&c, i);                                       \
 265     }                                                                         \
 266   while (0);
 267
 268 #define nat_init_simple_counter(c, n, sn)                                     \
 269   do                                                                          \
 270     {                                                                         \
 271       c.name = n;                                                             \
 272       c.stat_segment_name = sn;                                               \
 273       nat_validate_simple_counter (c, 0);                                     \
 274     }                                                                         \
 275   while (0);
 276
 277 static_always_inline void
 278 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 279 {
 280 #define _(x)                                                                  \
 281   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 282   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 283   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 284   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 285   foreach_nat_counter;
 286 #undef _
 287   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 288 }
 289
 290 static void
 291 nat44_ei_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 292 {
 293   nat44_ei_main_t *nm = &nat44_ei_main;
 294   nat44_ei_interface_t *i;
 295
 296   pool_foreach (i, nm->interfaces)
 297     {
 298       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 299         {
 300           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 301         }
 302     }
 303   pool_foreach (i, nm->output_feature_interfaces)
 304     {
 305       if (nat44_ei_interface_is_outside (i) && !nm->out2in_dpo)
 306         {
 307           nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 308         }
 309     }
 310 }
 311
 312 static_always_inline void
 313 nat44_ei_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 314 {
 315   nat44_ei_main_t *nm = &nat44_ei_main;
 316   nat44_ei_address_t *ap;
 317
 318   vec_foreach (ap, nm->addresses)
 319     {
 320       nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 321     }
 322 }
 323
 324 static_always_inline void
 325 nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 326 {
 327   nat44_ei_main_t *nm = &nat44_ei_main;
 328   nat44_ei_static_mapping_t *m;
 329
 330   pool_foreach (m, nm->static_mappings)
 331     {
 332       if (is_sm_addr_only (m->flags) &&
 333           !(m->local_addr.as_u32 == m->external_addr.as_u32))
 334         {
 335           nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 336                                         is_add);
 337         }
 338     }
 339 }
 340
 341 static int
 342 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
 343 {
 344   nat44_ei_main_t *nm = &nat44_ei_main;
 345   nat44_ei_static_mapping_t *m;
 346   pool_foreach (m, nm->static_mappings)
 347     {
 348       if (is_sm_addr_only (m->flags) || is_sm_identity_nat (m->flags))
 349         {
 350           continue;
 351         }
 352       if (m->external_addr.as_u32 == addr.as_u32)
 353         {
 354           return 1;
 355         }
 356     }
 357   return 0;
 358 }
 359
 360 clib_error_t *
 361 nat44_ei_init (vlib_main_t *vm)
 362 {
 363   nat44_ei_main_t *nm = &nat44_ei_main;
 364   vlib_thread_main_t *tm = vlib_get_thread_main ();
 365   vlib_thread_registration_t *tr;
 366   ip4_add_del_interface_address_callback_t cbi = { 0 };
 367   ip4_table_bind_callback_t cbt = { 0 };
 368   u32 i, num_threads = 0;
 369   uword *p, *bitmap = 0;
 370
 371   clib_memset (nm, 0, sizeof (*nm));
 372
 373   // required
 374   nm->vnet_main = vnet_get_main ();
 375   // convenience
 376   nm->ip4_main = &ip4_main;
 377   nm->api_main = vlibapi_get_main ();
 378   nm->ip4_lookup_main = &ip4_main.lookup_main;
 379
 380   // handoff stuff
 381   nm->fq_out2in_index = ~0;
 382   nm->fq_in2out_index = ~0;
 383   nm->fq_in2out_output_index = ~0;
 384
 385   nm->log_level = NAT_LOG_ERROR;
 386
 387   nat44_ei_set_node_indexes (nm, vm);
 388   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 389
 390   nat_init_simple_counter (nm->total_users, "total-users",
 391                            "/nat44-ei/total-users");
 392   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 393                            "/nat44-ei/total-sessions");
 394   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 395                            "/nat44-ei/user-limit-reached");
 396
 397 #define _(x)                                                                  \
 398   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 399                            "/nat44-ei/in2out/fastpath/" #x);                  \
 400   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 401                            "/nat44-ei/out2in/fastpath/" #x);                  \
 402   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 403                            "/nat44-ei/in2out/slowpath/" #x);                  \
 404   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 405                            "/nat44-ei/out2in/slowpath/" #x);
 406   foreach_nat_counter;
 407 #undef _
 408   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 409                            "/nat44-ei/hairpinning");
 410
 411   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 412   if (p)
 413     {
 414       tr = (vlib_thread_registration_t *) p[0];
 415       if (tr)
 416         {
 417           nm->num_workers = tr->count;
 418           nm->first_worker_index = tr->first_index;
 419         }
 420     }
 421   num_threads = tm->n_vlib_mains - 1;
 422   nm->port_per_thread = 0xffff - 1024;
 423   vec_validate (nm->per_thread_data, num_threads);
 424
 425   /* Use all available workers by default */
 426   if (nm->num_workers > 1)
 427     {
 428       for (i = 0; i < nm->num_workers; i++)
 429         bitmap = clib_bitmap_set (bitmap, i, 1);
 430       nat44_ei_set_workers (bitmap);
 431       clib_bitmap_free (bitmap);
 432     }
 433   else
 434     {
 435       nm->per_thread_data[0].snat_thread_index = 0;
 436     }
 437
 438   /* callbacks to call when interface address changes. */
 439   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 440   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 441   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 442   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 443
 444   /* callbacks to call when interface to table biding changes */
 445   cbt.function = nat44_ei_update_outside_fib;
 446   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 447
 448   nm->fib_src_low = fib_source_allocate (
 449     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 450   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 451                                         FIB_SOURCE_BH_SIMPLE);
 452
 453   // used only by out2in-dpo feature
 454   nat_dpo_module_init ();
 455   nat_ha_init (vm, nm->num_workers, num_threads);
 456
 457   nm->hairpinning_fq_index =
 458     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 459   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 460     vlib_frame_queue_main_init (
 461       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 462   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 463     vlib_frame_queue_main_init (
 464       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 465   return nat44_ei_api_hookup (vm);
 466 }
 467
 468 VLIB_INIT_FUNCTION (nat44_ei_init);
 469
 470 int
 471 nat44_ei_plugin_enable (nat44_ei_config_t c)
 472 {
 473   nat44_ei_main_t *nm = &nat44_ei_main;
 474
 475   fail_if_enabled ();
 476
 477   if (!c.users)
 478     c.users = 1024;
 479
 480   if (!c.sessions)
 481     c.sessions = 10 * 1024;
 482
 483   if (!c.user_sessions)
 484     c.user_sessions = c.sessions;
 485
 486   nm->rconfig = c;
 487
 488   if (!nm->frame_queue_nelts)
 489     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 490
 491   nm->translations = c.sessions;
 492   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 493   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 494
 495   nm->pat = (!c.static_mapping_only ||
 496              (c.static_mapping_only && c.connection_tracking));
 497
 498   nm->static_mapping_only = c.static_mapping_only;
 499   nm->static_mapping_connection_tracking = c.connection_tracking;
 500   nm->out2in_dpo = c.out2in_dpo;
 501   nm->forwarding_enabled = 0;
 502   nm->mss_clamping = 0;
 503
 504   nm->max_users_per_thread = c.users;
 505   nm->max_translations_per_thread = c.sessions;
 506   nm->max_translations_per_user = c.user_sessions;
 507
 508   nm->inside_vrf_id = c.inside_vrf;
 509   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 510     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 511
 512   nm->outside_vrf_id = c.outside_vrf;
 513   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 514     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 515
 516   nat_reset_timeouts (&nm->timeouts);
 517   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 518                     nm->user_buckets);
 519   nat44_ei_set_alloc_default ();
 520
 521   vlib_zero_simple_counter (&nm->total_users, 0);
 522   vlib_zero_simple_counter (&nm->total_sessions, 0);
 523   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 524
 525   if (nm->num_workers > 1)
 526     {
 527       if (nm->fq_in2out_index == ~0)
 528         {
 529           nm->fq_in2out_index = vlib_frame_queue_main_init (
 530             nm->in2out_node_index, nm->frame_queue_nelts);
 531         }
 532       if (nm->fq_out2in_index == ~0)
 533         {
 534           nm->fq_out2in_index = vlib_frame_queue_main_init (
 535             nm->out2in_node_index, nm->frame_queue_nelts);
 536         }
 537       if (nm->fq_in2out_output_index == ~0)
 538         {
 539           nm->fq_in2out_output_index = vlib_frame_queue_main_init (
 540             nm->in2out_output_node_index, nm->frame_queue_nelts);
 541         }
 542     }
 543
 544   nat_ha_enable ();
 545   nm->enabled = 1;
 546
 547   return 0;
 548 }
 549
 550 static_always_inline nat44_ei_outside_fib_t *
 551 nat44_ei_get_outside_fib (nat44_ei_outside_fib_t *outside_fibs, u32 fib_index)
 552 {
 553   nat44_ei_outside_fib_t *f;
 554   vec_foreach (f, outside_fibs)
 555     {
 556       if (f->fib_index == fib_index)
 557         {
 558           return f;
 559         }
 560     }
 561   return 0;
 562 }
 563
 564 static_always_inline nat44_ei_interface_t *
 565 nat44_ei_get_interface (nat44_ei_interface_t *interfaces, u32 sw_if_index)
 566 {
 567   nat44_ei_interface_t *i;
 568   pool_foreach (i, interfaces)
 569     {
 570       if (i->sw_if_index == sw_if_index)
 571         {
 572           return i;
 573         }
 574     }
 575   return 0;
 576 }
 577
 578 static_always_inline int
 579 nat44_ei_hairpinning_enable (u32 sw_if_index, u8 is_enable)
 580 {
 581   return vnet_feature_enable_disable ("ip4-local", "nat44-ei-hairpinning",
 582                                       sw_if_index, is_enable, 0, 0);
 583 }
 584
 585 int
 586 nat44_ei_add_interface (u32 sw_if_index, u8 is_inside)
 587 {
 588   const char *feature_name, *del_feature_name;
 589   nat44_ei_main_t *nm = &nat44_ei_main;
 590
 591   nat44_ei_outside_fib_t *outside_fib;
 592   nat44_ei_interface_t *i;
 593   u32 fib_index;
 594   int rv;
 595
 596   fail_if_disabled ();
 597
 598   if (nm->out2in_dpo && !is_inside)
 599     {
 600       nat44_ei_log_err ("error unsupported");
 601       return VNET_API_ERROR_UNSUPPORTED;
 602     }
 603
 604   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 605     {
 606       nat44_ei_log_err ("error interface already configured");
 607       return VNET_API_ERROR_VALUE_EXIST;
 608     }
 609
 610   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 611   if (i)
 612     {
 613       if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 614           (nat44_ei_interface_is_outside (i) && !is_inside))
 615         {
 616           return 0;
 617         }
 618       if (nm->num_workers > 1)
 619         {
 620           del_feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 621                                           "nat44-ei-out2in-worker-handoff";
 622           feature_name = "nat44-ei-handoff-classify";
 623         }
 624       else
 625         {
 626           del_feature_name =
 627             !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 628
 629           feature_name = "nat44-ei-classify";
 630         }
 631
 632       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 633       if (rv)
 634         {
 635           return rv;
 636         }
 637       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 638                                         sw_if_index, 0, 0, 0);
 639       if (rv)
 640         {
 641           return rv;
 642         }
 643       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 644                                         sw_if_index, 1, 0, 0);
 645       if (rv)
 646         {
 647           return rv;
 648         }
 649       if (!is_inside)
 650         {
 651           rv = nat44_ei_hairpinning_enable (sw_if_index, 0);
 652           if (rv)
 653             {
 654               return rv;
 655             }
 656         }
 657     }
 658   else
 659     {
 660       if (nm->num_workers > 1)
 661         {
 662           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 663                                      "nat44-ei-out2in-worker-handoff";
 664         }
 665       else
 666         {
 667           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 668         }
 669       nat_validate_interface_counters (nm, sw_if_index);
 670
 671       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 672       if (rv)
 673         {
 674           return rv;
 675         }
 676       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 677                                         sw_if_index, 1, 0, 0);
 678       if (rv)
 679         {
 680           return rv;
 681         }
 682       if (is_inside && !nm->out2in_dpo)
 683         {
 684           rv = nat44_ei_hairpinning_enable (sw_if_index, 1);
 685           if (rv)
 686             {
 687               return rv;
 688             }
 689         }
 690
 691       pool_get (nm->interfaces, i);
 692       i->sw_if_index = sw_if_index;
 693       i->flags = 0;
 694     }
 695
 696   fib_index =
 697     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 698
 699   if (!is_inside)
 700     {
 701       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 702
 703       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 704       if (outside_fib)
 705         {
 706           outside_fib->refcount++;
 707         }
 708       else
 709         {
 710           vec_add2 (nm->outside_fibs, outside_fib, 1);
 711           outside_fib->fib_index = fib_index;
 712           outside_fib->refcount = 1;
 713         }
 714
 715       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 716       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 717     }
 718   else
 719     {
 720       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 721     }
 722
 723   return 0;
 724 }
 725
 726 int
 727 nat44_ei_del_interface (u32 sw_if_index, u8 is_inside)
 728 {
 729   const char *feature_name, *del_feature_name;
 730   nat44_ei_main_t *nm = &nat44_ei_main;
 731
 732   nat44_ei_outside_fib_t *outside_fib;
 733   nat44_ei_interface_t *i;
 734   u32 fib_index;
 735   int rv;
 736
 737   fail_if_disabled ();
 738
 739   if (nm->out2in_dpo && !is_inside)
 740     {
 741       nat44_ei_log_err ("error unsupported");
 742       return VNET_API_ERROR_UNSUPPORTED;
 743     }
 744
 745   i = nat44_ei_get_interface (nm->interfaces, sw_if_index);
 746   if (i == 0)
 747     {
 748       nat44_ei_log_err ("error interface couldn't be found");
 749       return VNET_API_ERROR_NO_SUCH_ENTRY;
 750     }
 751
 752   if (nat44_ei_interface_is_inside (i) && nat44_ei_interface_is_outside (i))
 753     {
 754       if (nm->num_workers > 1)
 755         {
 756           del_feature_name = "nat44-ei-handoff-classify";
 757           feature_name = !is_inside ? "nat44-ei-in2out-worker-handoff" :
 758                                       "nat44-ei-out2in-worker-handoff";
 759         }
 760       else
 761         {
 762           del_feature_name = "nat44-ei-classify";
 763           feature_name = !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 764         }
 765
 766       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 767       if (rv)
 768         {
 769           return rv;
 770         }
 771       rv = vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 772                                         sw_if_index, 0, 0, 0);
 773       if (rv)
 774         {
 775           return rv;
 776         }
 777       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 778                                         sw_if_index, 1, 0, 0);
 779       if (rv)
 780         {
 781           return rv;
 782         }
 783       if (is_inside)
 784         {
 785           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 786         }
 787       else
 788         {
 789           rv = nat44_ei_hairpinning_enable (sw_if_index, 1);
 790           if (rv)
 791             {
 792               return rv;
 793             }
 794           i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 795         }
 796     }
 797   else
 798     {
 799       if (nm->num_workers > 1)
 800         {
 801           feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 802                                      "nat44-ei-out2in-worker-handoff";
 803         }
 804       else
 805         {
 806           feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 807         }
 808
 809       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 810       if (rv)
 811         {
 812           return rv;
 813         }
 814       rv = vnet_feature_enable_disable ("ip4-unicast", feature_name,
 815                                         sw_if_index, 0, 0, 0);
 816       if (rv)
 817         {
 818           return rv;
 819         }
 820       if (is_inside)
 821         {
 822           rv = nat44_ei_hairpinning_enable (sw_if_index, 0);
 823           if (rv)
 824             {
 825               return rv;
 826             }
 827         }
 828
 829       // remove interface
 830       pool_put (nm->interfaces, i);
 831     }
 832
 833   if (!is_inside)
 834     {
 835       fib_index =
 836         fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 837       outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 838       if (outside_fib)
 839         {
 840           outside_fib->refcount--;
 841           if (!outside_fib->refcount)
 842             {
 843               vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
 844             }
 845         }
 846
 847       nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
 848       nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
 849     }
 850
 851   return 0;
 852 }
 853
 854 int
 855 nat44_ei_add_output_interface (u32 sw_if_index)
 856 {
 857   nat44_ei_main_t *nm = &nat44_ei_main;
 858
 859   nat44_ei_outside_fib_t *outside_fib;
 860   nat44_ei_interface_t *i;
 861   u32 fib_index;
 862   int rv;
 863
 864   fail_if_disabled ();
 865
 866   if (nat44_ei_get_interface (nm->interfaces, sw_if_index))
 867     {
 868       nat44_ei_log_err ("error interface already configured");
 869       return VNET_API_ERROR_VALUE_EXIST;
 870     }
 871
 872   if (nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index))
 873     {
 874       nat44_ei_log_err ("error interface already configured");
 875       return VNET_API_ERROR_VALUE_EXIST;
 876     }
 877
 878   if (nm->num_workers > 1)
 879     {
 880       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 881       if (rv)
 882         {
 883           return rv;
 884         }
 885       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 886       if (rv)
 887         {
 888           return rv;
 889         }
 890       rv = vnet_feature_enable_disable (
 891         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 1, 0, 0);
 892       if (rv)
 893         {
 894           return rv;
 895         }
 896       rv = vnet_feature_enable_disable (
 897         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 1,
 898         0, 0);
 899       if (rv)
 900         {
 901           return rv;
 902         }
 903     }
 904   else
 905     {
 906       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 907       if (rv)
 908         {
 909           return rv;
 910         }
 911       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
 912       if (rv)
 913         {
 914           return rv;
 915         }
 916       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 917                                         sw_if_index, 1, 0, 0);
 918       if (rv)
 919         {
 920           return rv;
 921         }
 922       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 923                                         sw_if_index, 1, 0, 0);
 924       if (rv)
 925         {
 926           return rv;
 927         }
 928     }
 929
 930   nat_validate_interface_counters (nm, sw_if_index);
 931
 932   pool_get (nm->output_feature_interfaces, i);
 933   i->sw_if_index = sw_if_index;
 934   i->flags = 0;
 935   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 936   i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 937
 938   fib_index =
 939     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 940   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
 941   if (outside_fib)
 942     {
 943       outside_fib->refcount++;
 944     }
 945   else
 946     {
 947       vec_add2 (nm->outside_fibs, outside_fib, 1);
 948       outside_fib->fib_index = fib_index;
 949       outside_fib->refcount = 1;
 950     }
 951
 952   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
 953   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
 954
 955   return 0;
 956 }
 957
 958 int
 959 nat44_ei_del_output_interface (u32 sw_if_index)
 960 {
 961   nat44_ei_main_t *nm = &nat44_ei_main;
 962
 963   nat44_ei_outside_fib_t *outside_fib;
 964   nat44_ei_interface_t *i;
 965   u32 fib_index;
 966   int rv;
 967
 968   fail_if_disabled ();
 969
 970   i = nat44_ei_get_interface (nm->output_feature_interfaces, sw_if_index);
 971   if (!i)
 972     {
 973       nat44_ei_log_err ("error interface couldn't be found");
 974       return VNET_API_ERROR_NO_SUCH_ENTRY;
 975     }
 976
 977   if (nm->num_workers > 1)
 978     {
 979       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 980       if (rv)
 981         {
 982           return rv;
 983         }
 984       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
 985       if (rv)
 986         {
 987           return rv;
 988         }
 989       rv = vnet_feature_enable_disable (
 990         "ip4-unicast", "nat44-ei-out2in-worker-handoff", sw_if_index, 0, 0, 0);
 991       if (rv)
 992         {
 993           return rv;
 994         }
 995       rv = vnet_feature_enable_disable (
 996         "ip4-output", "nat44-ei-in2out-output-worker-handoff", sw_if_index, 0,
 997         0, 0);
 998       if (rv)
 999         {
1000           return rv;
1001         }
1002     }
1003   else
1004     {
1005       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1006       if (rv)
1007         {
1008           return rv;
1009         }
1010       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1011       if (rv)
1012         {
1013           return rv;
1014         }
1015       rv = vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
1016                                         sw_if_index, 0, 0, 0);
1017       if (rv)
1018         {
1019           return rv;
1020         }
1021       rv = vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
1022                                         sw_if_index, 0, 0, 0);
1023       if (rv)
1024         {
1025           return rv;
1026         }
1027     }
1028
1029   pool_put (nm->output_feature_interfaces, i);
1030
1031   fib_index =
1032     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1033   outside_fib = nat44_ei_get_outside_fib (nm->outside_fibs, fib_index);
1034   if (outside_fib)
1035     {
1036       outside_fib->refcount--;
1037       if (!outside_fib->refcount)
1038         {
1039           vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
1040         }
1041     }
1042
1043   nat44_ei_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1044   nat44_ei_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1045
1046   return 0;
1047 }
1048
1049 int
1050 nat44_ei_add_del_output_interface (u32 sw_if_index, int is_del)
1051 {
1052   if (is_del)
1053     {
1054       return nat44_ei_del_output_interface (sw_if_index);
1055     }
1056   else
1057     {
1058       return nat44_ei_add_output_interface (sw_if_index);
1059     }
1060 }
1061
1062 int
1063 nat44_ei_del_addresses ()
1064 {
1065   nat44_ei_main_t *nm = &nat44_ei_main;
1066   nat44_ei_address_t *a, *vec;
1067   int error = 0;
1068
1069   vec = vec_dup (nm->addresses);
1070   vec_foreach (a, vec)
1071     {
1072       error = nat44_ei_del_address (a->addr, 0);
1073
1074       if (error)
1075         {
1076           nat44_ei_log_err ("error occurred while removing adderess");
1077         }
1078     }
1079   vec_free (vec);
1080   vec_free (nm->addresses);
1081   nm->addresses = 0;
1082
1083   vec_free (nm->auto_add_sw_if_indices);
1084   nm->auto_add_sw_if_indices = 0;
1085   return error;
1086 }
1087
1088 int
1089 nat44_ei_del_interfaces ()
1090 {
1091   nat44_ei_main_t *nm = &nat44_ei_main;
1092   nat44_ei_interface_t *i, *pool;
1093   int error = 0;
1094
1095   pool = pool_dup (nm->interfaces);
1096   pool_foreach (i, pool)
1097     {
1098       if (nat44_ei_interface_is_inside (i))
1099         {
1100           error = nat44_ei_del_interface (i->sw_if_index, 1);
1101         }
1102       if (nat44_ei_interface_is_outside (i))
1103         {
1104           error = nat44_ei_del_interface (i->sw_if_index, 0);
1105         }
1106
1107       if (error)
1108         {
1109           nat44_ei_log_err ("error occurred while removing interface");
1110         }
1111     }
1112   pool_free (pool);
1113   pool_free (nm->interfaces);
1114   nm->interfaces = 0;
1115   return error;
1116 }
1117
1118 int
1119 nat44_ei_del_output_interfaces ()
1120 {
1121   nat44_ei_main_t *nm = &nat44_ei_main;
1122   nat44_ei_interface_t *i, *pool;
1123   int error = 0;
1124
1125   pool = pool_dup (nm->output_feature_interfaces);
1126   pool_foreach (i, pool)
1127     {
1128       error = nat44_ei_del_output_interface (i->sw_if_index);
1129       if (error)
1130         {
1131           nat44_ei_log_err ("error occurred while removing output interface");
1132         }
1133     }
1134   pool_free (pool);
1135   pool_free (nm->output_feature_interfaces);
1136   nm->output_feature_interfaces = 0;
1137   return error;
1138 }
1139
1140 int
1141 nat44_ei_del_static_mappings ()
1142 {
1143   nat44_ei_main_t *nm = &nat44_ei_main;
1144   nat44_ei_static_mapping_t *m, *pool;
1145   int error = 0;
1146
1147   pool = pool_dup (nm->static_mappings);
1148   pool_foreach (m, pool)
1149     {
1150       error = nat44_ei_del_static_mapping_internal (
1151         m->local_addr, m->external_addr, m->local_port, m->external_port,
1152         m->proto, m->vrf_id, ~0, m->flags);
1153       if (error)
1154         {
1155           nat44_ei_log_err ("error occurred while removing mapping");
1156         }
1157     }
1158   pool_free (pool);
1159   pool_free (nm->static_mappings);
1160   nm->static_mappings = 0;
1161
1162   vec_free (nm->to_resolve);
1163   nm->to_resolve = 0;
1164
1165   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
1166   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
1167
1168   return error;
1169 }
1170
1171 int
1172 nat44_ei_plugin_disable ()
1173 {
1174   nat44_ei_main_t *nm = &nat44_ei_main;
1175   nat44_ei_main_per_thread_data_t *tnm;
1176   int rc, error = 0;
1177
1178   nat_ha_disable ();
1179
1180   rc = nat44_ei_del_static_mappings ();
1181   if (rc)
1182     error = 1;
1183
1184   rc = nat44_ei_del_addresses ();
1185   if (rc)
1186     error = 1;
1187
1188   rc = nat44_ei_del_interfaces ();
1189   if (rc)
1190     error = 1;
1191
1192   rc = nat44_ei_del_output_interfaces ();
1193   if (rc)
1194     error = 1;
1195
1196   if (nm->pat)
1197     {
1198       clib_bihash_free_8_8 (&nm->in2out);
1199       clib_bihash_free_8_8 (&nm->out2in);
1200
1201       vec_foreach (tnm, nm->per_thread_data)
1202         {
1203           nat44_ei_worker_db_free (tnm);
1204         }
1205     }
1206
1207   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
1208
1209   nm->forwarding_enabled = 0;
1210   nm->enabled = 0;
1211
1212   return error;
1213 }
1214
1215 int
1216 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
1217                                        u32 thread_index, ip4_address_t addr,
1218                                        u16 port, nat_protocol_t protocol)
1219 {
1220   nat44_ei_main_t *nm = &nat44_ei_main;
1221   nat44_ei_address_t *a = 0;
1222   u32 address_index;
1223   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1224
1225   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1226     {
1227       if (addresses[address_index].addr.as_u32 != addr.as_u32)
1228         continue;
1229
1230       a = addresses + address_index;
1231       switch (protocol)
1232         {
1233 #define _(N, j, n, s)                                                         \
1234   case NAT_PROTOCOL_##N:                                                      \
1235     if (a->busy_##n##_port_refcounts[port_host_byte_order])                   \
1236       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
1237     ++a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1238     a->busy_##n##_ports_per_thread[thread_index]++;                           \
1239     a->busy_##n##_ports++;                                                    \
1240     return 0;
1241           foreach_nat_protocol
1242 #undef _
1243             default : nat_elog_info (nm, "unknown protocol");
1244           return 1;
1245         }
1246     }
1247
1248   return VNET_API_ERROR_NO_SUCH_ENTRY;
1249 }
1250
1251 void
1252 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
1253 {
1254   nat44_ei_main_t *nm = &nat44_ei_main;
1255   dpo_id_t dpo_v4 = DPO_INVALID;
1256   fib_prefix_t pfx = {
1257     .fp_proto = FIB_PROTOCOL_IP4,
1258     .fp_len = 32,
1259     .fp_addr.ip4.as_u32 = addr.as_u32,
1260   };
1261
1262   if (is_add)
1263     {
1264       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
1265       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
1266                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
1267       dpo_reset (&dpo_v4);
1268     }
1269   else
1270     {
1271       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
1272     }
1273 }
1274
1275 void
1276 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1277                                         u32 thread_index, ip4_address_t *addr,
1278                                         u16 port, nat_protocol_t protocol)
1279 {
1280   nat44_ei_main_t *nm = &nat44_ei_main;
1281   nat44_ei_address_t *a;
1282   u32 address_index;
1283   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1284
1285   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1286     {
1287       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1288         break;
1289     }
1290
1291   ASSERT (address_index < vec_len (addresses));
1292
1293   a = addresses + address_index;
1294
1295   switch (protocol)
1296     {
1297 #define _(N, i, n, s)                                                         \
1298   case NAT_PROTOCOL_##N:                                                      \
1299     ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1);         \
1300     --a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1301     a->busy_##n##_ports--;                                                    \
1302     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1303     break;
1304       foreach_nat_protocol
1305 #undef _
1306         default : nat_elog_info (nm, "unknown protocol");
1307       return;
1308     }
1309 }
1310
1311 void
1312 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1313                                u32 thread_index, u8 is_ha)
1314 {
1315   clib_bihash_kv_8_8_t kv;
1316
1317   /* session lookup tables */
1318   init_nat_i2o_k (&kv, s);
1319   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1320     nat_elog_warn (nm, "in2out key del failed");
1321   init_nat_o2i_k (&kv, s);
1322   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1323     nat_elog_warn (nm, "out2in key del failed");
1324
1325   if (!is_ha)
1326     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1327                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1328                              s->out2in.port, s->nat_proto);
1329
1330   if (nat44_ei_is_unk_proto_session (s))
1331     return;
1332
1333   if (!is_ha)
1334     {
1335       /* log NAT event */
1336       nat_ipfix_logging_nat44_ses_delete (
1337         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1338         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1339         s->in2out.fib_index);
1340
1341       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1342                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1343                    thread_index);
1344     }
1345
1346   if (nat44_ei_is_session_static (s))
1347     return;
1348
1349   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1350                                           &s->out2in.addr, s->out2in.port,
1351                                           s->nat_proto);
1352 }
1353
1354 nat44_ei_user_t *
1355 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1356                              u32 fib_index, u32 thread_index)
1357 {
1358   nat44_ei_user_t *u = 0;
1359   nat44_ei_user_key_t user_key;
1360   clib_bihash_kv_8_8_t kv, value;
1361   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1362   dlist_elt_t *per_user_list_head_elt;
1363
1364   user_key.addr.as_u32 = addr->as_u32;
1365   user_key.fib_index = fib_index;
1366   kv.key = user_key.as_u64;
1367
1368   /* Ever heard of the "user" = src ip4 address before? */
1369   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1370     {
1371       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1372         {
1373           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1374                                          0, 1);
1375           nat_elog_warn (nm, "maximum user limit reached");
1376           return NULL;
1377         }
1378       /* no, make a new one */
1379       pool_get (tnm->users, u);
1380       clib_memset (u, 0, sizeof (*u));
1381
1382       u->addr.as_u32 = addr->as_u32;
1383       u->fib_index = fib_index;
1384
1385       pool_get (tnm->list_pool, per_user_list_head_elt);
1386
1387       u->sessions_per_user_list_head_index =
1388         per_user_list_head_elt - tnm->list_pool;
1389
1390       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1391
1392       kv.value = u - tnm->users;
1393
1394       /* add user */
1395       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1396         {
1397           nat_elog_warn (nm, "user_hash key add failed");
1398           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1399           return NULL;
1400         }
1401
1402       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1403                                pool_elts (tnm->users));
1404     }
1405   else
1406     {
1407       u = pool_elt_at_index (tnm->users, value.value);
1408     }
1409
1410   return u;
1411 }
1412
1413 nat44_ei_session_t *
1414 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1415                                    u32 thread_index, f64 now)
1416 {
1417   nat44_ei_session_t *s;
1418   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1419   u32 oldest_per_user_translation_list_index, session_index;
1420   dlist_elt_t *oldest_per_user_translation_list_elt;
1421   dlist_elt_t *per_user_translation_list_elt;
1422
1423   /* Over quota? Recycle the least recently used translation */
1424   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1425     {
1426       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1427         tnm->list_pool, u->sessions_per_user_list_head_index);
1428
1429       ASSERT (oldest_per_user_translation_list_index != ~0);
1430
1431       /* Add it back to the end of the LRU list */
1432       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1433                           oldest_per_user_translation_list_index);
1434       /* Get the list element */
1435       oldest_per_user_translation_list_elt = pool_elt_at_index (
1436         tnm->list_pool, oldest_per_user_translation_list_index);
1437
1438       /* Get the session index from the list element */
1439       session_index = oldest_per_user_translation_list_elt->value;
1440
1441       /* Get the session */
1442       s = pool_elt_at_index (tnm->sessions, session_index);
1443
1444       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1445       if (nat44_ei_is_session_static (s))
1446         u->nstaticsessions--;
1447       else
1448         u->nsessions--;
1449       s->flags = 0;
1450       s->total_bytes = 0;
1451       s->total_pkts = 0;
1452       s->state = 0;
1453       s->ext_host_addr.as_u32 = 0;
1454       s->ext_host_port = 0;
1455       s->ext_host_nat_addr.as_u32 = 0;
1456       s->ext_host_nat_port = 0;
1457     }
1458   else
1459     {
1460       pool_get (tnm->sessions, s);
1461       clib_memset (s, 0, sizeof (*s));
1462
1463       /* Create list elts */
1464       pool_get (tnm->list_pool, per_user_translation_list_elt);
1465       clib_dlist_init (tnm->list_pool,
1466                        per_user_translation_list_elt - tnm->list_pool);
1467
1468       per_user_translation_list_elt->value = s - tnm->sessions;
1469       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1470       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1471
1472       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1473                           per_user_translation_list_elt - tnm->list_pool);
1474
1475       s->user_index = u - tnm->users;
1476       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1477                                pool_elts (tnm->sessions));
1478     }
1479
1480   s->ha_last_refreshed = now;
1481
1482   return s;
1483 }
1484
1485 void
1486 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1487                             u32 thread_index, u8 is_ha)
1488 {
1489   clib_bihash_kv_8_8_t kv;
1490
1491   init_nat_i2o_k (&kv, s);
1492   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1493     nat_elog_warn (nm, "in2out key del failed");
1494
1495   init_nat_o2i_k (&kv, s);
1496   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1497     nat_elog_warn (nm, "out2in key del failed");
1498
1499   if (!is_ha)
1500     {
1501       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1502                                &s->in2out.addr, s->in2out.port,
1503                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1504
1505       nat_ipfix_logging_nat44_ses_delete (
1506         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1507         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
1508         s->in2out.fib_index);
1509
1510       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1511                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1512                    thread_index);
1513     }
1514
1515   if (nat44_ei_is_session_static (s))
1516     return;
1517
1518   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1519                                           &s->out2in.addr, s->out2in.port,
1520                                           s->nat_proto);
1521 }
1522
1523 static_always_inline void
1524 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1525 {
1526   dlist_elt_t *elt;
1527   nat44_ei_session_t *s;
1528
1529   nat44_ei_main_t *nm = &nat44_ei_main;
1530   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1531
1532   // get head
1533   elt =
1534     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1535   // get first element
1536   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1537
1538   while (elt->value != ~0)
1539     {
1540       s = pool_elt_at_index (tnm->sessions, elt->value);
1541       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1542
1543       nat44_ei_free_session_data (nm, s, thread_index, 0);
1544       nat44_ei_delete_session (nm, s, thread_index);
1545     }
1546 }
1547
1548 int
1549 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1550 {
1551   int rv = 1;
1552
1553   nat44_ei_main_t *nm = &nat44_ei_main;
1554   nat44_ei_main_per_thread_data_t *tnm;
1555
1556   nat44_ei_user_key_t user_key;
1557   clib_bihash_kv_8_8_t kv, value;
1558
1559   user_key.addr.as_u32 = addr->as_u32;
1560   user_key.fib_index = fib_index;
1561   kv.key = user_key.as_u64;
1562
1563   if (nm->num_workers > 1)
1564     {
1565       vec_foreach (tnm, nm->per_thread_data)
1566         {
1567           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1568             {
1569               nat44_ei_user_del_sessions (
1570                 pool_elt_at_index (tnm->users, value.value),
1571                 tnm->thread_index);
1572               rv = 0;
1573               break;
1574             }
1575         }
1576     }
1577   else
1578     {
1579       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1580       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1581         {
1582           nat44_ei_user_del_sessions (
1583             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1584           rv = 0;
1585         }
1586     }
1587   return rv;
1588 }
1589
1590 void
1591 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1592                                       nat44_ei_main_per_thread_data_t *tnm,
1593                                       nat44_ei_user_key_t u_key, int addr_only,
1594                                       ip4_address_t e_addr, u16 e_port)
1595 {
1596   clib_bihash_kv_8_8_t kv, value;
1597   kv.key = u_key.as_u64;
1598   u64 user_index;
1599   dlist_elt_t *head, *elt;
1600   nat44_ei_user_t *u;
1601   nat44_ei_session_t *s;
1602   u32 elt_index, head_index, ses_index;
1603
1604   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1605     {
1606       user_index = value.value;
1607       u = pool_elt_at_index (tnm->users, user_index);
1608       if (u->nstaticsessions)
1609         {
1610           head_index = u->sessions_per_user_list_head_index;
1611           head = pool_elt_at_index (tnm->list_pool, head_index);
1612           elt_index = head->next;
1613           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1614           ses_index = elt->value;
1615           while (ses_index != ~0)
1616             {
1617               s = pool_elt_at_index (tnm->sessions, ses_index);
1618               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1619               ses_index = elt->value;
1620
1621               if (!addr_only)
1622                 {
1623                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1624                       (s->out2in.port != e_port))
1625                     continue;
1626                 }
1627
1628               if (!nat44_ei_is_session_static (s))
1629                 continue;
1630
1631               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1632                                              0);
1633               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1634
1635               if (!addr_only)
1636                 break;
1637             }
1638         }
1639     }
1640 }
1641
1642 u32
1643 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1644                                   u8 is_output)
1645 {
1646   nat44_ei_main_t *nm = &nat44_ei_main;
1647   u32 next_worker_index = 0;
1648   u32 hash;
1649
1650   next_worker_index = nm->first_worker_index;
1651   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1652          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1653
1654   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1655     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1656   else
1657     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1658
1659   return next_worker_index;
1660 }
1661
1662 u32
1663 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1664                                   u32 rx_fib_index0, u8 is_output)
1665 {
1666   nat44_ei_main_t *nm = &nat44_ei_main;
1667   udp_header_t *udp;
1668   u16 port;
1669   clib_bihash_kv_8_8_t kv, value;
1670   nat44_ei_static_mapping_t *m;
1671   u32 proto;
1672   u32 next_worker_index = 0;
1673
1674   /* first try static mappings without port */
1675   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1676     {
1677       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1678       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1679                                    &value))
1680         {
1681           m = pool_elt_at_index (nm->static_mappings, value.value);
1682           return m->workers[0];
1683         }
1684     }
1685
1686   proto = ip_proto_to_nat_proto (ip0->protocol);
1687   udp = ip4_next_header (ip0);
1688   port = vnet_buffer (b)->ip.reass.l4_dst_port;
1689
1690   /* unknown protocol */
1691   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1692     {
1693       /* use current thread */
1694       return vlib_get_thread_index ();
1695     }
1696
1697   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1698     {
1699       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1700       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1701       if (!icmp_type_is_error_message (
1702             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1703         port = vnet_buffer (b)->ip.reass.l4_src_port;
1704       else
1705         {
1706           /* if error message, then it's not fragmented and we can access it */
1707           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1708           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1709           void *l4_header = ip4_next_header (inner_ip);
1710           switch (proto)
1711             {
1712             case NAT_PROTOCOL_ICMP:
1713               icmp = (icmp46_header_t *) l4_header;
1714               echo = (icmp_echo_header_t *) (icmp + 1);
1715               port = echo->identifier;
1716               break;
1717             case NAT_PROTOCOL_UDP:
1718             case NAT_PROTOCOL_TCP:
1719               port = ((tcp_udp_header_t *) l4_header)->src_port;
1720               break;
1721             default:
1722               return vlib_get_thread_index ();
1723             }
1724         }
1725     }
1726
1727   /* try static mappings with port */
1728   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1729     {
1730       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1731       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1732                                    &value))
1733         {
1734           m = pool_elt_at_index (nm->static_mappings, value.value);
1735           return m->workers[0];
1736         }
1737     }
1738
1739   /* worker by outside port */
1740   next_worker_index = nm->first_worker_index;
1741   next_worker_index +=
1742     nm->workers[(clib_net_to_host_u16 (port) - 1024) / nm->port_per_thread];
1743   return next_worker_index;
1744 }
1745
1746 static int
1747 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1748                            u32 thread_index, nat_protocol_t proto,
1749                            ip4_address_t s_addr, ip4_address_t *addr,
1750                            u16 *port, u16 port_per_thread,
1751                            u32 snat_thread_index)
1752 {
1753   nat44_ei_main_t *nm = &nat44_ei_main;
1754   nat44_ei_address_t *a, *ga = 0;
1755   u32 portnum;
1756   int i;
1757
1758   if (vec_len (addresses) > 0)
1759     {
1760
1761       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1762
1763       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1764         {
1765           a = addresses + i;
1766           switch (proto)
1767             {
1768 #define _(N, j, n, s)                                                         \
1769   case NAT_PROTOCOL_##N:                                                      \
1770     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1771       {                                                                       \
1772         if (a->fib_index == fib_index)                                        \
1773           {                                                                   \
1774             while (1)                                                         \
1775               {                                                               \
1776                 portnum = (port_per_thread * snat_thread_index) +             \
1777                           nat_random_port (&nm->random_seed, 0,               \
1778                                            port_per_thread - 1) +             \
1779                           1024;                                               \
1780                 if (a->busy_##n##_port_refcounts[portnum])                    \
1781                   continue;                                                   \
1782                 --a->busy_##n##_port_refcounts[portnum];                      \
1783                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1784                 a->busy_##n##_ports++;                                        \
1785                 *addr = a->addr;                                              \
1786                 *port = clib_host_to_net_u16 (portnum);                       \
1787                 return 0;                                                     \
1788               }                                                               \
1789           }                                                                   \
1790         else if (a->fib_index == ~0)                                          \
1791           {                                                                   \
1792             ga = a;                                                           \
1793           }                                                                   \
1794       }                                                                       \
1795     break;
1796               foreach_nat_protocol;
1797             default:
1798               nat_elog_info (nm, "unknown protocol");
1799               return 1;
1800             }
1801         }
1802
1803       for (i = 0; i < s_addr_offset; ++i)
1804         {
1805           a = addresses + i;
1806           switch (proto)
1807             {
1808               foreach_nat_protocol;
1809             default:
1810               nat_elog_info (nm, "unknown protocol");
1811               return 1;
1812             }
1813         }
1814   if (ga)
1815     {
1816       a = ga;
1817       // fake fib index to reuse macro
1818       fib_index = ~0;
1819       switch (proto)
1820         {
1821           foreach_nat_protocol;
1822             default : nat_elog_info (nm, "unknown protocol");
1823           return 1;
1824         }
1825     }
1826     }
1827
1828 #undef _
1829
1830   /* Totally out of translations to use... */
1831   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1832   return 1;
1833 }
1834
1835 static int
1836 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1837                          u32 thread_index, nat_protocol_t proto,
1838                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1839                          u16 port_per_thread, u32 snat_thread_index)
1840 {
1841   nat44_ei_main_t *nm = &nat44_ei_main;
1842   nat44_ei_address_t *a = addresses;
1843   u16 portnum, ports;
1844
1845   ports = nm->end_port - nm->start_port + 1;
1846
1847   if (!vec_len (addresses))
1848     goto exhausted;
1849
1850   switch (proto)
1851     {
1852 #define _(N, i, n, s)                                                         \
1853   case NAT_PROTOCOL_##N:                                                      \
1854     if (a->busy_##n##_ports < ports)                                          \
1855       {                                                                       \
1856         while (1)                                                             \
1857           {                                                                   \
1858             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1859                                        nm->end_port);                         \
1860             if (a->busy_##n##_port_refcounts[portnum])                        \
1861               continue;                                                       \
1862             ++a->busy_##n##_port_refcounts[portnum];                          \
1863             a->busy_##n##_ports++;                                            \
1864             *addr = a->addr;                                                  \
1865             *port = clib_host_to_net_u16 (portnum);                           \
1866             return 0;                                                         \
1867           }                                                                   \
1868       }                                                                       \
1869     break;
1870       foreach_nat_protocol
1871 #undef _
1872         default : nat_elog_info (nm, "unknown protocol");
1873       return 1;
1874     }
1875
1876 exhausted:
1877   /* Totally out of translations to use... */
1878   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1879   return 1;
1880 }
1881
1882 static int
1883 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1884                         u32 thread_index, nat_protocol_t proto,
1885                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1886                         u16 port_per_thread, u32 snat_thread_index)
1887 {
1888   nat44_ei_main_t *nm = &nat44_ei_main;
1889   nat44_ei_address_t *a = addresses;
1890   u16 m, ports, portnum, A, j;
1891   m = 16 - (nm->psid_offset + nm->psid_length);
1892   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1893
1894   if (!vec_len (addresses))
1895     goto exhausted;
1896
1897   switch (proto)
1898     {
1899 #define _(N, i, n, s)                                                         \
1900   case NAT_PROTOCOL_##N:                                                      \
1901     if (a->busy_##n##_ports < ports)                                          \
1902       {                                                                       \
1903         while (1)                                                             \
1904           {                                                                   \
1905             A = nat_random_port (&nm->random_seed, 1,                         \
1906                                  pow2_mask (nm->psid_offset));                \
1907             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
1908             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
1909             if (a->busy_##n##_port_refcounts[portnum])                        \
1910               continue;                                                       \
1911             ++a->busy_##n##_port_refcounts[portnum];                          \
1912             a->busy_##n##_ports++;                                            \
1913             *addr = a->addr;                                                  \
1914             *port = clib_host_to_net_u16 (portnum);                           \
1915             return 0;                                                         \
1916           }                                                                   \
1917       }                                                                       \
1918     break;
1919       foreach_nat_protocol
1920 #undef _
1921         default : nat_elog_info (nm, "unknown protocol");
1922       return 1;
1923     }
1924
1925 exhausted:
1926   /* Totally out of translations to use... */
1927   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1928   return 1;
1929 }
1930
1931 void
1932 nat44_ei_set_alloc_default ()
1933 {
1934   nat44_ei_main_t *nm = &nat44_ei_main;
1935
1936   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1937   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1938 }
1939
1940 void
1941 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1942 {
1943   nat44_ei_main_t *nm = &nat44_ei_main;
1944
1945   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1946   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1947   nm->start_port = start_port;
1948   nm->end_port = end_port;
1949 }
1950
1951 void
1952 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1953 {
1954   nat44_ei_main_t *nm = &nat44_ei_main;
1955
1956   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1957   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1958   nm->psid = psid;
1959   nm->psid_offset = psid_offset;
1960   nm->psid_length = psid_length;
1961 }
1962
1963 void
1964 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
1965                          u32 thread_index)
1966 {
1967   nat44_ei_main_per_thread_data_t *tnm =
1968     vec_elt_at_index (nm->per_thread_data, thread_index);
1969   clib_bihash_kv_8_8_t kv, value;
1970   nat44_ei_user_t *u;
1971   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
1972                                       .fib_index = ses->in2out.fib_index };
1973   const u8 u_static = nat44_ei_is_session_static (ses);
1974
1975   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
1976   pool_put_index (tnm->list_pool, ses->per_user_index);
1977
1978   pool_put (tnm->sessions, ses);
1979   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1980                            pool_elts (tnm->sessions));
1981
1982   kv.key = u_key.as_u64;
1983   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1984     {
1985       u = pool_elt_at_index (tnm->users, value.value);
1986       if (u_static)
1987         u->nstaticsessions--;
1988       else
1989         u->nsessions--;
1990
1991       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1992     }
1993 }
1994
1995 int
1996 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
1997                       nat_protocol_t proto, u32 vrf_id, int is_in)
1998 {
1999   nat44_ei_main_per_thread_data_t *tnm;
2000   clib_bihash_kv_8_8_t kv, value;
2001   u32 fib_index;
2002   nat44_ei_session_t *s;
2003   clib_bihash_8_8_t *t;
2004
2005   fail_if_disabled ();
2006
2007   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
2008   init_nat_k (&kv, *addr, port, fib_index, proto);
2009   t = is_in ? &nm->in2out : &nm->out2in;
2010   if (!clib_bihash_search_8_8 (t, &kv, &value))
2011     {
2012       // this is called from API/CLI, so the world is stopped here
2013       // it's safe to manipulate arbitrary per-thread data
2014       u32 thread_index = nat_value_get_thread_index (&value);
2015       tnm = vec_elt_at_index (nm->per_thread_data, thread_index);
2016       u32 session_index = nat_value_get_session_index (&value);
2017       if (pool_is_free_index (tnm->sessions, session_index))
2018         return VNET_API_ERROR_UNSPECIFIED;
2019
2020       s = pool_elt_at_index (tnm->sessions, session_index);
2021       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
2022       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2023       return 0;
2024     }
2025
2026   return VNET_API_ERROR_NO_SUCH_ENTRY;
2027 }
2028
2029 u32
2030 nat44_ei_get_thread_idx_by_port (u16 e_port)
2031 {
2032   nat44_ei_main_t *nm = &nat44_ei_main;
2033   u32 thread_idx = nm->num_workers;
2034   if (nm->num_workers > 1)
2035     {
2036       thread_idx = nm->first_worker_index +
2037                    nm->workers[(e_port - 1024) / nm->port_per_thread];
2038     }
2039   return thread_idx;
2040 }
2041
2042 void
2043 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
2044                               int is_add)
2045 {
2046   nat44_ei_main_t *nm = &nat44_ei_main;
2047   fib_prefix_t prefix = {
2048     .fp_len = p_len,
2049     .fp_proto = FIB_PROTOCOL_IP4,
2050     .fp_addr = {
2051                 .ip4.as_u32 = addr->as_u32,
2052                 },
2053   };
2054   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2055
2056   if (is_add)
2057     {
2058       fib_table_entry_update_one_path (fib_index, &prefix, nm->fib_src_low,
2059                                        (FIB_ENTRY_FLAG_CONNECTED |
2060                                         FIB_ENTRY_FLAG_LOCAL |
2061                                         FIB_ENTRY_FLAG_EXCLUSIVE),
2062                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
2063                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
2064     }
2065   else
2066     {
2067       fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
2068     }
2069 }
2070
2071 int
2072 nat44_ei_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2073 {
2074   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2075   nat44_ei_main_t *nm = &nat44_ei_main;
2076   nat44_ei_address_t *a = 0;
2077   int i;
2078
2079   for (i = 0; i < vec_len (nm->addresses); i++)
2080     {
2081       a = nm->addresses + i;
2082
2083       if (a->addr.as_u32 != addr.as_u32)
2084         continue;
2085
2086       switch (proto)
2087         {
2088 #define _(N, j, n, s)                                                         \
2089   case NAT_PROTOCOL_##N:                                                      \
2090     if (a->busy_##n##_port_refcounts[port])                                   \
2091       goto done;                                                              \
2092     ++a->busy_##n##_port_refcounts[port];                                     \
2093     if (port > 1024)                                                          \
2094       {                                                                       \
2095         a->busy_##n##_ports++;                                                \
2096         a->busy_##n##_ports_per_thread[ti]++;                                 \
2097       }                                                                       \
2098     break;
2099           foreach_nat_protocol
2100 #undef _
2101             default : nat_elog_info (nm, "unknown protocol");
2102           goto done;
2103         }
2104
2105       return 0;
2106     }
2107
2108 done:
2109   return 1;
2110 }
2111
2112 int
2113 nat44_ei_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
2114 {
2115   u32 ti = nat44_ei_get_thread_idx_by_port (port);
2116   nat44_ei_main_t *nm = &nat44_ei_main;
2117   nat44_ei_address_t *a = 0;
2118   int i;
2119
2120   for (i = 0; i < vec_len (nm->addresses); i++)
2121     {
2122       a = nm->addresses + i;
2123
2124       if (a->addr.as_u32 != addr.as_u32)
2125         continue;
2126
2127       switch (proto)
2128         {
2129 #define _(N, j, n, s)                                                         \
2130   case NAT_PROTOCOL_##N:                                                      \
2131     --a->busy_##n##_port_refcounts[port];                                     \
2132     if (port > 1024)                                                          \
2133       {                                                                       \
2134         a->busy_##n##_ports--;                                                \
2135         a->busy_##n##_ports_per_thread[ti]--;                                 \
2136       }                                                                       \
2137     break;
2138           foreach_nat_protocol
2139 #undef _
2140             default : nat_elog_info (nm, "unknown protocol");
2141           goto done;
2142         }
2143
2144       return 0;
2145     }
2146
2147 done:
2148   return 1;
2149 }
2150
2151 void
2152 nat44_ei_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2153                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2154                              u32 flags, ip4_address_t pool_addr, u8 *tag)
2155 {
2156   nat44_ei_static_map_resolve_t *rp;
2157   nat44_ei_main_t *nm = &nat44_ei_main;
2158
2159   vec_add2 (nm->to_resolve, rp, 1);
2160   rp->l_addr.as_u32 = l_addr.as_u32;
2161   rp->l_port = l_port;
2162   rp->e_port = e_port;
2163   rp->sw_if_index = sw_if_index;
2164   rp->vrf_id = vrf_id;
2165   rp->proto = proto;
2166   rp->flags = flags;
2167   rp->pool_addr = pool_addr;
2168   rp->tag = vec_dup (tag);
2169 }
2170
2171 int
2172 nat44_ei_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2173                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2174                              u32 flags, int *out)
2175 {
2176   nat44_ei_static_map_resolve_t *rp;
2177   nat44_ei_main_t *nm = &nat44_ei_main;
2178   int i;
2179
2180   for (i = 0; i < vec_len (nm->to_resolve); i++)
2181     {
2182       rp = nm->to_resolve + i;
2183
2184       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
2185         {
2186           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
2187             {
2188               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2189                 {
2190                   if (rp->e_port != e_port || rp->proto != proto)
2191                     {
2192                       continue;
2193                     }
2194                 }
2195             }
2196           else if (rp->l_addr.as_u32 == l_addr.as_u32)
2197             {
2198               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
2199                 {
2200                   if (rp->l_port != l_port || rp->e_port != e_port ||
2201                       rp->proto != proto)
2202                     {
2203                       continue;
2204                     }
2205                 }
2206             }
2207           else
2208             {
2209               continue;
2210             }
2211           if (out)
2212             {
2213               *out = i;
2214             }
2215           return 0;
2216         }
2217     }
2218   return 1;
2219 }
2220
2221 int
2222 nat44_ei_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
2223                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
2224                              u32 flags)
2225 {
2226   nat44_ei_main_t *nm = &nat44_ei_main;
2227   int i;
2228   if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2229                                     sw_if_index, flags, &i))
2230     {
2231       vec_del1 (nm->to_resolve, i);
2232       return 0;
2233     }
2234   return 1;
2235 }
2236
2237 void
2238 delete_matching_dynamic_sessions (const nat44_ei_static_mapping_t *m,
2239                                   u32 worker_index)
2240 {
2241   nat44_ei_main_t *nm = &nat44_ei_main;
2242   clib_bihash_kv_8_8_t kv, value;
2243   nat44_ei_session_t *s;
2244   nat44_ei_user_key_t u_key;
2245   nat44_ei_user_t *u;
2246   nat44_ei_main_per_thread_data_t *tnm;
2247   dlist_elt_t *head, *elt;
2248   u32 elt_index, head_index;
2249   u32 ses_index;
2250   u64 user_index;
2251
2252   if (nm->static_mapping_only)
2253     return;
2254
2255   tnm = vec_elt_at_index (nm->per_thread_data, worker_index);
2256
2257   u_key.addr = m->local_addr;
2258   u_key.fib_index = m->fib_index;
2259   kv.key = u_key.as_u64;
2260   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2261     {
2262       user_index = value.value;
2263       u = pool_elt_at_index (tnm->users, user_index);
2264       if (u->nsessions)
2265         {
2266           head_index = u->sessions_per_user_list_head_index;
2267           head = pool_elt_at_index (tnm->list_pool, head_index);
2268           elt_index = head->next;
2269           elt = pool_elt_at_index (tnm->list_pool, elt_index);
2270           ses_index = elt->value;
2271           while (ses_index != ~0)
2272             {
2273               s = pool_elt_at_index (tnm->sessions, ses_index);
2274               elt = pool_elt_at_index (tnm->list_pool, elt->next);
2275               ses_index = elt->value;
2276
2277               if (nat44_ei_is_session_static (s))
2278                 continue;
2279
2280               if (!is_sm_addr_only (m->flags) &&
2281                   s->in2out.port != m->local_port)
2282                 continue;
2283
2284               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
2285                                              0);
2286               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
2287
2288               if (!is_sm_addr_only (m->flags))
2289                 break;
2290             }
2291         }
2292     }
2293 }
2294
2295 int
2296 nat44_ei_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2297                              u16 l_port, u16 e_port, nat_protocol_t proto,
2298                              u32 vrf_id, u32 sw_if_index, u32 flags,
2299                              ip4_address_t pool_addr, u8 *tag)
2300
2301 {
2302   nat44_ei_main_t *nm = &nat44_ei_main;
2303
2304   if (is_sm_switch_address (flags))
2305     {
2306       if (!nat44_ei_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2307                                         sw_if_index, flags, 0))
2308         {
2309           return VNET_API_ERROR_VALUE_EXIST;
2310         }
2311
2312       nat44_ei_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2313                                    sw_if_index, flags, pool_addr, tag);
2314
2315       ip4_address_t *first_int_addr =
2316         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2317       if (!first_int_addr)
2318         {
2319           // dhcp resolution required
2320           return 0;
2321         }
2322
2323       e_addr.as_u32 = first_int_addr->as_u32;
2324     }
2325
2326   return nat44_ei_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
2327                                                proto, vrf_id, sw_if_index,
2328                                                flags, pool_addr, tag);
2329 }
2330
2331 int
2332 nat44_ei_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
2333                              u16 l_port, u16 e_port, nat_protocol_t proto,
2334                              u32 vrf_id, u32 sw_if_index, u32 flags)
2335 {
2336   nat44_ei_main_t *nm = &nat44_ei_main;
2337
2338   if (is_sm_switch_address (flags))
2339     {
2340
2341       if (nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
2342                                        sw_if_index, flags))
2343         {
2344           return VNET_API_ERROR_NO_SUCH_ENTRY;
2345         }
2346
2347       ip4_address_t *first_int_addr =
2348         ip4_interface_first_address (nm->ip4_main, sw_if_index, 0);
2349       if (!first_int_addr)
2350         {
2351           // dhcp resolution required
2352           return 0;
2353         }
2354
2355       e_addr.as_u32 = first_int_addr->as_u32;
2356     }
2357
2358   return nat44_ei_del_static_mapping_internal (
2359     l_addr, e_addr, l_port, e_port, proto, vrf_id, sw_if_index, flags);
2360 }
2361
2362 static int
2363 nat44_ei_add_static_mapping_internal (ip4_address_t l_addr,
2364                                       ip4_address_t e_addr, u16 l_port,
2365                                       u16 e_port, nat_protocol_t proto,
2366                                       u32 vrf_id, u32 sw_if_index, u32 flags,
2367                                       ip4_address_t pool_addr, u8 *tag)
2368 {
2369   nat44_ei_main_t *nm = &nat44_ei_main;
2370   clib_bihash_kv_8_8_t kv, value;
2371   nat44_ei_lb_addr_port_t *local;
2372   nat44_ei_static_mapping_t *m;
2373   u32 fib_index = ~0;
2374   u32 worker_index;
2375
2376   fail_if_disabled ();
2377
2378   if (is_sm_addr_only (flags))
2379     {
2380       e_port = l_port = proto = 0;
2381     }
2382
2383   if (is_sm_identity_nat (flags))
2384     {
2385       l_port = e_port;
2386       l_addr.as_u32 = e_addr.as_u32;
2387     }
2388
2389   // fib index 0
2390   init_nat_k (&kv, e_addr, e_port, 0, proto);
2391
2392   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2393     {
2394       m = pool_elt_at_index (nm->static_mappings, value.value);
2395       if (!is_sm_identity_nat (m->flags))
2396         {
2397           return VNET_API_ERROR_VALUE_EXIST;
2398         }
2399
2400       // case:
2401       // adding local identity nat record for different vrf table
2402       pool_foreach (local, m->locals)
2403         {
2404           if (local->vrf_id == vrf_id)
2405             {
2406               return VNET_API_ERROR_VALUE_EXIST;
2407             }
2408         }
2409
2410       pool_get (m->locals, local);
2411
2412       local->vrf_id = vrf_id;
2413       local->fib_index = fib_table_find_or_create_and_lock (
2414         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2415
2416       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
2417                    m->proto, 0, m - nm->static_mappings);
2418       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2419
2420       return 0;
2421     }
2422
2423   if (vrf_id != ~0)
2424     {
2425       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
2426                                                      nm->fib_src_low);
2427     }
2428   else
2429     {
2430       // fallback to default vrf
2431       vrf_id = nm->inside_vrf_id;
2432       fib_index = nm->inside_fib_index;
2433       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2434     }
2435
2436   if (!is_sm_identity_nat (flags))
2437     {
2438       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2439       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2440         {
2441           return VNET_API_ERROR_VALUE_EXIST;
2442         }
2443     }
2444
2445   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2446     {
2447       if (nat44_ei_reserve_port (e_addr, e_port, proto))
2448         {
2449           // remove resolve record
2450           if ((is_sm_switch_address (flags)) && !is_sm_identity_nat (flags))
2451             {
2452               nat44_ei_del_resolve_record (l_addr, l_port, e_port, proto,
2453                                            vrf_id, sw_if_index, flags);
2454             }
2455           return VNET_API_ERROR_NO_SUCH_ENTRY;
2456         }
2457     }
2458
2459   pool_get (nm->static_mappings, m);
2460   clib_memset (m, 0, sizeof (*m));
2461
2462   m->flags = flags;
2463   m->local_addr = l_addr;
2464   m->external_addr = e_addr;
2465
2466   m->tag = vec_dup (tag);
2467
2468   if (!is_sm_addr_only (flags))
2469     {
2470       m->local_port = l_port;
2471       m->external_port = e_port;
2472       m->proto = proto;
2473     }
2474
2475   if (is_sm_identity_nat (flags))
2476     {
2477       pool_get (m->locals, local);
2478
2479       local->vrf_id = vrf_id;
2480       local->fib_index = fib_index;
2481     }
2482   else
2483     {
2484       m->vrf_id = vrf_id;
2485       m->fib_index = fib_index;
2486     }
2487
2488   init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2489                m - nm->static_mappings);
2490   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2491
2492   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2493                m - nm->static_mappings);
2494   clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2495
2496   if (nm->num_workers > 1)
2497     {
2498       // store worker index for this record
2499       ip4_header_t ip = {
2500         .src_address = m->local_addr,
2501       };
2502       worker_index = nat44_ei_get_in2out_worker_index (&ip, m->fib_index, 0);
2503       vec_add1 (m->workers, worker_index);
2504     }
2505   else
2506     {
2507       worker_index = nm->num_workers;
2508     }
2509   delete_matching_dynamic_sessions (m, worker_index);
2510
2511   if (is_sm_addr_only (flags))
2512     {
2513       nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
2514     }
2515
2516   return 0;
2517 }
2518
2519 static int
2520 nat44_ei_del_static_mapping_internal (ip4_address_t l_addr,
2521                                       ip4_address_t e_addr, u16 l_port,
2522                                       u16 e_port, nat_protocol_t proto,
2523                                       u32 vrf_id, u32 sw_if_index, u32 flags)
2524 {
2525   nat44_ei_main_per_thread_data_t *tnm;
2526   nat44_ei_main_t *nm = &nat44_ei_main;
2527   clib_bihash_kv_8_8_t kv, value;
2528   nat44_ei_lb_addr_port_t *local;
2529   nat44_ei_static_mapping_t *m;
2530   u32 fib_index = ~0;
2531   nat44_ei_user_key_t u_key;
2532
2533   fail_if_disabled ();
2534
2535   if (is_sm_addr_only (flags))
2536     {
2537       e_port = l_port = proto = 0;
2538     }
2539
2540   if (is_sm_identity_nat (flags))
2541     {
2542       l_port = e_port;
2543       l_addr.as_u32 = e_addr.as_u32;
2544     }
2545
2546   // fib index 0
2547   init_nat_k (&kv, e_addr, e_port, 0, proto);
2548
2549   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2550     {
2551       if (is_sm_switch_address (flags))
2552         {
2553           return 0;
2554         }
2555       return VNET_API_ERROR_NO_SUCH_ENTRY;
2556     }
2557
2558   m = pool_elt_at_index (nm->static_mappings, value.value);
2559
2560   if (is_sm_identity_nat (flags))
2561     {
2562       u8 found = 0;
2563
2564       if (vrf_id == ~0)
2565         {
2566           vrf_id = nm->inside_vrf_id;
2567         }
2568
2569       pool_foreach (local, m->locals)
2570         {
2571           if (local->vrf_id == vrf_id)
2572             {
2573               local = pool_elt_at_index (m->locals, local - m->locals);
2574               fib_index = local->fib_index;
2575               pool_put (m->locals, local);
2576               found = 1;
2577             }
2578         }
2579       if (!found)
2580         {
2581           return VNET_API_ERROR_NO_SUCH_ENTRY;
2582         }
2583     }
2584   else
2585     {
2586       fib_index = m->fib_index;
2587     }
2588
2589   if (!(is_sm_addr_only (flags) || nm->static_mapping_only))
2590     {
2591       if (nat44_ei_free_port (e_addr, e_port, proto))
2592         {
2593           return VNET_API_ERROR_INVALID_VALUE;
2594         }
2595     }
2596
2597   init_nat_k (&kv, l_addr, l_port, fib_index, proto);
2598   clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2599
2600   if (!nm->static_mapping_only || nm->static_mapping_connection_tracking)
2601     {
2602       // delete sessions for static mapping
2603       if (nm->num_workers > 1)
2604         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2605       else
2606         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2607
2608       u_key.addr = m->local_addr;
2609       u_key.fib_index = fib_index;
2610       kv.key = u_key.as_u64;
2611       nat44_ei_static_mapping_del_sessions (
2612         nm, tnm, u_key, is_sm_addr_only (flags), e_addr, e_port);
2613     }
2614
2615   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2616
2617   if (!pool_elts (m->locals))
2618     {
2619       // this is last record remove all required stuff
2620       // fib_index 0
2621       init_nat_k (&kv, e_addr, e_port, 0, proto);
2622       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2623
2624       vec_free (m->tag);
2625       vec_free (m->workers);
2626       pool_put (nm->static_mappings, m);
2627
2628       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
2629         {
2630           nat44_ei_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
2631         }
2632     }
2633
2634   return 0;
2635 }
2636
2637 int
2638 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2639                                u32 match_fib_index,
2640                                nat_protocol_t match_protocol,
2641                                ip4_address_t *mapping_addr, u16 *mapping_port,
2642                                u32 *mapping_fib_index, u8 by_external,
2643                                u8 *is_addr_only, u8 *is_identity_nat)
2644 {
2645   nat44_ei_main_t *nm = &nat44_ei_main;
2646   clib_bihash_kv_8_8_t kv, value;
2647   nat44_ei_static_mapping_t *m;
2648   u16 port;
2649
2650   if (by_external)
2651     {
2652       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2653       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2654                                   &value))
2655         {
2656           /* Try address only mapping */
2657           init_nat_k (&kv, match_addr, 0, 0, 0);
2658           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2659                                       &value))
2660             return 1;
2661         }
2662       m = pool_elt_at_index (nm->static_mappings, value.value);
2663
2664       *mapping_fib_index = m->fib_index;
2665       *mapping_addr = m->local_addr;
2666       port = m->local_port;
2667     }
2668   else
2669     {
2670       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2671                   match_protocol);
2672       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2673         {
2674           /* Try address only mapping */
2675           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2676           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2677                                       &value))
2678             return 1;
2679         }
2680       m = pool_elt_at_index (nm->static_mappings, value.value);
2681
2682       *mapping_fib_index = nm->outside_fib_index;
2683       *mapping_addr = m->external_addr;
2684       port = m->external_port;
2685     }
2686
2687   /* Address only mapping doesn't change port */
2688   if (is_sm_addr_only (m->flags))
2689     *mapping_port = match_port;
2690   else
2691     *mapping_port = port;
2692
2693   if (PREDICT_FALSE (is_addr_only != 0))
2694     *is_addr_only = is_sm_addr_only (m->flags);
2695
2696   if (PREDICT_FALSE (is_identity_nat != 0))
2697     *is_identity_nat = is_sm_identity_nat (m->flags);
2698
2699   return 0;
2700 }
2701
2702 static void
2703 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2704 {
2705   pool_free (tnm->list_pool);
2706   pool_free (tnm->lru_pool);
2707   pool_free (tnm->sessions);
2708   pool_free (tnm->users);
2709
2710   clib_bihash_free_8_8 (&tnm->user_hash);
2711 }
2712
2713 u8 *
2714 format_nat44_ei_key (u8 *s, va_list *args)
2715 {
2716   u64 key = va_arg (*args, u64);
2717
2718   ip4_address_t addr;
2719   u16 port;
2720   nat_protocol_t protocol;
2721   u32 fib_index;
2722
2723   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2724
2725   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2726               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2727               fib_index);
2728   return s;
2729 }
2730
2731 u8 *
2732 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2733 {
2734   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2735   nat44_ei_user_key_t k;
2736
2737   k.as_u64 = v->key;
2738
2739   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2740               k.fib_index, v->value);
2741
2742   return s;
2743 }
2744
2745 u8 *
2746 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2747 {
2748   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2749
2750   s = format (s, "%U thread-index %llu session-index %llu",
2751               format_nat44_ei_key, v->key, nat_value_get_thread_index (v),
2752               nat_value_get_session_index (v));
2753
2754   return s;
2755 }
2756
2757 u8 *
2758 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2759 {
2760   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2761
2762   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2763               v->value);
2764
2765   return s;
2766 }
2767
2768 static void
2769 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2770                          u32 translations, u32 translation_buckets,
2771                          u32 user_buckets)
2772 {
2773   dlist_elt_t *head;
2774
2775   pool_alloc (tnm->list_pool, translations);
2776   pool_alloc (tnm->lru_pool, translations);
2777   pool_alloc (tnm->sessions, translations);
2778
2779   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2780
2781   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2782                                      format_nat44_ei_user_kvp);
2783
2784   pool_get (tnm->lru_pool, head);
2785   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2786   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2787
2788   pool_get (tnm->lru_pool, head);
2789   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2790   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2791
2792   pool_get (tnm->lru_pool, head);
2793   tnm->udp_lru_head_index = head - tnm->lru_pool;
2794   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2795
2796   pool_get (tnm->lru_pool, head);
2797   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2798   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2799
2800   pool_get (tnm->lru_pool, head);
2801   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2802   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2803 }
2804
2805 static void
2806 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2807 {
2808   nat44_ei_main_t *nm = &nat44_ei_main;
2809   nat44_ei_main_per_thread_data_t *tnm;
2810
2811   u32 static_mapping_buckets = 1024;
2812   u32 static_mapping_memory_size = 64 << 20;
2813
2814   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2815                         "static_mapping_by_local", static_mapping_buckets,
2816                         static_mapping_memory_size);
2817   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2818                         "static_mapping_by_external", static_mapping_buckets,
2819                         static_mapping_memory_size);
2820   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2821                                      format_nat44_ei_static_mapping_kvp);
2822   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2823                                      format_nat44_ei_static_mapping_kvp);
2824
2825   if (nm->pat)
2826     {
2827       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2828       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2829       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2830                                          format_nat44_ei_session_kvp);
2831       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2832                                          format_nat44_ei_session_kvp);
2833       vec_foreach (tnm, nm->per_thread_data)
2834         {
2835           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2836                                    user_buckets);
2837         }
2838     }
2839 }
2840
2841 void
2842 nat44_ei_sessions_clear ()
2843 {
2844   nat44_ei_main_t *nm = &nat44_ei_main;
2845   nat44_ei_main_per_thread_data_t *tnm;
2846
2847   if (nm->pat)
2848     {
2849       clib_bihash_free_8_8 (&nm->in2out);
2850       clib_bihash_free_8_8 (&nm->out2in);
2851       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2852       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2853       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2854                                          format_nat44_ei_session_kvp);
2855       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2856                                          format_nat44_ei_session_kvp);
2857       vec_foreach (tnm, nm->per_thread_data)
2858         {
2859           nat44_ei_worker_db_free (tnm);
2860           nat44_ei_worker_db_init (tnm, nm->translations,
2861                                    nm->translation_buckets, nm->user_buckets);
2862         }
2863     }
2864
2865   vlib_zero_simple_counter (&nm->total_users, 0);
2866   vlib_zero_simple_counter (&nm->total_sessions, 0);
2867   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2868 }
2869
2870 static void
2871 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2872                              u32 new_fib_index, u32 old_fib_index)
2873 {
2874   nat44_ei_main_t *nm = &nat44_ei_main;
2875   nat44_ei_outside_fib_t *outside_fib;
2876   nat44_ei_interface_t *i;
2877   u8 is_add = 1;
2878   u8 match = 0;
2879
2880   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2881       (!vec_len (nm->outside_fibs)))
2882     {
2883       return;
2884     }
2885
2886   pool_foreach (i, nm->interfaces)
2887     {
2888       if (i->sw_if_index == sw_if_index)
2889         {
2890           if (!(nat44_ei_interface_is_outside (i)))
2891             return;
2892           match = 1;
2893         }
2894     }
2895
2896   pool_foreach (i, nm->output_feature_interfaces)
2897     {
2898       if (i->sw_if_index == sw_if_index)
2899         {
2900           if (!(nat44_ei_interface_is_outside (i)))
2901             return;
2902           match = 1;
2903         }
2904     }
2905
2906   if (!match)
2907     return;
2908
2909   vec_foreach (outside_fib, nm->outside_fibs)
2910     {
2911       if (outside_fib->fib_index == old_fib_index)
2912         {
2913           outside_fib->refcount--;
2914           if (!outside_fib->refcount)
2915             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2916           break;
2917         }
2918     }
2919
2920   vec_foreach (outside_fib, nm->outside_fibs)
2921     {
2922       if (outside_fib->fib_index == new_fib_index)
2923         {
2924           outside_fib->refcount++;
2925           is_add = 0;
2926           break;
2927         }
2928     }
2929
2930   if (is_add)
2931     {
2932       vec_add2 (nm->outside_fibs, outside_fib, 1);
2933       outside_fib->refcount = 1;
2934       outside_fib->fib_index = new_fib_index;
2935     }
2936 }
2937
2938 int
2939 nat44_ei_add_address (ip4_address_t *addr, u32 vrf_id)
2940 {
2941   nat44_ei_main_t *nm = &nat44_ei_main;
2942   vlib_thread_main_t *tm = vlib_get_thread_main ();
2943   nat44_ei_address_t *ap;
2944
2945   fail_if_disabled ();
2946
2947   /* Check if address already exists */
2948   vec_foreach (ap, nm->addresses)
2949     {
2950       if (ap->addr.as_u32 == addr->as_u32)
2951         {
2952           nat44_ei_log_err ("address exist");
2953           return VNET_API_ERROR_VALUE_EXIST;
2954         }
2955     }
2956
2957   vec_add2 (nm->addresses, ap, 1);
2958
2959   ap->fib_index = ~0;
2960   ap->addr = *addr;
2961
2962   if (vrf_id != ~0)
2963     {
2964       ap->fib_index = fib_table_find_or_create_and_lock (
2965         FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2966     }
2967
2968 #define _(N, i, n, s)                                                         \
2969   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
2970                sizeof (ap->busy_##n##_port_refcounts));                       \
2971   ap->busy_##n##_ports = 0;                                                   \
2972   ap->busy_##n##_ports_per_thread = 0;                                        \
2973   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
2974                            tm->n_vlib_mains - 1, 0);
2975   foreach_nat_protocol
2976 #undef _
2977
2978     nat44_ei_add_del_addr_to_fib_foreach_out_if (addr, 1);
2979
2980   return 0;
2981 }
2982
2983 int
2984 nat44_ei_del_address (ip4_address_t addr, u8 delete_sm)
2985 {
2986   nat44_ei_main_t *nm = &nat44_ei_main;
2987   nat44_ei_address_t *a = 0;
2988   nat44_ei_session_t *ses;
2989   u32 *ses_to_be_removed = 0, *ses_index;
2990   nat44_ei_main_per_thread_data_t *tnm;
2991   nat44_ei_static_mapping_t *m;
2992   int j;
2993
2994   fail_if_disabled ();
2995
2996   /* Find SNAT address */
2997   for (j = 0; j < vec_len (nm->addresses); j++)
2998     {
2999       if (nm->addresses[j].addr.as_u32 == addr.as_u32)
3000         {
3001           a = nm->addresses + j;
3002           break;
3003         }
3004     }
3005   if (!a)
3006     {
3007       nat44_ei_log_err ("no such address");
3008       return VNET_API_ERROR_NO_SUCH_ENTRY;
3009     }
3010
3011   if (delete_sm)
3012     {
3013       pool_foreach (m, nm->static_mappings)
3014         {
3015           if (m->external_addr.as_u32 == addr.as_u32)
3016             nat44_ei_del_static_mapping_internal (
3017               m->local_addr, m->external_addr, m->local_port, m->external_port,
3018               m->proto, m->vrf_id, ~0, m->flags);
3019         }
3020     }
3021   else
3022     {
3023       /* Check if address is used in some static mapping */
3024       if (nat44_ei_is_address_used_in_static_mapping (addr))
3025         {
3026           nat44_ei_log_err ("address used in static mapping");
3027           return VNET_API_ERROR_UNSPECIFIED;
3028         }
3029     }
3030
3031   /* Delete sessions using address */
3032   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
3033     {
3034       vec_foreach (tnm, nm->per_thread_data)
3035         {
3036           pool_foreach (ses, tnm->sessions)
3037             {
3038               if (ses->out2in.addr.as_u32 == addr.as_u32)
3039                 {
3040                   nat44_ei_free_session_data (nm, ses,
3041                                               tnm - nm->per_thread_data, 0);
3042                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
3043                 }
3044             }
3045           vec_foreach (ses_index, ses_to_be_removed)
3046             {
3047               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
3048               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
3049             }
3050           vec_free (ses_to_be_removed);
3051         }
3052     }
3053
3054   nat44_ei_add_del_addr_to_fib_foreach_out_if (&addr, 0);
3055
3056   if (a->fib_index != ~0)
3057     {
3058       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
3059     }
3060
3061 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
3062   foreach_nat_protocol
3063 #undef _
3064
3065     vec_del1 (nm->addresses, j);
3066   return 0;
3067 }
3068
3069 int
3070 nat44_ei_add_interface_address (u32 sw_if_index)
3071 {
3072   nat44_ei_main_t *nm = &nat44_ei_main;
3073   ip4_main_t *ip4_main = nm->ip4_main;
3074   ip4_address_t *first_int_addr;
3075   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3076   int i;
3077
3078   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3079     {
3080       if (auto_add_sw_if_indices[i] == sw_if_index)
3081         {
3082           return VNET_API_ERROR_VALUE_EXIST;
3083         }
3084     }
3085
3086   /* add to the auto-address list */
3087   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
3088
3089   // if the address is already bound - or static - add it now
3090   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3091   if (first_int_addr)
3092     {
3093       (void) nat44_ei_add_address (first_int_addr, ~0);
3094     }
3095
3096   return 0;
3097 }
3098
3099 int
3100 nat44_ei_del_interface_address (u32 sw_if_index)
3101 {
3102   nat44_ei_main_t *nm = &nat44_ei_main;
3103   ip4_main_t *ip4_main = nm->ip4_main;
3104   ip4_address_t *first_int_addr;
3105   nat44_ei_static_map_resolve_t *rp;
3106   u32 *indices_to_delete = 0;
3107   int i, j;
3108   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
3109
3110   fail_if_disabled ();
3111
3112   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3113
3114   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3115     {
3116       if (auto_add_sw_if_indices[i] == sw_if_index)
3117         {
3118           first_int_addr =
3119             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3120           if (first_int_addr)
3121             {
3122               (void) nat44_ei_del_address (first_int_addr[0], 1);
3123             }
3124           else
3125             {
3126               for (j = 0; j < vec_len (nm->to_resolve); j++)
3127                 {
3128                   rp = nm->to_resolve + j;
3129                   if (rp->sw_if_index == sw_if_index)
3130                     {
3131                       vec_add1 (indices_to_delete, j);
3132                     }
3133                 }
3134               if (vec_len (indices_to_delete))
3135                 {
3136                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3137                     {
3138                       vec_del1 (nm->to_resolve, j);
3139                     }
3140                   vec_free (indices_to_delete);
3141                 }
3142             }
3143
3144           vec_del1 (nm->auto_add_sw_if_indices, i);
3145           return 0;
3146         }
3147     }
3148   return VNET_API_ERROR_NO_SUCH_ENTRY;
3149 }
3150
3151 static_always_inline int
3152 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3153 {
3154   u32 *i;
3155   vec_foreach (i, sw_if_indices)
3156     {
3157       if (*i == sw_if_index)
3158         {
3159           return 1;
3160         }
3161     }
3162   return 0;
3163 }
3164
3165 static void
3166 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3167                                            u32 sw_if_index,
3168                                            ip4_address_t *address,
3169                                            u32 address_length,
3170                                            u32 if_address_index, u32 is_delete)
3171 {
3172   nat44_ei_main_t *nm = &nat44_ei_main;
3173   nat44_ei_static_map_resolve_t *rp;
3174   nat44_ei_address_t *addresses = nm->addresses;
3175   int rv, i;
3176
3177   if (!nm->enabled)
3178     {
3179       return;
3180     }
3181
3182   if (!is_sw_if_index_reg_for_auto_resolve (nm->auto_add_sw_if_indices,
3183                                             sw_if_index))
3184     {
3185       return;
3186     }
3187
3188   if (!is_delete)
3189     {
3190       /* Don't trip over lease renewal, static config */
3191       for (i = 0; i < vec_len (addresses); i++)
3192         {
3193           if (addresses[i].addr.as_u32 == address->as_u32)
3194             {
3195               return;
3196             }
3197         }
3198
3199       (void) nat44_ei_add_address (address, ~0);
3200
3201       /* Scan static map resolution vector */
3202       for (i = 0; i < vec_len (nm->to_resolve); i++)
3203         {
3204           rp = nm->to_resolve + i;
3205           if (is_sm_addr_only (rp->flags))
3206             {
3207               continue;
3208             }
3209           /* On this interface? */
3210           if (rp->sw_if_index == sw_if_index)
3211             {
3212               rv = nat44_ei_add_static_mapping_internal (
3213                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3214                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3215               if (rv)
3216                 {
3217                   nat_elog_notice_X1 (
3218                     nm, "add_static_mapping_internal returned %d", "i4", rv);
3219                 }
3220             }
3221         }
3222     }
3223   else
3224     {
3225       // remove all static mapping records
3226       (void) nat44_ei_del_address (address[0], 1);
3227     }
3228 }
3229
3230 int
3231 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
3232 {
3233   fail_if_enabled ();
3234   nat44_ei_main_t *nm = &nat44_ei_main;
3235   nm->frame_queue_nelts = frame_queue_nelts;
3236   return 0;
3237 }
3238
3239 static void
3240 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
3241                                       u32 sw_if_index, ip4_address_t *address,
3242                                       u32 address_length, u32 if_address_index,
3243                                       u32 is_delete)
3244 {
3245   nat44_ei_main_t *nm = &nat44_ei_main;
3246   nat44_ei_static_map_resolve_t *rp;
3247   nat44_ei_static_mapping_t *m;
3248   clib_bihash_kv_8_8_t kv, value;
3249   int i, rv = 0, match = 0;
3250
3251   if (!nm->enabled)
3252     {
3253       return;
3254     }
3255
3256   for (i = 0; i < vec_len (nm->to_resolve); i++)
3257     {
3258       rp = nm->to_resolve + i;
3259
3260       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3261         {
3262           match = 1;
3263           break;
3264         }
3265     }
3266
3267   if (!match)
3268     {
3269       return;
3270     }
3271
3272   init_nat_k (&kv, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port,
3273               nm->outside_fib_index,
3274               is_sm_addr_only (rp->flags) ? 0 : rp->proto);
3275   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
3276     m = 0;
3277   else
3278     m = pool_elt_at_index (nm->static_mappings, value.value);
3279
3280   if (is_delete)
3281     {
3282       if (!m)
3283         return;
3284       rv = nat44_ei_del_static_mapping_internal (
3285         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3286         ~0, rp->flags);
3287       if (rv)
3288         {
3289           nat_elog_notice_X1 (nm, "nat44_ei_del_static_mapping returned %d",
3290                               "i4", rv);
3291         }
3292     }
3293   else
3294     {
3295       if (m)
3296         return;
3297       rv = nat44_ei_add_static_mapping_internal (
3298         rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto, rp->vrf_id,
3299         ~0, rp->flags, rp->pool_addr, rp->tag);
3300
3301       if (rv)
3302         {
3303           nat_elog_notice_X1 (nm, "nat44_ei_add_static_mapping returned %d",
3304                               "i4", rv);
3305         }
3306     }
3307 }
3308
3309 static_always_inline uword
3310 nat44_ei_classify_inline_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
3311                              vlib_frame_t *frame)
3312 {
3313   u32 n_left_from, *from, *to_next;
3314   nat44_ei_classify_next_t next_index;
3315   nat44_ei_main_t *nm = &nat44_ei_main;
3316   nat44_ei_static_mapping_t *m;
3317   u32 next_in2out = 0, next_out2in = 0;
3318
3319   from = vlib_frame_vector_args (frame);
3320   n_left_from = frame->n_vectors;
3321   next_index = node->cached_next_index;
3322
3323   while (n_left_from > 0)
3324     {
3325       u32 n_left_to_next;
3326
3327       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3328
3329       while (n_left_from > 0 && n_left_to_next > 0)
3330         {
3331           u32 bi0;
3332           vlib_buffer_t *b0;
3333           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3334           ip4_header_t *ip0;
3335           nat44_ei_address_t *ap;
3336           clib_bihash_kv_8_8_t kv0, value0;
3337
3338           /* speculatively enqueue b0 to the current next frame */
3339           bi0 = from[0];
3340           to_next[0] = bi0;
3341           from += 1;
3342           to_next += 1;
3343           n_left_from -= 1;
3344           n_left_to_next -= 1;
3345
3346           b0 = vlib_get_buffer (vm, bi0);
3347           ip0 = vlib_buffer_get_current (b0);
3348
3349           vec_foreach (ap, nm->addresses)
3350             {
3351               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
3352                 {
3353                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3354                   goto enqueue0;
3355                 }
3356             }
3357
3358           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
3359             {
3360               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
3361               /* try to classify the fragment based on IP header alone */
3362               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3363                                            &kv0, &value0))
3364                 {
3365                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3366                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3367                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3368                   goto enqueue0;
3369                 }
3370               init_nat_k (&kv0, ip0->dst_address,
3371                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
3372                           ip_proto_to_nat_proto (ip0->protocol));
3373               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
3374                                            &kv0, &value0))
3375                 {
3376                   m = pool_elt_at_index (nm->static_mappings, value0.value);
3377                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
3378                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3379                 }
3380             }
3381
3382         enqueue0:
3383           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
3384                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
3385             {
3386               nat44_ei_classify_trace_t *t =
3387                 vlib_add_trace (vm, node, b0, sizeof (*t));
3388               t->cached = 0;
3389               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
3390             }
3391
3392           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
3393           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
3394
3395           /* verify speculative enqueue, maybe switch current next frame */
3396           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3397                                            n_left_to_next, bi0, next0);
3398         }
3399
3400       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3401     }
3402
3403   vlib_node_increment_counter (
3404     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
3405   vlib_node_increment_counter (
3406     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3407   return frame->n_vectors;
3408 }
3409
3410 VLIB_NODE_FN (nat44_ei_classify_node)
3411 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3412 {
3413   return nat44_ei_classify_inline_fn (vm, node, frame);
3414 }
3415
3416 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3417   .name = "nat44-ei-classify",
3418   .vector_size = sizeof (u32),
3419   .format_trace = format_nat44_ei_classify_trace,
3420   .type = VLIB_NODE_TYPE_INTERNAL,
3421   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3422   .error_strings = nat44_ei_classify_error_strings,
3423   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3424   .next_nodes = {
3425     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3426     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3427     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3428   },
3429 };
3430
3431 VLIB_NODE_FN (nat44_ei_handoff_classify_node)
3432 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
3433 {
3434   return nat44_ei_classify_inline_fn (vm, node, frame);
3435 }
3436
3437 VLIB_REGISTER_NODE (nat44_ei_handoff_classify_node) = {
3438   .name = "nat44-ei-handoff-classify",
3439   .vector_size = sizeof (u32),
3440   .format_trace = format_nat44_ei_classify_trace,
3441   .type = VLIB_NODE_TYPE_INTERNAL,
3442   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3443   .error_strings = nat44_ei_classify_error_strings,
3444   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3445   .next_nodes = {
3446     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out-worker-handoff",
3447     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in-worker-handoff",
3448     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3449   },
3450 };
3451
3452 /*
3453  * fd.io coding-style-patch-verification: ON
3454  *
3455  * Local Variables:
3456  * eval: (c-set-style "gnu")
3457  * End:
3458  */