src/plugins/nat/nat44-ei/nat44_ei.c

   1 /*
   2  * nat44_ei.c - nat44 endpoint dependent plugin
   3  *
   4  * Copyright (c) 2020 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14  * License for the specific language governing permissions and limitations
  15  * under the License.
  16  */
  17
  18 #include <vnet/plugin/plugin.h>
  19 #include <vpp/app/version.h>
  20
  21 #include <vnet/vnet.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/ip/ip4.h>
  24 #include <vnet/ip/ip_table.h>
  25 #include <vnet/ip/reass/ip4_sv_reass.h>
  26 #include <vnet/fib/fib_table.h>
  27 #include <vnet/fib/ip4_fib.h>
  28 #include <vnet/plugin/plugin.h>
  29
  30 // nat lib
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/nat_inlines.h>
  34 #include <nat/lib/ipfix_logging.h>
  35
  36 #include <nat/nat44-ei/nat44_ei_dpo.h>
  37 #include <nat/nat44-ei/nat44_ei_inlines.h>
  38 #include <nat/nat44-ei/nat44_ei.h>
  39
  40 nat44_ei_main_t nat44_ei_main;
  41
  42 extern vlib_node_registration_t nat44_ei_hairpinning_node;
  43 extern vlib_node_registration_t nat44_ei_hairpin_dst_node;
  44 extern vlib_node_registration_t
  45   nat44_ei_in2out_hairpinning_finish_ip4_lookup_node;
  46 extern vlib_node_registration_t
  47   nat44_ei_in2out_hairpinning_finish_interface_output_node;
  48
  49 #define skip_if_disabled()                                                    \
  50   do                                                                          \
  51     {                                                                         \
  52       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  53       if (PREDICT_FALSE (!nm->enabled))                                       \
  54         return;                                                               \
  55     }                                                                         \
  56   while (0)
  57
  58 #define fail_if_enabled()                                                     \
  59   do                                                                          \
  60     {                                                                         \
  61       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  62       if (PREDICT_FALSE (nm->enabled))                                        \
  63         {                                                                     \
  64           nat44_ei_log_err ("plugin enabled");                                \
  65           return 1;                                                           \
  66         }                                                                     \
  67     }                                                                         \
  68   while (0)
  69
  70 #define fail_if_disabled()                                                    \
  71   do                                                                          \
  72     {                                                                         \
  73       nat44_ei_main_t *nm = &nat44_ei_main;                                   \
  74       if (PREDICT_FALSE (!nm->enabled))                                       \
  75         {                                                                     \
  76           nat44_ei_log_err ("plugin disabled");                               \
  77           return 1;                                                           \
  78         }                                                                     \
  79     }                                                                         \
  80   while (0)
  81
  82 /* Hook up input features */
  83 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ei-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  87                                "ip4-sv-reassembly-feature"),
  88 };
  89 VNET_FEATURE_INIT (ip4_nat44_ei_in2out, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-ei-in2out",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  93                                "ip4-sv-reassembly-feature"),
  94 };
  95 VNET_FEATURE_INIT (ip4_nat44_ei_out2in, static) = {
  96   .arc_name = "ip4-unicast",
  97   .node_name = "nat44-ei-out2in",
  98   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  99                                "ip4-sv-reassembly-feature",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output, static) = {
 103   .arc_name = "ip4-output",
 104   .node_name = "nat44-ei-in2out-output",
 105   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 106                                "ip4-sv-reassembly-output-feature"),
 107 };
 108 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_fast, static) = {
 109   .arc_name = "ip4-unicast",
 110   .node_name = "nat44-ei-in2out-fast",
 111   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 112                                "ip4-sv-reassembly-feature"),
 113 };
 114 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_fast, static) = {
 115   .arc_name = "ip4-unicast",
 116   .node_name = "nat44-ei-out2in-fast",
 117   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 118                                "ip4-sv-reassembly-feature",
 119                                "ip4-dhcp-client-detect"),
 120 };
 121 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_dst, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-ei-hairpin-dst",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 125                                "ip4-sv-reassembly-feature"),
 126 };
 127 VNET_FEATURE_INIT (ip4_nat44_ei_hairpin_src, static) = {
 128   .arc_name = "ip4-output",
 129   .node_name = "nat44-ei-hairpin-src",
 130   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 131                                "ip4-sv-reassembly-output-feature"),
 132 };
 133 VNET_FEATURE_INIT (ip4_nat44_ei_hairpinning, static) = {
 134   .arc_name = "ip4-local",
 135   .node_name = "nat44-ei-hairpinning",
 136   .runs_before = VNET_FEATURES ("ip4-local-end-of-arc"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_worker_handoff, static) = {
 139   .arc_name = "ip4-unicast",
 140   .node_name = "nat44-ei-in2out-worker-handoff",
 141   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 142 };
 143 VNET_FEATURE_INIT (ip4_nat44_ei_out2in_worker_handoff, static) = {
 144   .arc_name = "ip4-unicast",
 145   .node_name = "nat44-ei-out2in-worker-handoff",
 146   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 147                                "ip4-dhcp-client-detect"),
 148 };
 149 VNET_FEATURE_INIT (ip4_nat44_ei_in2out_output_worker_handoff, static) = {
 150   .arc_name = "ip4-output",
 151   .node_name = "nat44-ei-in2out-output-worker-handoff",
 152   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa",
 153                                "ip4-sv-reassembly-output-feature"),
 154 };
 155
 156 VLIB_PLUGIN_REGISTER () = {
 157   .version = VPP_BUILD_VER,
 158   .description = "IPv4 Endpoint-Independent NAT (NAT44 EI)",
 159 };
 160
 161 #define foreach_nat44_ei_classify_error                                       \
 162   _ (NEXT_IN2OUT, "next in2out")                                              \
 163   _ (NEXT_OUT2IN, "next out2in")                                              \
 164   _ (FRAG_CACHED, "fragment cached")
 165
 166 typedef enum
 167 {
 168 #define _(sym, str) NAT44_EI_CLASSIFY_ERROR_##sym,
 169   foreach_nat44_ei_classify_error
 170 #undef _
 171     NAT44_EI_CLASSIFY_N_ERROR,
 172 } nat44_ei_classify_error_t;
 173
 174 static char *nat44_ei_classify_error_strings[] = {
 175 #define _(sym, string) string,
 176   foreach_nat44_ei_classify_error
 177 #undef _
 178 };
 179
 180 typedef enum
 181 {
 182   NAT44_EI_CLASSIFY_NEXT_IN2OUT,
 183   NAT44_EI_CLASSIFY_NEXT_OUT2IN,
 184   NAT44_EI_CLASSIFY_NEXT_DROP,
 185   NAT44_EI_CLASSIFY_N_NEXT,
 186 } nat44_ei_classify_next_t;
 187
 188 typedef struct
 189 {
 190   u8 next_in2out;
 191   u8 cached;
 192 } nat44_ei_classify_trace_t;
 193
 194 void nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len,
 195                                    u32 sw_if_index, int is_add);
 196
 197 static u8 *
 198 format_nat44_ei_classify_trace (u8 *s, va_list *args)
 199 {
 200   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 201   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 202   nat44_ei_classify_trace_t *t = va_arg (*args, nat44_ei_classify_trace_t *);
 203   char *next;
 204
 205   if (t->cached)
 206     s = format (s, "nat44-ei-classify: fragment cached");
 207   else
 208     {
 209       next = t->next_in2out ? "nat44-ei-in2out" : "nat44-ei-out2in";
 210       s = format (s, "nat44-ei-classify: next %s", next);
 211     }
 212
 213   return s;
 214 }
 215
 216 static void nat44_ei_db_free ();
 217
 218 static void nat44_ei_db_init (u32 translations, u32 translation_buckets,
 219                               u32 user_buckets);
 220
 221 static void nat44_ei_ip4_add_del_interface_address_cb (
 222   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 223   u32 address_length, u32 if_address_index, u32 is_delete);
 224
 225 static void nat44_ei_ip4_add_del_addr_only_sm_cb (
 226   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
 227   u32 address_length, u32 if_address_index, u32 is_delete);
 228
 229 static void nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque,
 230                                          u32 sw_if_index, u32 new_fib_index,
 231                                          u32 old_fib_index);
 232
 233 void
 234 nat44_ei_set_node_indexes (nat44_ei_main_t *nm, vlib_main_t *vm)
 235 {
 236   vlib_node_t *node;
 237   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-out2in");
 238   nm->out2in_node_index = node->index;
 239   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out");
 240   nm->in2out_node_index = node->index;
 241   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ei-in2out-output");
 242   nm->in2out_output_node_index = node->index;
 243 }
 244
 245 int
 246 nat44_ei_set_workers (uword *bitmap)
 247 {
 248   nat44_ei_main_t *nm = &nat44_ei_main;
 249   int i, j = 0;
 250
 251   if (nm->num_workers < 2)
 252     return VNET_API_ERROR_FEATURE_DISABLED;
 253
 254   if (clib_bitmap_last_set (bitmap) >= nm->num_workers)
 255     return VNET_API_ERROR_INVALID_WORKER;
 256
 257   vec_free (nm->workers);
 258   clib_bitmap_foreach (i, bitmap)
 259     {
 260       vec_add1 (nm->workers, i);
 261       nm->per_thread_data[nm->first_worker_index + i].snat_thread_index = j;
 262       nm->per_thread_data[nm->first_worker_index + i].thread_index = i;
 263       j++;
 264     }
 265
 266   nm->port_per_thread = (0xffff - 1024) / _vec_len (nm->workers);
 267
 268   return 0;
 269 }
 270
 271 #define nat_validate_simple_counter(c, i)                                     \
 272   do                                                                          \
 273     {                                                                         \
 274       vlib_validate_simple_counter (&c, i);                                   \
 275       vlib_zero_simple_counter (&c, i);                                       \
 276     }                                                                         \
 277   while (0);
 278
 279 #define nat_init_simple_counter(c, n, sn)                                     \
 280   do                                                                          \
 281     {                                                                         \
 282       c.name = n;                                                             \
 283       c.stat_segment_name = sn;                                               \
 284       nat_validate_simple_counter (c, 0);                                     \
 285     }                                                                         \
 286   while (0);
 287
 288 static_always_inline void
 289 nat_validate_interface_counters (nat44_ei_main_t *nm, u32 sw_if_index)
 290 {
 291 #define _(x)                                                                  \
 292   nat_validate_simple_counter (nm->counters.fastpath.in2out.x, sw_if_index);  \
 293   nat_validate_simple_counter (nm->counters.fastpath.out2in.x, sw_if_index);  \
 294   nat_validate_simple_counter (nm->counters.slowpath.in2out.x, sw_if_index);  \
 295   nat_validate_simple_counter (nm->counters.slowpath.out2in.x, sw_if_index);
 296   foreach_nat_counter;
 297 #undef _
 298   nat_validate_simple_counter (nm->counters.hairpinning, sw_if_index);
 299 }
 300
 301 clib_error_t *
 302 nat44_ei_init (vlib_main_t *vm)
 303 {
 304   nat44_ei_main_t *nm = &nat44_ei_main;
 305   vlib_thread_main_t *tm = vlib_get_thread_main ();
 306   vlib_thread_registration_t *tr;
 307   ip4_add_del_interface_address_callback_t cbi = { 0 };
 308   ip4_table_bind_callback_t cbt = { 0 };
 309   u32 i, num_threads = 0;
 310   uword *p, *bitmap = 0;
 311
 312   clib_memset (nm, 0, sizeof (*nm));
 313
 314   // required
 315   nm->vnet_main = vnet_get_main ();
 316   // convenience
 317   nm->ip4_main = &ip4_main;
 318   nm->api_main = vlibapi_get_main ();
 319   nm->ip4_lookup_main = &ip4_main.lookup_main;
 320
 321   // handoff stuff
 322   nm->fq_out2in_index = ~0;
 323   nm->fq_in2out_index = ~0;
 324   nm->fq_in2out_output_index = ~0;
 325   nm->worker_in2out_cb = nat44_ei_get_in2out_worker_index;
 326   nm->worker_out2in_cb = nat44_ei_get_out2in_worker_index;
 327
 328   nm->log_level = NAT_LOG_ERROR;
 329
 330   nat44_ei_set_node_indexes (nm, vm);
 331   nm->log_class = vlib_log_register_class ("nat44-ei", 0);
 332
 333   nat_init_simple_counter (nm->total_users, "total-users",
 334                            "/nat44-ei/total-users");
 335   nat_init_simple_counter (nm->total_sessions, "total-sessions",
 336                            "/nat44-ei/total-sessions");
 337   nat_init_simple_counter (nm->user_limit_reached, "user-limit-reached",
 338                            "/nat44-ei/user-limit-reached");
 339
 340 #define _(x)                                                                  \
 341   nat_init_simple_counter (nm->counters.fastpath.in2out.x, #x,                \
 342                            "/nat44-ei/in2out/fastpath/" #x);                  \
 343   nat_init_simple_counter (nm->counters.fastpath.out2in.x, #x,                \
 344                            "/nat44-ei/out2in/fastpath/" #x);                  \
 345   nat_init_simple_counter (nm->counters.slowpath.in2out.x, #x,                \
 346                            "/nat44-ei/in2out/slowpath/" #x);                  \
 347   nat_init_simple_counter (nm->counters.slowpath.out2in.x, #x,                \
 348                            "/nat44-ei/out2in/slowpath/" #x);
 349   foreach_nat_counter;
 350 #undef _
 351   nat_init_simple_counter (nm->counters.hairpinning, "hairpinning",
 352                            "/nat44-ei/hairpinning");
 353
 354   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 355   if (p)
 356     {
 357       tr = (vlib_thread_registration_t *) p[0];
 358       if (tr)
 359         {
 360           nm->num_workers = tr->count;
 361           nm->first_worker_index = tr->first_index;
 362         }
 363     }
 364   num_threads = tm->n_vlib_mains - 1;
 365   nm->port_per_thread = 0xffff - 1024;
 366   vec_validate (nm->per_thread_data, num_threads);
 367
 368   /* Use all available workers by default */
 369   if (nm->num_workers > 1)
 370     {
 371
 372       for (i = 0; i < nm->num_workers; i++)
 373         bitmap = clib_bitmap_set (bitmap, i, 1);
 374       nat44_ei_set_workers (bitmap);
 375       clib_bitmap_free (bitmap);
 376     }
 377   else
 378     nm->per_thread_data[0].snat_thread_index = 0;
 379
 380   /* callbacks to call when interface address changes. */
 381   cbi.function = nat44_ei_ip4_add_del_interface_address_cb;
 382   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 383   cbi.function = nat44_ei_ip4_add_del_addr_only_sm_cb;
 384   vec_add1 (nm->ip4_main->add_del_interface_address_callbacks, cbi);
 385
 386   /* callbacks to call when interface to table biding changes */
 387   cbt.function = nat44_ei_update_outside_fib;
 388   vec_add1 (nm->ip4_main->table_bind_callbacks, cbt);
 389
 390   nm->fib_src_low = fib_source_allocate (
 391     "nat44-ei-low", FIB_SOURCE_PRIORITY_LOW, FIB_SOURCE_BH_SIMPLE);
 392   nm->fib_src_hi = fib_source_allocate ("nat44-ei-hi", FIB_SOURCE_PRIORITY_HI,
 393                                         FIB_SOURCE_BH_SIMPLE);
 394
 395   // used only by out2in-dpo feature
 396   nat_dpo_module_init ();
 397   nat_ha_init (vm, nm->num_workers, num_threads);
 398
 399   nm->hairpinning_fq_index =
 400     vlib_frame_queue_main_init (nat44_ei_hairpinning_node.index, 0);
 401   nm->hairpin_dst_fq_index =
 402     vlib_frame_queue_main_init (nat44_ei_hairpin_dst_node.index, 0);
 403   nm->in2out_hairpinning_finish_ip4_lookup_node_fq_index =
 404     vlib_frame_queue_main_init (
 405       nat44_ei_in2out_hairpinning_finish_ip4_lookup_node.index, 0);
 406   nm->in2out_hairpinning_finish_interface_output_node_fq_index =
 407     vlib_frame_queue_main_init (
 408       nat44_ei_in2out_hairpinning_finish_interface_output_node.index, 0);
 409   return nat44_ei_api_hookup (vm);
 410 }
 411
 412 VLIB_INIT_FUNCTION (nat44_ei_init);
 413
 414 int
 415 nat44_ei_plugin_enable (nat44_ei_config_t c)
 416 {
 417   nat44_ei_main_t *nm = &nat44_ei_main;
 418
 419   fail_if_enabled ();
 420
 421   if (!c.users)
 422     c.users = 1024;
 423
 424   if (!c.sessions)
 425     c.sessions = 10 * 1024;
 426
 427   nm->rconfig = c;
 428
 429   if (!nm->frame_queue_nelts)
 430     nm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
 431
 432   nm->translations = c.sessions;
 433   nm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
 434   nm->user_buckets = nat_calc_bihash_buckets (c.users);
 435
 436   nm->pat = (!c.static_mapping_only ||
 437              (c.static_mapping_only && c.connection_tracking));
 438
 439   nm->static_mapping_only = c.static_mapping_only;
 440   nm->static_mapping_connection_tracking = c.connection_tracking;
 441   nm->out2in_dpo = c.out2in_dpo;
 442   nm->forwarding_enabled = 0;
 443   nm->mss_clamping = 0;
 444
 445   nm->max_users_per_thread = c.users;
 446   nm->max_translations_per_thread = c.sessions;
 447   nm->max_translations_per_user =
 448     c.user_sessions ? c.user_sessions : nm->max_translations_per_thread;
 449
 450   nm->inside_vrf_id = c.inside_vrf;
 451   nm->inside_fib_index = fib_table_find_or_create_and_lock (
 452     FIB_PROTOCOL_IP4, c.inside_vrf, nm->fib_src_hi);
 453
 454   nm->outside_vrf_id = c.outside_vrf;
 455   nm->outside_fib_index = fib_table_find_or_create_and_lock (
 456     FIB_PROTOCOL_IP4, c.outside_vrf, nm->fib_src_hi);
 457
 458   nat_reset_timeouts (&nm->timeouts);
 459   nat44_ei_db_init (nm->translations, nm->translation_buckets,
 460                     nm->user_buckets);
 461   nat44_ei_set_alloc_default ();
 462
 463   // TODO: zero simple counter for all counters missing
 464
 465   vlib_zero_simple_counter (&nm->total_users, 0);
 466   vlib_zero_simple_counter (&nm->total_sessions, 0);
 467   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
 468
 469   nat_ha_enable ();
 470   nm->enabled = 1;
 471
 472   return 0;
 473 }
 474
 475 void
 476 nat44_ei_addresses_free (nat44_ei_address_t **addresses)
 477 {
 478   nat44_ei_address_t *ap;
 479   vec_foreach (ap, *addresses)
 480     {
 481 #define _(N, i, n, s) vec_free (ap->busy_##n##_ports_per_thread);
 482       foreach_nat_protocol
 483 #undef _
 484     }
 485   vec_free (*addresses);
 486   *addresses = 0;
 487 }
 488
 489 int
 490 nat44_ei_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
 491 {
 492   const char *feature_name, *del_feature_name;
 493   nat44_ei_main_t *nm = &nat44_ei_main;
 494   nat44_ei_interface_t *i;
 495   nat44_ei_address_t *ap;
 496   nat44_ei_static_mapping_t *m;
 497   nat44_ei_outside_fib_t *outside_fib;
 498   u32 fib_index =
 499     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 500
 501   fail_if_disabled ();
 502
 503   if (nm->out2in_dpo && !is_inside)
 504     {
 505       nat44_ei_log_err ("error unsupported");
 506       return VNET_API_ERROR_UNSUPPORTED;
 507     }
 508
 509   pool_foreach (i, nm->output_feature_interfaces)
 510     {
 511       if (i->sw_if_index == sw_if_index)
 512         {
 513           nat44_ei_log_err ("error interface already configured");
 514           return VNET_API_ERROR_VALUE_EXIST;
 515         }
 516     }
 517
 518   if (nm->static_mapping_only && !(nm->static_mapping_connection_tracking))
 519     feature_name = is_inside ? "nat44-ei-in2out-fast" : "nat44-ei-out2in-fast";
 520   else
 521     {
 522       if (nm->num_workers > 1)
 523         feature_name = is_inside ? "nat44-ei-in2out-worker-handoff" :
 524                                    "nat44-ei-out2in-worker-handoff";
 525       else
 526         feature_name = is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 527     }
 528
 529   if (nm->fq_in2out_index == ~0 && nm->num_workers > 1)
 530     nm->fq_in2out_index = vlib_frame_queue_main_init (nm->in2out_node_index,
 531                                                       nm->frame_queue_nelts);
 532
 533   if (nm->fq_out2in_index == ~0 && nm->num_workers > 1)
 534     nm->fq_out2in_index = vlib_frame_queue_main_init (nm->out2in_node_index,
 535                                                       nm->frame_queue_nelts);
 536
 537   if (!is_inside)
 538     {
 539       vec_foreach (outside_fib, nm->outside_fibs)
 540         {
 541           if (outside_fib->fib_index == fib_index)
 542             {
 543               if (is_del)
 544                 {
 545                   outside_fib->refcount--;
 546                   if (!outside_fib->refcount)
 547                     vec_del1 (nm->outside_fibs,
 548                               outside_fib - nm->outside_fibs);
 549                 }
 550               else
 551                 outside_fib->refcount++;
 552               goto feature_set;
 553             }
 554         }
 555       if (!is_del)
 556         {
 557           vec_add2 (nm->outside_fibs, outside_fib, 1);
 558           outside_fib->refcount = 1;
 559           outside_fib->fib_index = fib_index;
 560         }
 561     }
 562
 563 feature_set:
 564   pool_foreach (i, nm->interfaces)
 565     {
 566       if (i->sw_if_index == sw_if_index)
 567         {
 568           if (is_del)
 569             {
 570               if (nat44_ei_interface_is_inside (i) &&
 571                   nat44_ei_interface_is_outside (i))
 572                 {
 573                   if (is_inside)
 574                     i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 575                   else
 576                     i->flags &= ~NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 577
 578                   if (nm->num_workers > 1)
 579                     {
 580                       del_feature_name = "nat44-handoff-classify";
 581                       feature_name = !is_inside ?
 582                                        "nat44-ei-in2out-worker-handoff" :
 583                                        "nat44-ei-out2in-worker-handoff";
 584                     }
 585                   else
 586                     {
 587                       del_feature_name = "nat44-ei-classify";
 588                       feature_name =
 589                         !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 590                     }
 591
 592                   int rv =
 593                     ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 594                   if (rv)
 595                     return rv;
 596                   vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 597                                                sw_if_index, 0, 0, 0);
 598                   vnet_feature_enable_disable ("ip4-unicast", feature_name,
 599                                                sw_if_index, 1, 0, 0);
 600                   if (!is_inside)
 601                     {
 602                       vnet_feature_enable_disable ("ip4-local",
 603                                                    "nat44-ei-hairpinning",
 604                                                    sw_if_index, 1, 0, 0);
 605                     }
 606                 }
 607               else
 608                 {
 609                   int rv =
 610                     ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 611                   if (rv)
 612                     return rv;
 613                   vnet_feature_enable_disable ("ip4-unicast", feature_name,
 614                                                sw_if_index, 0, 0, 0);
 615                   pool_put (nm->interfaces, i);
 616                   if (is_inside)
 617                     {
 618                       vnet_feature_enable_disable ("ip4-local",
 619                                                    "nat44-ei-hairpinning",
 620                                                    sw_if_index, 0, 0, 0);
 621                     }
 622                 }
 623             }
 624           else
 625             {
 626               if ((nat44_ei_interface_is_inside (i) && is_inside) ||
 627                   (nat44_ei_interface_is_outside (i) && !is_inside))
 628                 return 0;
 629
 630               if (nm->num_workers > 1)
 631                 {
 632                   del_feature_name = !is_inside ?
 633                                        "nat44-ei-in2out-worker-handoff" :
 634                                        "nat44-ei-out2in-worker-handoff";
 635                   feature_name = "nat44-handoff-classify";
 636                 }
 637               else
 638                 {
 639                   del_feature_name =
 640                     !is_inside ? "nat44-ei-in2out" : "nat44-ei-out2in";
 641                   feature_name = "nat44-ei-classify";
 642                 }
 643
 644               int rv =
 645                 ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 646               if (rv)
 647                 return rv;
 648               vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
 649                                            sw_if_index, 0, 0, 0);
 650               vnet_feature_enable_disable ("ip4-unicast", feature_name,
 651                                            sw_if_index, 1, 0, 0);
 652               if (!is_inside)
 653                 {
 654                   vnet_feature_enable_disable (
 655                     "ip4-local", "nat44-ei-hairpinning", sw_if_index, 0, 0, 0);
 656                 }
 657               goto set_flags;
 658             }
 659
 660           goto fib;
 661         }
 662     }
 663
 664   if (is_del)
 665     {
 666       nat44_ei_log_err ("error interface couldn't be found");
 667       return VNET_API_ERROR_NO_SUCH_ENTRY;
 668     }
 669
 670   pool_get (nm->interfaces, i);
 671   i->sw_if_index = sw_if_index;
 672   i->flags = 0;
 673   nat_validate_interface_counters (nm, sw_if_index);
 674
 675   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
 676                                0);
 677
 678   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 679   if (rv)
 680     return rv;
 681
 682   if (is_inside && !nm->out2in_dpo)
 683     {
 684       vnet_feature_enable_disable ("ip4-local", "nat44-ei-hairpinning",
 685                                    sw_if_index, 1, 0, 0);
 686     }
 687
 688 set_flags:
 689   if (is_inside)
 690     {
 691       i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 692       return 0;
 693     }
 694   else
 695     i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 696
 697   /* Add/delete external addresses to FIB */
 698 fib:
 699   vec_foreach (ap, nm->addresses)
 700     nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, !is_del);
 701
 702   pool_foreach (m, nm->static_mappings)
 703     {
 704       if (!(nat44_ei_is_addr_only_static_mapping (m)) ||
 705           (m->local_addr.as_u32 == m->external_addr.as_u32))
 706         continue;
 707
 708       nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 709                                     !is_del);
 710     }
 711
 712   return 0;
 713 }
 714
 715 int
 716 nat44_ei_interface_add_del_output_feature (u32 sw_if_index, u8 is_inside,
 717                                            int is_del)
 718 {
 719   nat44_ei_main_t *nm = &nat44_ei_main;
 720   nat44_ei_interface_t *i;
 721   nat44_ei_address_t *ap;
 722   nat44_ei_static_mapping_t *m;
 723   nat44_ei_outside_fib_t *outside_fib;
 724   u32 fib_index =
 725     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
 726
 727   fail_if_disabled ();
 728
 729   if (nm->static_mapping_only && !(nm->static_mapping_connection_tracking))
 730     {
 731       nat44_ei_log_err ("error unsupported");
 732       return VNET_API_ERROR_UNSUPPORTED;
 733     }
 734
 735   pool_foreach (i, nm->interfaces)
 736     {
 737       if (i->sw_if_index == sw_if_index)
 738         {
 739           nat44_ei_log_err ("error interface already configured");
 740           return VNET_API_ERROR_VALUE_EXIST;
 741         }
 742     }
 743
 744   if (!is_inside)
 745     {
 746       vec_foreach (outside_fib, nm->outside_fibs)
 747         {
 748           if (outside_fib->fib_index == fib_index)
 749             {
 750               if (is_del)
 751                 {
 752                   outside_fib->refcount--;
 753                   if (!outside_fib->refcount)
 754                     vec_del1 (nm->outside_fibs,
 755                               outside_fib - nm->outside_fibs);
 756                 }
 757               else
 758                 outside_fib->refcount++;
 759               goto feature_set;
 760             }
 761         }
 762       if (!is_del)
 763         {
 764           vec_add2 (nm->outside_fibs, outside_fib, 1);
 765           outside_fib->refcount = 1;
 766           outside_fib->fib_index = fib_index;
 767         }
 768     }
 769
 770 feature_set:
 771   if (is_inside)
 772     {
 773       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
 774       if (rv)
 775         return rv;
 776       rv =
 777         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
 778       if (rv)
 779         return rv;
 780       vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-hairpin-dst",
 781                                    sw_if_index, !is_del, 0, 0);
 782       vnet_feature_enable_disable ("ip4-output", "nat44-ei-hairpin-src",
 783                                    sw_if_index, !is_del, 0, 0);
 784       goto fq;
 785     }
 786
 787   if (nm->num_workers > 1)
 788     {
 789       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
 790       if (rv)
 791         return rv;
 792       rv =
 793         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
 794       if (rv)
 795         return rv;
 796       vnet_feature_enable_disable ("ip4-unicast",
 797                                    "nat44-ei-out2in-worker-handoff",
 798                                    sw_if_index, !is_del, 0, 0);
 799       vnet_feature_enable_disable ("ip4-output",
 800                                    "nat44-ei-in2out-output-worker-handoff",
 801                                    sw_if_index, !is_del, 0, 0);
 802     }
 803   else
 804     {
 805       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
 806       if (rv)
 807         return rv;
 808       rv =
 809         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
 810       if (rv)
 811         return rv;
 812       vnet_feature_enable_disable ("ip4-unicast", "nat44-ei-out2in",
 813                                    sw_if_index, !is_del, 0, 0);
 814       vnet_feature_enable_disable ("ip4-output", "nat44-ei-in2out-output",
 815                                    sw_if_index, !is_del, 0, 0);
 816     }
 817
 818 fq:
 819   if (nm->fq_in2out_output_index == ~0 && nm->num_workers > 1)
 820     nm->fq_in2out_output_index =
 821       vlib_frame_queue_main_init (nm->in2out_output_node_index, 0);
 822
 823   if (nm->fq_out2in_index == ~0 && nm->num_workers > 1)
 824     nm->fq_out2in_index =
 825       vlib_frame_queue_main_init (nm->out2in_node_index, 0);
 826
 827   pool_foreach (i, nm->output_feature_interfaces)
 828     {
 829       if (i->sw_if_index == sw_if_index)
 830         {
 831           if (is_del)
 832             pool_put (nm->output_feature_interfaces, i);
 833           else
 834             return VNET_API_ERROR_VALUE_EXIST;
 835
 836           goto fib;
 837         }
 838     }
 839
 840   if (is_del)
 841     {
 842       nat44_ei_log_err ("error interface couldn't be found");
 843       return VNET_API_ERROR_NO_SUCH_ENTRY;
 844     }
 845
 846   pool_get (nm->output_feature_interfaces, i);
 847   i->sw_if_index = sw_if_index;
 848   i->flags = 0;
 849   nat_validate_interface_counters (nm, sw_if_index);
 850   if (is_inside)
 851     i->flags |= NAT44_EI_INTERFACE_FLAG_IS_INSIDE;
 852   else
 853     i->flags |= NAT44_EI_INTERFACE_FLAG_IS_OUTSIDE;
 854
 855   /* Add/delete external addresses to FIB */
 856 fib:
 857   if (is_inside)
 858     return 0;
 859
 860   vec_foreach (ap, nm->addresses)
 861     nat44_ei_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, !is_del);
 862
 863   pool_foreach (m, nm->static_mappings)
 864     {
 865       if (!((nat44_ei_is_addr_only_static_mapping (m))) ||
 866           (m->local_addr.as_u32 == m->external_addr.as_u32))
 867         continue;
 868
 869       nat44_ei_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 870                                     !is_del);
 871     }
 872
 873   return 0;
 874 }
 875
 876 int
 877 nat44_ei_plugin_disable ()
 878 {
 879   nat44_ei_main_t *nm = &nat44_ei_main;
 880   nat44_ei_interface_t *i, *vec;
 881   int error = 0;
 882
 883   // first unregister all nodes from interfaces
 884   vec = vec_dup (nm->interfaces);
 885   vec_foreach (i, vec)
 886     {
 887       if (nat44_ei_interface_is_inside (i))
 888         error = nat44_ei_interface_add_del (i->sw_if_index, 1, 1);
 889       if (nat44_ei_interface_is_outside (i))
 890         error = nat44_ei_interface_add_del (i->sw_if_index, 0, 1);
 891
 892       if (error)
 893         {
 894           nat44_ei_log_err ("error occurred while removing interface %u",
 895                             i->sw_if_index);
 896         }
 897     }
 898   vec_free (vec);
 899   nm->interfaces = 0;
 900
 901   vec = vec_dup (nm->output_feature_interfaces);
 902   vec_foreach (i, vec)
 903     {
 904       if (nat44_ei_interface_is_inside (i))
 905         error =
 906           nat44_ei_interface_add_del_output_feature (i->sw_if_index, 1, 1);
 907       if (nat44_ei_interface_is_outside (i))
 908         error =
 909           nat44_ei_interface_add_del_output_feature (i->sw_if_index, 0, 1);
 910
 911       if (error)
 912         {
 913           nat44_ei_log_err ("error occurred while removing interface %u",
 914                             i->sw_if_index);
 915         }
 916     }
 917   vec_free (vec);
 918   nm->output_feature_interfaces = 0;
 919
 920   nat_ha_disable ();
 921   nat44_ei_db_free ();
 922
 923   nat44_ei_addresses_free (&nm->addresses);
 924
 925   vec_free (nm->to_resolve);
 926   vec_free (nm->auto_add_sw_if_indices);
 927
 928   nm->to_resolve = 0;
 929   nm->auto_add_sw_if_indices = 0;
 930
 931   nm->forwarding_enabled = 0;
 932
 933   nm->enabled = 0;
 934   clib_memset (&nm->rconfig, 0, sizeof (nm->rconfig));
 935
 936   return error;
 937 }
 938
 939 int
 940 nat44_ei_set_outside_address_and_port (nat44_ei_address_t *addresses,
 941                                        u32 thread_index, ip4_address_t addr,
 942                                        u16 port, nat_protocol_t protocol)
 943 {
 944   nat44_ei_main_t *nm = &nat44_ei_main;
 945   nat44_ei_address_t *a = 0;
 946   u32 address_index;
 947   u16 port_host_byte_order = clib_net_to_host_u16 (port);
 948
 949   for (address_index = 0; address_index < vec_len (addresses); address_index++)
 950     {
 951       if (addresses[address_index].addr.as_u32 != addr.as_u32)
 952         continue;
 953
 954       a = addresses + address_index;
 955       switch (protocol)
 956         {
 957 #define _(N, j, n, s)                                                         \
 958   case NAT_PROTOCOL_##N:                                                      \
 959     if (a->busy_##n##_port_refcounts[port_host_byte_order])                   \
 960       return VNET_API_ERROR_INSTANCE_IN_USE;                                  \
 961     ++a->busy_##n##_port_refcounts[port_host_byte_order];                     \
 962     a->busy_##n##_ports_per_thread[thread_index]++;                           \
 963     a->busy_##n##_ports++;                                                    \
 964     return 0;
 965           foreach_nat_protocol
 966 #undef _
 967             default : nat_elog_info (nm, "unknown protocol");
 968           return 1;
 969         }
 970     }
 971
 972   return VNET_API_ERROR_NO_SUCH_ENTRY;
 973 }
 974
 975 void
 976 nat44_ei_add_del_address_dpo (ip4_address_t addr, u8 is_add)
 977 {
 978   nat44_ei_main_t *nm = &nat44_ei_main;
 979   dpo_id_t dpo_v4 = DPO_INVALID;
 980   fib_prefix_t pfx = {
 981     .fp_proto = FIB_PROTOCOL_IP4,
 982     .fp_len = 32,
 983     .fp_addr.ip4.as_u32 = addr.as_u32,
 984   };
 985
 986   if (is_add)
 987     {
 988       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
 989       fib_table_entry_special_dpo_add (0, &pfx, nm->fib_src_hi,
 990                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
 991       dpo_reset (&dpo_v4);
 992     }
 993   else
 994     {
 995       fib_table_entry_special_remove (0, &pfx, nm->fib_src_hi);
 996     }
 997 }
 998
 999 void
1000 nat44_ei_free_outside_address_and_port (nat44_ei_address_t *addresses,
1001                                         u32 thread_index, ip4_address_t *addr,
1002                                         u16 port, nat_protocol_t protocol)
1003 {
1004   nat44_ei_main_t *nm = &nat44_ei_main;
1005   nat44_ei_address_t *a;
1006   u32 address_index;
1007   u16 port_host_byte_order = clib_net_to_host_u16 (port);
1008
1009   for (address_index = 0; address_index < vec_len (addresses); address_index++)
1010     {
1011       if (addresses[address_index].addr.as_u32 == addr->as_u32)
1012         break;
1013     }
1014
1015   ASSERT (address_index < vec_len (addresses));
1016
1017   a = addresses + address_index;
1018
1019   switch (protocol)
1020     {
1021 #define _(N, i, n, s)                                                         \
1022   case NAT_PROTOCOL_##N:                                                      \
1023     ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1);         \
1024     --a->busy_##n##_port_refcounts[port_host_byte_order];                     \
1025     a->busy_##n##_ports--;                                                    \
1026     a->busy_##n##_ports_per_thread[thread_index]--;                           \
1027     break;
1028       foreach_nat_protocol
1029 #undef _
1030         default : nat_elog_info (nm, "unknown protocol");
1031       return;
1032     }
1033 }
1034
1035 void
1036 nat44_ei_free_session_data_v2 (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1037                                u32 thread_index, u8 is_ha)
1038 {
1039   clib_bihash_kv_8_8_t kv;
1040
1041   /* session lookup tables */
1042   init_nat_i2o_k (&kv, s);
1043   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1044     nat_elog_warn (nm, "in2out key del failed");
1045   init_nat_o2i_k (&kv, s);
1046   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1047     nat_elog_warn (nm, "out2in key del failed");
1048
1049   if (!is_ha)
1050     nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1051                              &s->in2out.addr, s->in2out.port, &s->out2in.addr,
1052                              s->out2in.port, s->nat_proto);
1053
1054   if (nat44_ei_is_unk_proto_session (s))
1055     return;
1056
1057   if (!is_ha)
1058     {
1059       /* log NAT event */
1060       nat_ipfix_logging_nat44_ses_delete (
1061         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1062         s->nat_proto, s->in2out.port, s->out2in.port, s->in2out.fib_index);
1063
1064       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1065                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1066                    thread_index);
1067     }
1068
1069   if (nat44_ei_is_session_static (s))
1070     return;
1071
1072   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1073                                           &s->out2in.addr, s->out2in.port,
1074                                           s->nat_proto);
1075 }
1076
1077 nat44_ei_user_t *
1078 nat44_ei_user_get_or_create (nat44_ei_main_t *nm, ip4_address_t *addr,
1079                              u32 fib_index, u32 thread_index)
1080 {
1081   nat44_ei_user_t *u = 0;
1082   nat44_ei_user_key_t user_key;
1083   clib_bihash_kv_8_8_t kv, value;
1084   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1085   dlist_elt_t *per_user_list_head_elt;
1086
1087   user_key.addr.as_u32 = addr->as_u32;
1088   user_key.fib_index = fib_index;
1089   kv.key = user_key.as_u64;
1090
1091   /* Ever heard of the "user" = src ip4 address before? */
1092   if (clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1093     {
1094       if (pool_elts (tnm->users) >= nm->max_users_per_thread)
1095         {
1096           vlib_increment_simple_counter (&nm->user_limit_reached, thread_index,
1097                                          0, 1);
1098           nat_elog_warn (nm, "maximum user limit reached");
1099           return NULL;
1100         }
1101       /* no, make a new one */
1102       pool_get (tnm->users, u);
1103       clib_memset (u, 0, sizeof (*u));
1104
1105       u->addr.as_u32 = addr->as_u32;
1106       u->fib_index = fib_index;
1107
1108       pool_get (tnm->list_pool, per_user_list_head_elt);
1109
1110       u->sessions_per_user_list_head_index =
1111         per_user_list_head_elt - tnm->list_pool;
1112
1113       clib_dlist_init (tnm->list_pool, u->sessions_per_user_list_head_index);
1114
1115       kv.value = u - tnm->users;
1116
1117       /* add user */
1118       if (clib_bihash_add_del_8_8 (&tnm->user_hash, &kv, 1))
1119         {
1120           nat_elog_warn (nm, "user_hash key add failed");
1121           nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1122           return NULL;
1123         }
1124
1125       vlib_set_simple_counter (&nm->total_users, thread_index, 0,
1126                                pool_elts (tnm->users));
1127     }
1128   else
1129     {
1130       u = pool_elt_at_index (tnm->users, value.value);
1131     }
1132
1133   return u;
1134 }
1135
1136 nat44_ei_session_t *
1137 nat44_ei_session_alloc_or_recycle (nat44_ei_main_t *nm, nat44_ei_user_t *u,
1138                                    u32 thread_index, f64 now)
1139 {
1140   nat44_ei_session_t *s;
1141   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1142   u32 oldest_per_user_translation_list_index, session_index;
1143   dlist_elt_t *oldest_per_user_translation_list_elt;
1144   dlist_elt_t *per_user_translation_list_elt;
1145
1146   /* Over quota? Recycle the least recently used translation */
1147   if ((u->nsessions + u->nstaticsessions) >= nm->max_translations_per_user)
1148     {
1149       oldest_per_user_translation_list_index = clib_dlist_remove_head (
1150         tnm->list_pool, u->sessions_per_user_list_head_index);
1151
1152       ASSERT (oldest_per_user_translation_list_index != ~0);
1153
1154       /* Add it back to the end of the LRU list */
1155       clib_dlist_addtail (tnm->list_pool, u->sessions_per_user_list_head_index,
1156                           oldest_per_user_translation_list_index);
1157       /* Get the list element */
1158       oldest_per_user_translation_list_elt = pool_elt_at_index (
1159         tnm->list_pool, oldest_per_user_translation_list_index);
1160
1161       /* Get the session index from the list element */
1162       session_index = oldest_per_user_translation_list_elt->value;
1163
1164       /* Get the session */
1165       s = pool_elt_at_index (tnm->sessions, session_index);
1166
1167       nat44_ei_free_session_data_v2 (nm, s, thread_index, 0);
1168       if (nat44_ei_is_session_static (s))
1169         u->nstaticsessions--;
1170       else
1171         u->nsessions--;
1172       s->flags = 0;
1173       s->total_bytes = 0;
1174       s->total_pkts = 0;
1175       s->state = 0;
1176       s->ext_host_addr.as_u32 = 0;
1177       s->ext_host_port = 0;
1178       s->ext_host_nat_addr.as_u32 = 0;
1179       s->ext_host_nat_port = 0;
1180     }
1181   else
1182     {
1183       pool_get (tnm->sessions, s);
1184       clib_memset (s, 0, sizeof (*s));
1185
1186       /* Create list elts */
1187       pool_get (tnm->list_pool, per_user_translation_list_elt);
1188       clib_dlist_init (tnm->list_pool,
1189                        per_user_translation_list_elt - tnm->list_pool);
1190
1191       per_user_translation_list_elt->value = s - tnm->sessions;
1192       s->per_user_index = per_user_translation_list_elt - tnm->list_pool;
1193       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1194
1195       clib_dlist_addtail (tnm->list_pool, s->per_user_list_head_index,
1196                           per_user_translation_list_elt - tnm->list_pool);
1197
1198       s->user_index = u - tnm->users;
1199       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1200                                pool_elts (tnm->sessions));
1201     }
1202
1203   s->ha_last_refreshed = now;
1204
1205   return s;
1206 }
1207
1208 void
1209 nat44_ei_free_session_data (nat44_ei_main_t *nm, nat44_ei_session_t *s,
1210                             u32 thread_index, u8 is_ha)
1211 {
1212   clib_bihash_kv_8_8_t kv;
1213
1214   init_nat_i2o_k (&kv, s);
1215   if (clib_bihash_add_del_8_8 (&nm->in2out, &kv, 0))
1216     nat_elog_warn (nm, "in2out key del failed");
1217
1218   init_nat_o2i_k (&kv, s);
1219   if (clib_bihash_add_del_8_8 (&nm->out2in, &kv, 0))
1220     nat_elog_warn (nm, "out2in key del failed");
1221
1222   if (!is_ha)
1223     {
1224       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
1225                                &s->in2out.addr, s->in2out.port,
1226                                &s->out2in.addr, s->out2in.port, s->nat_proto);
1227
1228       nat_ipfix_logging_nat44_ses_delete (
1229         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
1230         s->nat_proto, s->in2out.port, s->out2in.port, s->in2out.fib_index);
1231
1232       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
1233                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
1234                    thread_index);
1235     }
1236
1237   if (nat44_ei_is_session_static (s))
1238     return;
1239
1240   nat44_ei_free_outside_address_and_port (nm->addresses, thread_index,
1241                                           &s->out2in.addr, s->out2in.port,
1242                                           s->nat_proto);
1243 }
1244
1245 static_always_inline void
1246 nat44_ei_user_del_sessions (nat44_ei_user_t *u, u32 thread_index)
1247 {
1248   dlist_elt_t *elt;
1249   nat44_ei_session_t *s;
1250
1251   nat44_ei_main_t *nm = &nat44_ei_main;
1252   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
1253
1254   // get head
1255   elt =
1256     pool_elt_at_index (tnm->list_pool, u->sessions_per_user_list_head_index);
1257   // get first element
1258   elt = pool_elt_at_index (tnm->list_pool, elt->next);
1259
1260   while (elt->value != ~0)
1261     {
1262       s = pool_elt_at_index (tnm->sessions, elt->value);
1263       elt = pool_elt_at_index (tnm->list_pool, elt->next);
1264
1265       nat44_ei_free_session_data (nm, s, thread_index, 0);
1266       nat44_ei_delete_session (nm, s, thread_index);
1267     }
1268 }
1269
1270 int
1271 nat44_ei_user_del (ip4_address_t *addr, u32 fib_index)
1272 {
1273   int rv = 1;
1274
1275   nat44_ei_main_t *nm = &nat44_ei_main;
1276   nat44_ei_main_per_thread_data_t *tnm;
1277
1278   nat44_ei_user_key_t user_key;
1279   clib_bihash_kv_8_8_t kv, value;
1280
1281   user_key.addr.as_u32 = addr->as_u32;
1282   user_key.fib_index = fib_index;
1283   kv.key = user_key.as_u64;
1284
1285   if (nm->num_workers > 1)
1286     {
1287       vec_foreach (tnm, nm->per_thread_data)
1288         {
1289           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1290             {
1291               nat44_ei_user_del_sessions (
1292                 pool_elt_at_index (tnm->users, value.value),
1293                 tnm->thread_index);
1294               rv = 0;
1295               break;
1296             }
1297         }
1298     }
1299   else
1300     {
1301       tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1302       if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1303         {
1304           nat44_ei_user_del_sessions (
1305             pool_elt_at_index (tnm->users, value.value), tnm->thread_index);
1306           rv = 0;
1307         }
1308     }
1309   return rv;
1310 }
1311
1312 void
1313 nat44_ei_static_mapping_del_sessions (nat44_ei_main_t *nm,
1314                                       nat44_ei_main_per_thread_data_t *tnm,
1315                                       nat44_ei_user_key_t u_key, int addr_only,
1316                                       ip4_address_t e_addr, u16 e_port)
1317 {
1318   clib_bihash_kv_8_8_t kv, value;
1319   kv.key = u_key.as_u64;
1320   u64 user_index;
1321   dlist_elt_t *head, *elt;
1322   nat44_ei_user_t *u;
1323   nat44_ei_session_t *s;
1324   u32 elt_index, head_index, ses_index;
1325
1326   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1327     {
1328       user_index = value.value;
1329       u = pool_elt_at_index (tnm->users, user_index);
1330       if (u->nstaticsessions)
1331         {
1332           head_index = u->sessions_per_user_list_head_index;
1333           head = pool_elt_at_index (tnm->list_pool, head_index);
1334           elt_index = head->next;
1335           elt = pool_elt_at_index (tnm->list_pool, elt_index);
1336           ses_index = elt->value;
1337           while (ses_index != ~0)
1338             {
1339               s = pool_elt_at_index (tnm->sessions, ses_index);
1340               elt = pool_elt_at_index (tnm->list_pool, elt->next);
1341               ses_index = elt->value;
1342
1343               if (!addr_only)
1344                 {
1345                   if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1346                       (s->out2in.port != e_port))
1347                     continue;
1348                 }
1349
1350               if (!nat44_ei_is_session_static (s))
1351                 continue;
1352
1353               nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data,
1354                                              0);
1355               nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1356
1357               if (!addr_only)
1358                 break;
1359             }
1360         }
1361     }
1362 }
1363
1364 u32
1365 nat44_ei_get_in2out_worker_index (ip4_header_t *ip0, u32 rx_fib_index0,
1366                                   u8 is_output)
1367 {
1368   nat44_ei_main_t *nm = &nat44_ei_main;
1369   u32 next_worker_index = 0;
1370   u32 hash;
1371
1372   next_worker_index = nm->first_worker_index;
1373   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
1374          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
1375
1376   if (PREDICT_TRUE (is_pow2 (_vec_len (nm->workers))))
1377     next_worker_index += nm->workers[hash & (_vec_len (nm->workers) - 1)];
1378   else
1379     next_worker_index += nm->workers[hash % _vec_len (nm->workers)];
1380
1381   return next_worker_index;
1382 }
1383
1384 u32
1385 nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
1386                                   u32 rx_fib_index0, u8 is_output)
1387 {
1388   nat44_ei_main_t *nm = &nat44_ei_main;
1389   udp_header_t *udp;
1390   u16 port;
1391   clib_bihash_kv_8_8_t kv, value;
1392   nat44_ei_static_mapping_t *m;
1393   u32 proto;
1394   u32 next_worker_index = 0;
1395
1396   /* first try static mappings without port */
1397   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1398     {
1399       init_nat_k (&kv, ip0->dst_address, 0, rx_fib_index0, 0);
1400       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1401                                    &value))
1402         {
1403           m = pool_elt_at_index (nm->static_mappings, value.value);
1404           return m->workers[0];
1405         }
1406     }
1407
1408   proto = ip_proto_to_nat_proto (ip0->protocol);
1409   udp = ip4_next_header (ip0);
1410   port = udp->dst_port;
1411
1412   /* unknown protocol */
1413   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
1414     {
1415       /* use current thread */
1416       return vlib_get_thread_index ();
1417     }
1418
1419   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
1420     {
1421       icmp46_header_t *icmp = (icmp46_header_t *) udp;
1422       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
1423       if (!icmp_type_is_error_message (
1424             vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
1425         port = vnet_buffer (b)->ip.reass.l4_src_port;
1426       else
1427         {
1428           /* if error message, then it's not fragmented and we can access it */
1429           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
1430           proto = ip_proto_to_nat_proto (inner_ip->protocol);
1431           void *l4_header = ip4_next_header (inner_ip);
1432           switch (proto)
1433             {
1434             case NAT_PROTOCOL_ICMP:
1435               icmp = (icmp46_header_t *) l4_header;
1436               echo = (icmp_echo_header_t *) (icmp + 1);
1437               port = echo->identifier;
1438               break;
1439             case NAT_PROTOCOL_UDP:
1440             case NAT_PROTOCOL_TCP:
1441               port = ((tcp_udp_header_t *) l4_header)->src_port;
1442               break;
1443             default:
1444               return vlib_get_thread_index ();
1445             }
1446         }
1447     }
1448
1449   /* try static mappings with port */
1450   if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
1451     {
1452       init_nat_k (&kv, ip0->dst_address, port, rx_fib_index0, proto);
1453       if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
1454                                    &value))
1455         {
1456           m = pool_elt_at_index (nm->static_mappings, value.value);
1457           return m->workers[0];
1458         }
1459     }
1460
1461   /* worker by outside port */
1462   next_worker_index = nm->first_worker_index;
1463   next_worker_index +=
1464     nm->workers[(clib_net_to_host_u16 (port) - 1024) / nm->port_per_thread];
1465   return next_worker_index;
1466 }
1467
1468 static int
1469 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
1470                            u32 thread_index, nat_protocol_t proto,
1471                            ip4_address_t s_addr, ip4_address_t *addr,
1472                            u16 *port, u16 port_per_thread,
1473                            u32 snat_thread_index)
1474 {
1475   nat44_ei_main_t *nm = &nat44_ei_main;
1476   nat44_ei_address_t *a, *ga = 0;
1477   u32 portnum;
1478   int i;
1479
1480   if (vec_len (addresses) > 0)
1481     {
1482
1483       int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
1484
1485       for (i = s_addr_offset; i < vec_len (addresses); ++i)
1486         {
1487           a = addresses + i;
1488           switch (proto)
1489             {
1490 #define _(N, j, n, s)                                                         \
1491   case NAT_PROTOCOL_##N:                                                      \
1492     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
1493       {                                                                       \
1494         if (a->fib_index == fib_index)                                        \
1495           {                                                                   \
1496             while (1)                                                         \
1497               {                                                               \
1498                 portnum = (port_per_thread * snat_thread_index) +             \
1499                           nat_random_port (&nm->random_seed, 0,               \
1500                                            port_per_thread - 1) +             \
1501                           1024;                                               \
1502                 if (a->busy_##n##_port_refcounts[portnum])                    \
1503                   continue;                                                   \
1504                 --a->busy_##n##_port_refcounts[portnum];                      \
1505                 a->busy_##n##_ports_per_thread[thread_index]++;               \
1506                 a->busy_##n##_ports++;                                        \
1507                 *addr = a->addr;                                              \
1508                 *port = clib_host_to_net_u16 (portnum);                       \
1509                 return 0;                                                     \
1510               }                                                               \
1511           }                                                                   \
1512         else if (a->fib_index == ~0)                                          \
1513           {                                                                   \
1514             ga = a;                                                           \
1515           }                                                                   \
1516       }                                                                       \
1517     break;
1518               foreach_nat_protocol;
1519             default:
1520               nat_elog_info (nm, "unknown protocol");
1521               return 1;
1522             }
1523         }
1524
1525       for (i = 0; i < s_addr_offset; ++i)
1526         {
1527           a = addresses + i;
1528           switch (proto)
1529             {
1530               foreach_nat_protocol;
1531             default:
1532               nat_elog_info (nm, "unknown protocol");
1533               return 1;
1534             }
1535         }
1536   if (ga)
1537     {
1538       a = ga;
1539       // fake fib index to reuse macro
1540       fib_index = ~0;
1541       switch (proto)
1542         {
1543           foreach_nat_protocol;
1544             default : nat_elog_info (nm, "unknown protocol");
1545           return 1;
1546         }
1547     }
1548     }
1549
1550 #undef _
1551
1552   /* Totally out of translations to use... */
1553   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1554   return 1;
1555 }
1556
1557 static int
1558 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
1559                          u32 thread_index, nat_protocol_t proto,
1560                          ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1561                          u16 port_per_thread, u32 snat_thread_index)
1562 {
1563   nat44_ei_main_t *nm = &nat44_ei_main;
1564   nat44_ei_address_t *a = addresses;
1565   u16 portnum, ports;
1566
1567   ports = nm->end_port - nm->start_port + 1;
1568
1569   if (!vec_len (addresses))
1570     goto exhausted;
1571
1572   switch (proto)
1573     {
1574 #define _(N, i, n, s)                                                         \
1575   case NAT_PROTOCOL_##N:                                                      \
1576     if (a->busy_##n##_ports < ports)                                          \
1577       {                                                                       \
1578         while (1)                                                             \
1579           {                                                                   \
1580             portnum = nat_random_port (&nm->random_seed, nm->start_port,      \
1581                                        nm->end_port);                         \
1582             if (a->busy_##n##_port_refcounts[portnum])                        \
1583               continue;                                                       \
1584             ++a->busy_##n##_port_refcounts[portnum];                          \
1585             a->busy_##n##_ports++;                                            \
1586             *addr = a->addr;                                                  \
1587             *port = clib_host_to_net_u16 (portnum);                           \
1588             return 0;                                                         \
1589           }                                                                   \
1590       }                                                                       \
1591     break;
1592       foreach_nat_protocol
1593 #undef _
1594         default : nat_elog_info (nm, "unknown protocol");
1595       return 1;
1596     }
1597
1598 exhausted:
1599   /* Totally out of translations to use... */
1600   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1601   return 1;
1602 }
1603
1604 static int
1605 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
1606                         u32 thread_index, nat_protocol_t proto,
1607                         ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
1608                         u16 port_per_thread, u32 snat_thread_index)
1609 {
1610   nat44_ei_main_t *nm = &nat44_ei_main;
1611   nat44_ei_address_t *a = addresses;
1612   u16 m, ports, portnum, A, j;
1613   m = 16 - (nm->psid_offset + nm->psid_length);
1614   ports = (1 << (16 - nm->psid_length)) - (1 << m);
1615
1616   if (!vec_len (addresses))
1617     goto exhausted;
1618
1619   switch (proto)
1620     {
1621 #define _(N, i, n, s)                                                         \
1622   case NAT_PROTOCOL_##N:                                                      \
1623     if (a->busy_##n##_ports < ports)                                          \
1624       {                                                                       \
1625         while (1)                                                             \
1626           {                                                                   \
1627             A = nat_random_port (&nm->random_seed, 1,                         \
1628                                  pow2_mask (nm->psid_offset));                \
1629             j = nat_random_port (&nm->random_seed, 0, pow2_mask (m));         \
1630             portnum = A | (nm->psid << nm->psid_offset) | (j << (16 - m));    \
1631             if (a->busy_##n##_port_refcounts[portnum])                        \
1632               continue;                                                       \
1633             ++a->busy_##n##_port_refcounts[portnum];                          \
1634             a->busy_##n##_ports++;                                            \
1635             *addr = a->addr;                                                  \
1636             *port = clib_host_to_net_u16 (portnum);                           \
1637             return 0;                                                         \
1638           }                                                                   \
1639       }                                                                       \
1640     break;
1641       foreach_nat_protocol
1642 #undef _
1643         default : nat_elog_info (nm, "unknown protocol");
1644       return 1;
1645     }
1646
1647 exhausted:
1648   /* Totally out of translations to use... */
1649   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
1650   return 1;
1651 }
1652
1653 void
1654 nat44_ei_set_alloc_default ()
1655 {
1656   nat44_ei_main_t *nm = &nat44_ei_main;
1657
1658   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
1659   nm->alloc_addr_and_port = nat44_ei_alloc_default_cb;
1660 }
1661
1662 void
1663 nat44_ei_set_alloc_range (u16 start_port, u16 end_port)
1664 {
1665   nat44_ei_main_t *nm = &nat44_ei_main;
1666
1667   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_RANGE;
1668   nm->alloc_addr_and_port = nat44_ei_alloc_range_cb;
1669   nm->start_port = start_port;
1670   nm->end_port = end_port;
1671 }
1672
1673 void
1674 nat44_ei_set_alloc_mape (u16 psid, u16 psid_offset, u16 psid_length)
1675 {
1676   nat44_ei_main_t *nm = &nat44_ei_main;
1677
1678   nm->addr_and_port_alloc_alg = NAT44_EI_ADDR_AND_PORT_ALLOC_ALG_MAPE;
1679   nm->alloc_addr_and_port = nat44_ei_alloc_mape_cb;
1680   nm->psid = psid;
1681   nm->psid_offset = psid_offset;
1682   nm->psid_length = psid_length;
1683 }
1684
1685 static void
1686 nat44_ei_add_static_mapping_when_resolved (ip4_address_t l_addr, u16 l_port,
1687                                            u16 e_port, nat_protocol_t proto,
1688                                            u32 sw_if_index, u32 vrf_id,
1689                                            int addr_only, int identity_nat,
1690                                            u8 *tag)
1691 {
1692   nat44_ei_main_t *nm = &nat44_ei_main;
1693   nat44_ei_static_map_resolve_t *rp;
1694
1695   vec_add2 (nm->to_resolve, rp, 1);
1696   clib_memset (rp, 0, sizeof (*rp));
1697
1698   rp->l_addr.as_u32 = l_addr.as_u32;
1699   rp->l_port = l_port;
1700   rp->e_port = e_port;
1701   rp->sw_if_index = sw_if_index;
1702   rp->vrf_id = vrf_id;
1703   rp->proto = proto;
1704   rp->addr_only = addr_only;
1705   rp->identity_nat = identity_nat;
1706   rp->tag = vec_dup (tag);
1707 }
1708
1709 void
1710 nat44_ei_delete_session (nat44_ei_main_t *nm, nat44_ei_session_t *ses,
1711                          u32 thread_index)
1712 {
1713   nat44_ei_main_per_thread_data_t *tnm =
1714     vec_elt_at_index (nm->per_thread_data, thread_index);
1715   clib_bihash_kv_8_8_t kv, value;
1716   nat44_ei_user_t *u;
1717   const nat44_ei_user_key_t u_key = { .addr = ses->in2out.addr,
1718                                       .fib_index = ses->in2out.fib_index };
1719   const u8 u_static = nat44_ei_is_session_static (ses);
1720
1721   clib_dlist_remove (tnm->list_pool, ses->per_user_index);
1722   pool_put_index (tnm->list_pool, ses->per_user_index);
1723
1724   pool_put (tnm->sessions, ses);
1725   vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
1726                            pool_elts (tnm->sessions));
1727
1728   kv.key = u_key.as_u64;
1729   if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
1730     {
1731       u = pool_elt_at_index (tnm->users, value.value);
1732       if (u_static)
1733         u->nstaticsessions--;
1734       else
1735         u->nsessions--;
1736
1737       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
1738     }
1739 }
1740
1741 int
1742 nat44_ei_del_session (nat44_ei_main_t *nm, ip4_address_t *addr, u16 port,
1743                       nat_protocol_t proto, u32 vrf_id, int is_in)
1744 {
1745   nat44_ei_main_per_thread_data_t *tnm;
1746   clib_bihash_kv_8_8_t kv, value;
1747   ip4_header_t ip;
1748   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
1749   nat44_ei_session_t *s;
1750   clib_bihash_8_8_t *t;
1751
1752   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
1753   if (nm->num_workers > 1)
1754     tnm = vec_elt_at_index (nm->per_thread_data,
1755                             nm->worker_in2out_cb (&ip, fib_index, 0));
1756   else
1757     tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
1758
1759   init_nat_k (&kv, *addr, port, fib_index, proto);
1760   t = is_in ? &nm->in2out : &nm->out2in;
1761   if (!clib_bihash_search_8_8 (t, &kv, &value))
1762     {
1763       if (pool_is_free_index (tnm->sessions, value.value))
1764         return VNET_API_ERROR_UNSPECIFIED;
1765
1766       s = pool_elt_at_index (tnm->sessions, value.value);
1767       nat44_ei_free_session_data_v2 (nm, s, tnm - nm->per_thread_data, 0);
1768       nat44_ei_delete_session (nm, s, tnm - nm->per_thread_data);
1769       return 0;
1770     }
1771
1772   return VNET_API_ERROR_NO_SUCH_ENTRY;
1773 }
1774
1775 u32
1776 nat44_ei_get_thread_idx_by_port (u16 e_port)
1777 {
1778   nat44_ei_main_t *nm = &nat44_ei_main;
1779   u32 thread_idx = nm->num_workers;
1780   if (nm->num_workers > 1)
1781     {
1782       thread_idx = nm->first_worker_index +
1783                    nm->workers[(e_port - 1024) / nm->port_per_thread];
1784     }
1785   return thread_idx;
1786 }
1787
1788 void
1789 nat44_ei_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
1790                               int is_add)
1791 {
1792   nat44_ei_main_t *nm = &nat44_ei_main;
1793   fib_prefix_t prefix = {
1794     .fp_len = p_len,
1795     .fp_proto = FIB_PROTOCOL_IP4,
1796     .fp_addr = {
1797                 .ip4.as_u32 = addr->as_u32,
1798                 },
1799   };
1800   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1801
1802   if (is_add)
1803     fib_table_entry_update_one_path (
1804       fib_index, &prefix, nm->fib_src_low,
1805       (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_LOCAL |
1806        FIB_ENTRY_FLAG_EXCLUSIVE),
1807       DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
1808   else
1809     fib_table_entry_delete (fib_index, &prefix, nm->fib_src_low);
1810 }
1811
1812 int
1813 nat44_ei_add_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
1814                                  u16 l_port, u16 e_port, nat_protocol_t proto,
1815                                  u32 sw_if_index, u32 vrf_id, u8 addr_only,
1816                                  u8 identity_nat, u8 *tag, u8 is_add)
1817 {
1818   nat44_ei_main_t *nm = &nat44_ei_main;
1819   nat44_ei_static_mapping_t *m = 0;
1820   clib_bihash_kv_8_8_t kv, value;
1821   nat44_ei_address_t *a = 0;
1822   u32 fib_index = ~0;
1823   nat44_ei_interface_t *interface;
1824   nat44_ei_main_per_thread_data_t *tnm;
1825   nat44_ei_user_key_t u_key;
1826   nat44_ei_user_t *u;
1827   dlist_elt_t *head, *elt;
1828   u32 elt_index, head_index;
1829   u32 ses_index;
1830   u64 user_index;
1831   nat44_ei_session_t *s;
1832   nat44_ei_static_map_resolve_t *rp, *rp_match = 0;
1833   nat44_ei_lb_addr_port_t *local;
1834   u32 find = ~0;
1835   int i;
1836
1837   if (sw_if_index != ~0)
1838     {
1839       ip4_address_t *first_int_addr;
1840
1841       for (i = 0; i < vec_len (nm->to_resolve); i++)
1842         {
1843           rp = nm->to_resolve + i;
1844           if (rp->sw_if_index != sw_if_index ||
1845               rp->l_addr.as_u32 != l_addr.as_u32 || rp->vrf_id != vrf_id ||
1846               rp->addr_only != addr_only)
1847             continue;
1848
1849           if (!addr_only)
1850             {
1851               if ((rp->l_port != l_port && rp->e_port != e_port) ||
1852                   rp->proto != proto)
1853                 continue;
1854             }
1855
1856           rp_match = rp;
1857           break;
1858         }
1859
1860       /* Might be already set... */
1861       first_int_addr = ip4_interface_first_address (
1862         nm->ip4_main, sw_if_index, 0 /* just want the address */);
1863
1864       if (is_add)
1865         {
1866           if (rp_match)
1867             return VNET_API_ERROR_VALUE_EXIST;
1868
1869           nat44_ei_add_static_mapping_when_resolved (
1870             l_addr, l_port, e_port, proto, sw_if_index, vrf_id, addr_only,
1871             identity_nat, tag);
1872
1873           /* DHCP resolution required? */
1874           if (!first_int_addr)
1875             return 0;
1876
1877           e_addr.as_u32 = first_int_addr->as_u32;
1878           /* Identity mapping? */
1879           if (l_addr.as_u32 == 0)
1880             l_addr.as_u32 = e_addr.as_u32;
1881         }
1882       else
1883         {
1884           if (!rp_match)
1885             return VNET_API_ERROR_NO_SUCH_ENTRY;
1886
1887           vec_del1 (nm->to_resolve, i);
1888
1889           if (!first_int_addr)
1890             return 0;
1891
1892           e_addr.as_u32 = first_int_addr->as_u32;
1893           /* Identity mapping? */
1894           if (l_addr.as_u32 == 0)
1895             l_addr.as_u32 = e_addr.as_u32;
1896         }
1897     }
1898
1899   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
1900   if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
1901     m = pool_elt_at_index (nm->static_mappings, value.value);
1902
1903   if (is_add)
1904     {
1905       if (m)
1906         {
1907           // identity mapping for second vrf
1908           if (nat44_ei_is_identity_static_mapping (m))
1909             {
1910               pool_foreach (local, m->locals)
1911                 {
1912                   if (local->vrf_id == vrf_id)
1913                     return VNET_API_ERROR_VALUE_EXIST;
1914                 }
1915               pool_get (m->locals, local);
1916               local->vrf_id = vrf_id;
1917               local->fib_index = fib_table_find_or_create_and_lock (
1918                 FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
1919               init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
1920                            m->proto, 0, m - nm->static_mappings);
1921               clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
1922               return 0;
1923             }
1924           return VNET_API_ERROR_VALUE_EXIST;
1925         }
1926
1927       /* Convert VRF id to FIB index */
1928       if (vrf_id != ~0)
1929         {
1930           fib_index = fib_table_find_or_create_and_lock (
1931             FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
1932         }
1933       /* If not specified use inside VRF id from NAT44 plugin config */
1934       else
1935         {
1936           fib_index = nm->inside_fib_index;
1937           vrf_id = nm->inside_vrf_id;
1938           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
1939         }
1940
1941       if (!identity_nat)
1942         {
1943           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
1944                       addr_only ? 0 : proto);
1945           if (!clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
1946                                        &value))
1947             return VNET_API_ERROR_VALUE_EXIST;
1948         }
1949
1950       /* Find external address in allocated addresses and reserve port for
1951          address and port pair mapping when dynamic translations enabled */
1952       if (!(addr_only || nm->static_mapping_only))
1953         {
1954           for (i = 0; i < vec_len (nm->addresses); i++)
1955             {
1956               if (nm->addresses[i].addr.as_u32 == e_addr.as_u32)
1957                 {
1958                   a = nm->addresses + i;
1959                   /* External port must be unused */
1960                   switch (proto)
1961                     {
1962 #define _(N, j, n, s)                                                         \
1963   case NAT_PROTOCOL_##N:                                                      \
1964     if (a->busy_##n##_port_refcounts[e_port])                                 \
1965       return VNET_API_ERROR_INVALID_VALUE;                                    \
1966     ++a->busy_##n##_port_refcounts[e_port];                                   \
1967     if (e_port > 1024)                                                        \
1968       {                                                                       \
1969         a->busy_##n##_ports++;                                                \
1970         a->busy_##n##_ports_per_thread[nat44_ei_get_thread_idx_by_port (      \
1971           e_port)]++;                                                         \
1972       }                                                                       \
1973     break;
1974                       foreach_nat_protocol
1975 #undef _
1976                         default : nat_elog_info (nm, "unknown protocol");
1977                       return VNET_API_ERROR_INVALID_VALUE_2;
1978                     }
1979                   break;
1980                 }
1981             }
1982           /* External address must be allocated */
1983           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1984             {
1985               if (sw_if_index != ~0)
1986                 {
1987                   for (i = 0; i < vec_len (nm->to_resolve); i++)
1988                     {
1989                       rp = nm->to_resolve + i;
1990                       if (rp->addr_only)
1991                         continue;
1992                       if (rp->sw_if_index != sw_if_index &&
1993                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1994                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1995                           rp->e_port != e_port && rp->proto != proto)
1996                         continue;
1997
1998                       vec_del1 (nm->to_resolve, i);
1999                       break;
2000                     }
2001                 }
2002               return VNET_API_ERROR_NO_SUCH_ENTRY;
2003             }
2004         }
2005
2006       pool_get (nm->static_mappings, m);
2007       clib_memset (m, 0, sizeof (*m));
2008       m->tag = vec_dup (tag);
2009       m->local_addr = l_addr;
2010       m->external_addr = e_addr;
2011
2012       if (addr_only)
2013         m->flags |= NAT44_EI_STATIC_MAPPING_FLAG_ADDR_ONLY;
2014       else
2015         {
2016           m->local_port = l_port;
2017           m->external_port = e_port;
2018           m->proto = proto;
2019         }
2020
2021       if (identity_nat)
2022         {
2023           m->flags |= NAT44_EI_STATIC_MAPPING_FLAG_IDENTITY_NAT;
2024           pool_get (m->locals, local);
2025           local->vrf_id = vrf_id;
2026           local->fib_index = fib_index;
2027         }
2028       else
2029         {
2030           m->vrf_id = vrf_id;
2031           m->fib_index = fib_index;
2032         }
2033
2034       if (nm->num_workers > 1)
2035         {
2036           ip4_header_t ip = {
2037             .src_address = m->local_addr,
2038           };
2039           vec_add1 (m->workers, nm->worker_in2out_cb (&ip, m->fib_index, 0));
2040           tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2041         }
2042       else
2043         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2044
2045       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
2046                    m - nm->static_mappings);
2047       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 1);
2048
2049       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
2050                    m - nm->static_mappings);
2051       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 1);
2052
2053       /* Delete dynamic sessions matching local address (+ local port) */
2054       // TODO: based on type of NAT EI/ED
2055       if (!(nm->static_mapping_only))
2056         {
2057           u_key.addr = m->local_addr;
2058           u_key.fib_index = m->fib_index;
2059           kv.key = u_key.as_u64;
2060           if (!clib_bihash_search_8_8 (&tnm->user_hash, &kv, &value))
2061             {
2062               user_index = value.value;
2063               u = pool_elt_at_index (tnm->users, user_index);
2064               if (u->nsessions)
2065                 {
2066                   head_index = u->sessions_per_user_list_head_index;
2067                   head = pool_elt_at_index (tnm->list_pool, head_index);
2068                   elt_index = head->next;
2069                   elt = pool_elt_at_index (tnm->list_pool, elt_index);
2070                   ses_index = elt->value;
2071                   while (ses_index != ~0)
2072                     {
2073                       s = pool_elt_at_index (tnm->sessions, ses_index);
2074                       elt = pool_elt_at_index (tnm->list_pool, elt->next);
2075                       ses_index = elt->value;
2076
2077                       if (nat44_ei_is_session_static (s))
2078                         continue;
2079
2080                       if (!addr_only && s->in2out.port != m->local_port)
2081                         continue;
2082
2083                       nat44_ei_free_session_data_v2 (
2084                         nm, s, tnm - nm->per_thread_data, 0);
2085                       nat44_ei_delete_session (nm, s,
2086                                                tnm - nm->per_thread_data);
2087
2088                       if (!addr_only)
2089                         break;
2090                     }
2091                 }
2092             }
2093         }
2094     }
2095   else
2096     {
2097       if (!m)
2098         {
2099           if (sw_if_index != ~0)
2100             return 0;
2101           else
2102             return VNET_API_ERROR_NO_SUCH_ENTRY;
2103         }
2104
2105       if (identity_nat)
2106         {
2107           if (vrf_id == ~0)
2108             vrf_id = nm->inside_vrf_id;
2109
2110           pool_foreach (local, m->locals)
2111             {
2112               if (local->vrf_id == vrf_id)
2113                 find = local - m->locals;
2114             }
2115           if (find == ~0)
2116             return VNET_API_ERROR_NO_SUCH_ENTRY;
2117
2118           local = pool_elt_at_index (m->locals, find);
2119           fib_index = local->fib_index;
2120           pool_put (m->locals, local);
2121         }
2122       else
2123         fib_index = m->fib_index;
2124
2125       /* Free external address port */
2126       if (!(addr_only || nm->static_mapping_only))
2127         {
2128           for (i = 0; i < vec_len (nm->addresses); i++)
2129             {
2130               if (nm->addresses[i].addr.as_u32 == e_addr.as_u32)
2131                 {
2132                   a = nm->addresses + i;
2133                   switch (proto)
2134                     {
2135 #define _(N, j, n, s)                                                         \
2136   case NAT_PROTOCOL_##N:                                                      \
2137     --a->busy_##n##_port_refcounts[e_port];                                   \
2138     if (e_port > 1024)                                                        \
2139       {                                                                       \
2140         a->busy_##n##_ports--;                                                \
2141         a->busy_##n##_ports_per_thread[nat44_ei_get_thread_idx_by_port (      \
2142           e_port)]--;                                                         \
2143       }                                                                       \
2144     break;
2145                       foreach_nat_protocol
2146 #undef _
2147                         default : return VNET_API_ERROR_INVALID_VALUE_2;
2148                     }
2149                   break;
2150                 }
2151             }
2152         }
2153
2154       if (nm->num_workers > 1)
2155         tnm = vec_elt_at_index (nm->per_thread_data, m->workers[0]);
2156       else
2157         tnm = vec_elt_at_index (nm->per_thread_data, nm->num_workers);
2158
2159       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
2160       clib_bihash_add_del_8_8 (&nm->static_mapping_by_local, &kv, 0);
2161
2162       /* Delete session(s) for static mapping if exist */
2163       if (!(nm->static_mapping_only) ||
2164           (nm->static_mapping_only && nm->static_mapping_connection_tracking))
2165         {
2166           u_key.addr = m->local_addr;
2167           u_key.fib_index = fib_index;
2168           kv.key = u_key.as_u64;
2169           nat44_ei_static_mapping_del_sessions (nm, tnm, u_key, addr_only,
2170                                                 e_addr, e_port);
2171         }
2172
2173       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2174       if (pool_elts (m->locals))
2175         return 0;
2176
2177       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
2178       clib_bihash_add_del_8_8 (&nm->static_mapping_by_external, &kv, 0);
2179
2180       vec_free (m->tag);
2181       vec_free (m->workers);
2182       /* Delete static mapping from pool */
2183       pool_put (nm->static_mappings, m);
2184     }
2185
2186   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
2187     return 0;
2188
2189   /* Add/delete external address to FIB */
2190   pool_foreach (interface, nm->interfaces)
2191     {
2192       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2193         continue;
2194
2195       nat44_ei_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index,
2196                                     is_add);
2197       break;
2198     }
2199   pool_foreach (interface, nm->output_feature_interfaces)
2200     {
2201       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2202         continue;
2203
2204       nat44_ei_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index,
2205                                     is_add);
2206       break;
2207     }
2208   return 0;
2209 }
2210
2211 int
2212 nat44_ei_static_mapping_match (ip4_address_t match_addr, u16 match_port,
2213                                u32 match_fib_index,
2214                                nat_protocol_t match_protocol,
2215                                ip4_address_t *mapping_addr, u16 *mapping_port,
2216                                u32 *mapping_fib_index, u8 by_external,
2217                                u8 *is_addr_only, u8 *is_identity_nat)
2218 {
2219   nat44_ei_main_t *nm = &nat44_ei_main;
2220   clib_bihash_kv_8_8_t kv, value;
2221   nat44_ei_static_mapping_t *m;
2222   u16 port;
2223
2224   if (by_external)
2225     {
2226       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2227       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2228                                   &value))
2229         {
2230           /* Try address only mapping */
2231           init_nat_k (&kv, match_addr, 0, 0, 0);
2232           if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv,
2233                                       &value))
2234             return 1;
2235         }
2236       m = pool_elt_at_index (nm->static_mappings, value.value);
2237
2238       *mapping_fib_index = m->fib_index;
2239       *mapping_addr = m->local_addr;
2240       port = m->local_port;
2241     }
2242   else
2243     {
2244       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2245                   match_protocol);
2246       if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
2247         {
2248           /* Try address only mapping */
2249           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2250           if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv,
2251                                       &value))
2252             return 1;
2253         }
2254       m = pool_elt_at_index (nm->static_mappings, value.value);
2255
2256       *mapping_fib_index = nm->outside_fib_index;
2257       *mapping_addr = m->external_addr;
2258       port = m->external_port;
2259     }
2260
2261   /* Address only mapping doesn't change port */
2262   if (nat44_ei_is_addr_only_static_mapping (m))
2263     *mapping_port = match_port;
2264   else
2265     *mapping_port = port;
2266
2267   if (PREDICT_FALSE (is_addr_only != 0))
2268     *is_addr_only = nat44_ei_is_addr_only_static_mapping (m);
2269
2270   if (PREDICT_FALSE (is_identity_nat != 0))
2271     *is_identity_nat = nat44_ei_is_identity_static_mapping (m);
2272
2273   return 0;
2274 }
2275
2276 static void
2277 nat44_ei_worker_db_free (nat44_ei_main_per_thread_data_t *tnm)
2278 {
2279   pool_free (tnm->list_pool);
2280   pool_free (tnm->lru_pool);
2281   pool_free (tnm->sessions);
2282   pool_free (tnm->users);
2283
2284   clib_bihash_free_8_8 (&tnm->user_hash);
2285 }
2286
2287 u8 *
2288 format_nat44_ei_key (u8 *s, va_list *args)
2289 {
2290   u64 key = va_arg (*args, u64);
2291
2292   ip4_address_t addr;
2293   u16 port;
2294   nat_protocol_t protocol;
2295   u32 fib_index;
2296
2297   split_nat_key (key, &addr, &port, &fib_index, &protocol);
2298
2299   s = format (s, "%U proto %U port %d fib %d", format_ip4_address, &addr,
2300               format_nat_protocol, protocol, clib_net_to_host_u16 (port),
2301               fib_index);
2302   return s;
2303 }
2304
2305 u8 *
2306 format_nat44_ei_user_kvp (u8 *s, va_list *args)
2307 {
2308   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2309   nat44_ei_user_key_t k;
2310
2311   k.as_u64 = v->key;
2312
2313   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2314               k.fib_index, v->value);
2315
2316   return s;
2317 }
2318
2319 u8 *
2320 format_nat44_ei_session_kvp (u8 *s, va_list *args)
2321 {
2322   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2323
2324   s =
2325     format (s, "%U session-index %llu", format_nat44_ei_key, v->key, v->value);
2326
2327   return s;
2328 }
2329
2330 u8 *
2331 format_nat44_ei_static_mapping_kvp (u8 *s, va_list *args)
2332 {
2333   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2334
2335   s = format (s, "%U static-mapping-index %llu", format_nat44_ei_key, v->key,
2336               v->value);
2337
2338   return s;
2339 }
2340
2341 static void
2342 nat44_ei_worker_db_init (nat44_ei_main_per_thread_data_t *tnm,
2343                          u32 translations, u32 translation_buckets,
2344                          u32 user_buckets)
2345 {
2346   dlist_elt_t *head;
2347
2348   pool_alloc (tnm->list_pool, translations);
2349   pool_alloc (tnm->lru_pool, translations);
2350   pool_alloc (tnm->sessions, translations);
2351
2352   clib_bihash_init_8_8 (&tnm->user_hash, "users", user_buckets, 0);
2353
2354   clib_bihash_set_kvp_format_fn_8_8 (&tnm->user_hash,
2355                                      format_nat44_ei_user_kvp);
2356
2357   pool_get (tnm->lru_pool, head);
2358   tnm->tcp_trans_lru_head_index = head - tnm->lru_pool;
2359   clib_dlist_init (tnm->lru_pool, tnm->tcp_trans_lru_head_index);
2360
2361   pool_get (tnm->lru_pool, head);
2362   tnm->tcp_estab_lru_head_index = head - tnm->lru_pool;
2363   clib_dlist_init (tnm->lru_pool, tnm->tcp_estab_lru_head_index);
2364
2365   pool_get (tnm->lru_pool, head);
2366   tnm->udp_lru_head_index = head - tnm->lru_pool;
2367   clib_dlist_init (tnm->lru_pool, tnm->udp_lru_head_index);
2368
2369   pool_get (tnm->lru_pool, head);
2370   tnm->icmp_lru_head_index = head - tnm->lru_pool;
2371   clib_dlist_init (tnm->lru_pool, tnm->icmp_lru_head_index);
2372
2373   pool_get (tnm->lru_pool, head);
2374   tnm->unk_proto_lru_head_index = head - tnm->lru_pool;
2375   clib_dlist_init (tnm->lru_pool, tnm->unk_proto_lru_head_index);
2376 }
2377
2378 static void
2379 nat44_ei_db_free ()
2380 {
2381   nat44_ei_main_t *nm = &nat44_ei_main;
2382   nat44_ei_main_per_thread_data_t *tnm;
2383
2384   pool_free (nm->static_mappings);
2385   clib_bihash_free_8_8 (&nm->static_mapping_by_local);
2386   clib_bihash_free_8_8 (&nm->static_mapping_by_external);
2387
2388   if (nm->pat)
2389     {
2390       clib_bihash_free_8_8 (&nm->in2out);
2391       clib_bihash_free_8_8 (&nm->out2in);
2392       vec_foreach (tnm, nm->per_thread_data)
2393         {
2394           nat44_ei_worker_db_free (tnm);
2395         }
2396     }
2397 }
2398
2399 static void
2400 nat44_ei_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
2401 {
2402   nat44_ei_main_t *nm = &nat44_ei_main;
2403   nat44_ei_main_per_thread_data_t *tnm;
2404
2405   u32 static_mapping_buckets = 1024;
2406   u32 static_mapping_memory_size = 64 << 20;
2407
2408   clib_bihash_init_8_8 (&nm->static_mapping_by_local,
2409                         "static_mapping_by_local", static_mapping_buckets,
2410                         static_mapping_memory_size);
2411   clib_bihash_init_8_8 (&nm->static_mapping_by_external,
2412                         "static_mapping_by_external", static_mapping_buckets,
2413                         static_mapping_memory_size);
2414   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_local,
2415                                      format_nat44_ei_static_mapping_kvp);
2416   clib_bihash_set_kvp_format_fn_8_8 (&nm->static_mapping_by_external,
2417                                      format_nat44_ei_static_mapping_kvp);
2418
2419   if (nm->pat)
2420     {
2421       clib_bihash_init_8_8 (&nm->in2out, "in2out", translation_buckets, 0);
2422       clib_bihash_init_8_8 (&nm->out2in, "out2in", translation_buckets, 0);
2423       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2424                                          format_nat44_ei_session_kvp);
2425       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2426                                          format_nat44_ei_session_kvp);
2427       vec_foreach (tnm, nm->per_thread_data)
2428         {
2429           nat44_ei_worker_db_init (tnm, translations, translation_buckets,
2430                                    user_buckets);
2431         }
2432     }
2433 }
2434
2435 void
2436 nat44_ei_sessions_clear ()
2437 {
2438   nat44_ei_main_t *nm = &nat44_ei_main;
2439   nat44_ei_main_per_thread_data_t *tnm;
2440
2441   if (nm->pat)
2442     {
2443       clib_bihash_free_8_8 (&nm->in2out);
2444       clib_bihash_free_8_8 (&nm->out2in);
2445       clib_bihash_init_8_8 (&nm->in2out, "in2out", nm->translation_buckets, 0);
2446       clib_bihash_init_8_8 (&nm->out2in, "out2in", nm->translation_buckets, 0);
2447       clib_bihash_set_kvp_format_fn_8_8 (&nm->in2out,
2448                                          format_nat44_ei_session_kvp);
2449       clib_bihash_set_kvp_format_fn_8_8 (&nm->out2in,
2450                                          format_nat44_ei_session_kvp);
2451       vec_foreach (tnm, nm->per_thread_data)
2452         {
2453           nat44_ei_worker_db_free (tnm);
2454           nat44_ei_worker_db_init (tnm, nm->translations,
2455                                    nm->translation_buckets, nm->user_buckets);
2456         }
2457     }
2458
2459   vlib_zero_simple_counter (&nm->total_users, 0);
2460   vlib_zero_simple_counter (&nm->total_sessions, 0);
2461   vlib_zero_simple_counter (&nm->user_limit_reached, 0);
2462 }
2463
2464 static void
2465 nat44_ei_update_outside_fib (ip4_main_t *im, uword opaque, u32 sw_if_index,
2466                              u32 new_fib_index, u32 old_fib_index)
2467 {
2468   nat44_ei_main_t *nm = &nat44_ei_main;
2469   nat44_ei_outside_fib_t *outside_fib;
2470   nat44_ei_interface_t *i;
2471   u8 is_add = 1;
2472   u8 match = 0;
2473
2474   if (!nm->enabled || (new_fib_index == old_fib_index) ||
2475       (!vec_len (nm->outside_fibs)))
2476     {
2477       return;
2478     }
2479
2480   pool_foreach (i, nm->interfaces)
2481     {
2482       if (i->sw_if_index == sw_if_index)
2483         {
2484           if (!(nat44_ei_interface_is_outside (i)))
2485             return;
2486           match = 1;
2487         }
2488     }
2489
2490   pool_foreach (i, nm->output_feature_interfaces)
2491     {
2492       if (i->sw_if_index == sw_if_index)
2493         {
2494           if (!(nat44_ei_interface_is_outside (i)))
2495             return;
2496           match = 1;
2497         }
2498     }
2499
2500   if (!match)
2501     return;
2502
2503   vec_foreach (outside_fib, nm->outside_fibs)
2504     {
2505       if (outside_fib->fib_index == old_fib_index)
2506         {
2507           outside_fib->refcount--;
2508           if (!outside_fib->refcount)
2509             vec_del1 (nm->outside_fibs, outside_fib - nm->outside_fibs);
2510           break;
2511         }
2512     }
2513
2514   vec_foreach (outside_fib, nm->outside_fibs)
2515     {
2516       if (outside_fib->fib_index == new_fib_index)
2517         {
2518           outside_fib->refcount++;
2519           is_add = 0;
2520           break;
2521         }
2522     }
2523
2524   if (is_add)
2525     {
2526       vec_add2 (nm->outside_fibs, outside_fib, 1);
2527       outside_fib->refcount = 1;
2528       outside_fib->fib_index = new_fib_index;
2529     }
2530 }
2531
2532 int
2533 nat44_ei_add_address (nat44_ei_main_t *nm, ip4_address_t *addr, u32 vrf_id)
2534 {
2535   nat44_ei_address_t *ap;
2536   nat44_ei_interface_t *i;
2537   vlib_thread_main_t *tm = vlib_get_thread_main ();
2538
2539   /* Check if address already exists */
2540   vec_foreach (ap, nm->addresses)
2541     {
2542       if (ap->addr.as_u32 == addr->as_u32)
2543         {
2544           nat44_ei_log_err ("address exist");
2545           return VNET_API_ERROR_VALUE_EXIST;
2546         }
2547     }
2548
2549   vec_add2 (nm->addresses, ap, 1);
2550
2551   ap->addr = *addr;
2552   if (vrf_id != ~0)
2553     ap->fib_index = fib_table_find_or_create_and_lock (
2554       FIB_PROTOCOL_IP4, vrf_id, nm->fib_src_low);
2555   else
2556     ap->fib_index = ~0;
2557
2558 #define _(N, i, n, s)                                                         \
2559   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
2560                sizeof (ap->busy_##n##_port_refcounts));                       \
2561   ap->busy_##n##_ports = 0;                                                   \
2562   ap->busy_##n##_ports_per_thread = 0;                                        \
2563   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
2564                            tm->n_vlib_mains - 1, 0);
2565   foreach_nat_protocol
2566 #undef _
2567
2568     /* Add external address to FIB */
2569     pool_foreach (i, nm->interfaces)
2570   {
2571     if (nat44_ei_interface_is_inside (i) || nm->out2in_dpo)
2572       continue;
2573
2574     nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
2575     break;
2576   }
2577   pool_foreach (i, nm->output_feature_interfaces)
2578     {
2579       if (nat44_ei_interface_is_inside (i) || nm->out2in_dpo)
2580         continue;
2581
2582       nat44_ei_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
2583       break;
2584     }
2585
2586   return 0;
2587 }
2588
2589 int
2590 nat44_ei_add_interface_address (nat44_ei_main_t *nm, u32 sw_if_index,
2591                                 int is_del)
2592 {
2593   ip4_main_t *ip4_main = nm->ip4_main;
2594   ip4_address_t *first_int_addr;
2595   nat44_ei_static_map_resolve_t *rp;
2596   u32 *indices_to_delete = 0;
2597   int i, j;
2598   u32 *auto_add_sw_if_indices = nm->auto_add_sw_if_indices;
2599
2600   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
2601                                                 0 /* just want the address */);
2602
2603   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
2604     {
2605       if (auto_add_sw_if_indices[i] == sw_if_index)
2606         {
2607           if (is_del)
2608             {
2609               /* if have address remove it */
2610               if (first_int_addr)
2611                 (void) nat44_ei_del_address (nm, first_int_addr[0], 1);
2612               else
2613                 {
2614                   for (j = 0; j < vec_len (nm->to_resolve); j++)
2615                     {
2616                       rp = nm->to_resolve + j;
2617                       if (rp->sw_if_index == sw_if_index)
2618                         vec_add1 (indices_to_delete, j);
2619                     }
2620                   if (vec_len (indices_to_delete))
2621                     {
2622                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
2623                         vec_del1 (nm->to_resolve, j);
2624                       vec_free (indices_to_delete);
2625                     }
2626                 }
2627               vec_del1 (nm->auto_add_sw_if_indices, i);
2628             }
2629           else
2630             return VNET_API_ERROR_VALUE_EXIST;
2631
2632           return 0;
2633         }
2634     }
2635
2636   if (is_del)
2637     return VNET_API_ERROR_NO_SUCH_ENTRY;
2638
2639   /* add to the auto-address list */
2640   vec_add1 (nm->auto_add_sw_if_indices, sw_if_index);
2641
2642   /* If the address is already bound - or static - add it now */
2643   if (first_int_addr)
2644     (void) nat44_ei_add_address (nm, first_int_addr, ~0);
2645
2646   return 0;
2647 }
2648
2649 static int
2650 nat44_ei_is_address_used_in_static_mapping (ip4_address_t addr)
2651 {
2652   nat44_ei_main_t *nm = &nat44_ei_main;
2653   nat44_ei_static_mapping_t *m;
2654   pool_foreach (m, nm->static_mappings)
2655     {
2656       if (nat44_ei_is_addr_only_static_mapping (m) ||
2657           nat44_ei_is_identity_static_mapping (m))
2658         continue;
2659       if (m->external_addr.as_u32 == addr.as_u32)
2660         return 1;
2661     }
2662   return 0;
2663 }
2664
2665 int
2666 nat44_ei_del_address (nat44_ei_main_t *nm, ip4_address_t addr, u8 delete_sm)
2667 {
2668   nat44_ei_address_t *a = 0;
2669   nat44_ei_session_t *ses;
2670   u32 *ses_to_be_removed = 0, *ses_index;
2671   nat44_ei_main_per_thread_data_t *tnm;
2672   nat44_ei_interface_t *interface;
2673   nat44_ei_static_mapping_t *m;
2674   int i;
2675
2676   /* Find SNAT address */
2677   for (i = 0; i < vec_len (nm->addresses); i++)
2678     {
2679       if (nm->addresses[i].addr.as_u32 == addr.as_u32)
2680         {
2681           a = nm->addresses + i;
2682           break;
2683         }
2684     }
2685   if (!a)
2686     {
2687       nat44_ei_log_err ("no such address");
2688       return VNET_API_ERROR_NO_SUCH_ENTRY;
2689     }
2690
2691   if (delete_sm)
2692     {
2693       pool_foreach (m, nm->static_mappings)
2694         {
2695           if (m->external_addr.as_u32 == addr.as_u32)
2696             (void) nat44_ei_add_del_static_mapping (
2697               m->local_addr, m->external_addr, m->local_port, m->external_port,
2698               m->proto, ~0 /* sw_if_index */, m->vrf_id,
2699               nat44_ei_is_addr_only_static_mapping (m),
2700               nat44_ei_is_identity_static_mapping (m), m->tag, 0);
2701         }
2702     }
2703   else
2704     {
2705       /* Check if address is used in some static mapping */
2706       if (nat44_ei_is_address_used_in_static_mapping (addr))
2707         {
2708           nat44_ei_log_err ("address used in static mapping");
2709           return VNET_API_ERROR_UNSPECIFIED;
2710         }
2711     }
2712
2713   if (a->fib_index != ~0)
2714     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nm->fib_src_low);
2715
2716   /* Delete sessions using address */
2717   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
2718     {
2719       vec_foreach (tnm, nm->per_thread_data)
2720         {
2721           pool_foreach (ses, tnm->sessions)
2722             {
2723               if (ses->out2in.addr.as_u32 == addr.as_u32)
2724                 {
2725                   nat44_ei_free_session_data (nm, ses,
2726                                               tnm - nm->per_thread_data, 0);
2727                   vec_add1 (ses_to_be_removed, ses - tnm->sessions);
2728                 }
2729             }
2730           vec_foreach (ses_index, ses_to_be_removed)
2731             {
2732               ses = pool_elt_at_index (tnm->sessions, ses_index[0]);
2733               nat44_ei_delete_session (nm, ses, tnm - nm->per_thread_data);
2734             }
2735           vec_free (ses_to_be_removed);
2736         }
2737     }
2738
2739 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
2740   foreach_nat_protocol
2741 #undef _
2742     vec_del1 (nm->addresses, i);
2743
2744   /* Delete external address from FIB */
2745   pool_foreach (interface, nm->interfaces)
2746     {
2747       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2748         continue;
2749       nat44_ei_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
2750       break;
2751     }
2752
2753   pool_foreach (interface, nm->output_feature_interfaces)
2754     {
2755       if (nat44_ei_interface_is_inside (interface) || nm->out2in_dpo)
2756         continue;
2757       nat44_ei_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
2758       break;
2759     }
2760
2761   return 0;
2762 }
2763
2764 static void
2765 nat44_ei_ip4_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
2766                                            u32 sw_if_index,
2767                                            ip4_address_t *address,
2768                                            u32 address_length,
2769                                            u32 if_address_index, u32 is_delete)
2770 {
2771   nat44_ei_main_t *nm = &nat44_ei_main;
2772   nat44_ei_static_map_resolve_t *rp;
2773   ip4_address_t l_addr;
2774   int i, j;
2775   int rv;
2776   nat44_ei_address_t *addresses = nm->addresses;
2777
2778   if (!nm->enabled)
2779     return;
2780
2781   for (i = 0; i < vec_len (nm->auto_add_sw_if_indices); i++)
2782     {
2783       if (sw_if_index == nm->auto_add_sw_if_indices[i])
2784         goto match;
2785     }
2786
2787   return;
2788
2789 match:
2790   if (!is_delete)
2791     {
2792       /* Don't trip over lease renewal, static config */
2793       for (j = 0; j < vec_len (addresses); j++)
2794         if (addresses[j].addr.as_u32 == address->as_u32)
2795           return;
2796
2797       (void) nat44_ei_add_address (nm, address, ~0);
2798       /* Scan static map resolution vector */
2799       for (j = 0; j < vec_len (nm->to_resolve); j++)
2800         {
2801           rp = nm->to_resolve + j;
2802           if (rp->addr_only)
2803             continue;
2804           /* On this interface? */
2805           if (rp->sw_if_index == sw_if_index)
2806             {
2807               /* Indetity mapping? */
2808               if (rp->l_addr.as_u32 == 0)
2809                 l_addr.as_u32 = address[0].as_u32;
2810               else
2811                 l_addr.as_u32 = rp->l_addr.as_u32;
2812               /* Add the static mapping */
2813               rv = nat44_ei_add_del_static_mapping (
2814                 l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
2815                 ~0 /* sw_if_index */, rp->vrf_id, rp->addr_only,
2816                 rp->identity_nat, rp->tag, 1);
2817               if (rv)
2818                 nat_elog_notice_X1 (
2819                   nm, "nat44_ei_add_del_static_mapping returned %d", "i4", rv);
2820             }
2821         }
2822       return;
2823     }
2824   else
2825     {
2826       (void) nat44_ei_del_address (nm, address[0], 1);
2827       return;
2828     }
2829 }
2830
2831 int
2832 nat44_ei_set_frame_queue_nelts (u32 frame_queue_nelts)
2833 {
2834   fail_if_enabled ();
2835   nat44_ei_main_t *nm = &nat44_ei_main;
2836   nm->frame_queue_nelts = frame_queue_nelts;
2837   return 0;
2838 }
2839
2840 static void
2841 nat44_ei_ip4_add_del_addr_only_sm_cb (ip4_main_t *im, uword opaque,
2842                                       u32 sw_if_index, ip4_address_t *address,
2843                                       u32 address_length, u32 if_address_index,
2844                                       u32 is_delete)
2845 {
2846   nat44_ei_main_t *nm = &nat44_ei_main;
2847   nat44_ei_static_map_resolve_t *rp;
2848   nat44_ei_static_mapping_t *m;
2849   clib_bihash_kv_8_8_t kv, value;
2850   int i, rv;
2851   ip4_address_t l_addr;
2852
2853   if (!nm->enabled)
2854     return;
2855
2856   for (i = 0; i < vec_len (nm->to_resolve); i++)
2857     {
2858       rp = nm->to_resolve + i;
2859       if (rp->addr_only == 0)
2860         continue;
2861       if (rp->sw_if_index == sw_if_index)
2862         goto match;
2863     }
2864
2865   return;
2866
2867 match:
2868   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
2869               nm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
2870   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
2871     m = 0;
2872   else
2873     m = pool_elt_at_index (nm->static_mappings, value.value);
2874
2875   if (!is_delete)
2876     {
2877       /* Don't trip over lease renewal, static config */
2878       if (m)
2879         return;
2880     }
2881   else
2882     {
2883       if (!m)
2884         return;
2885     }
2886
2887   /* Indetity mapping? */
2888   if (rp->l_addr.as_u32 == 0)
2889     l_addr.as_u32 = address[0].as_u32;
2890   else
2891     l_addr.as_u32 = rp->l_addr.as_u32;
2892   /* Add the static mapping */
2893
2894   rv = nat44_ei_add_del_static_mapping (
2895     l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
2896     ~0 /* sw_if_index */, rp->vrf_id, rp->addr_only, rp->identity_nat, rp->tag,
2897     !is_delete);
2898   if (rv)
2899     nat_elog_notice_X1 (nm, "nat44_ei_add_del_static_mapping returned %d",
2900                         "i4", rv);
2901 }
2902
2903 VLIB_NODE_FN (nat44_ei_classify_node)
2904 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2905 {
2906   u32 n_left_from, *from, *to_next;
2907   nat44_ei_classify_next_t next_index;
2908   nat44_ei_main_t *nm = &nat44_ei_main;
2909   nat44_ei_static_mapping_t *m;
2910   u32 next_in2out = 0, next_out2in = 0;
2911
2912   from = vlib_frame_vector_args (frame);
2913   n_left_from = frame->n_vectors;
2914   next_index = node->cached_next_index;
2915
2916   while (n_left_from > 0)
2917     {
2918       u32 n_left_to_next;
2919
2920       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2921
2922       while (n_left_from > 0 && n_left_to_next > 0)
2923         {
2924           u32 bi0;
2925           vlib_buffer_t *b0;
2926           u32 next0 = NAT44_EI_CLASSIFY_NEXT_IN2OUT;
2927           ip4_header_t *ip0;
2928           nat44_ei_address_t *ap;
2929           clib_bihash_kv_8_8_t kv0, value0;
2930
2931           /* speculatively enqueue b0 to the current next frame */
2932           bi0 = from[0];
2933           to_next[0] = bi0;
2934           from += 1;
2935           to_next += 1;
2936           n_left_from -= 1;
2937           n_left_to_next -= 1;
2938
2939           b0 = vlib_get_buffer (vm, bi0);
2940           ip0 = vlib_buffer_get_current (b0);
2941
2942           vec_foreach (ap, nm->addresses)
2943             {
2944               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
2945                 {
2946                   next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2947                   goto enqueue0;
2948                 }
2949             }
2950
2951           if (PREDICT_FALSE (pool_elts (nm->static_mappings)))
2952             {
2953               init_nat_k (&kv0, ip0->dst_address, 0, 0, 0);
2954               /* try to classify the fragment based on IP header alone */
2955               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
2956                                            &kv0, &value0))
2957                 {
2958                   m = pool_elt_at_index (nm->static_mappings, value0.value);
2959                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
2960                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2961                   goto enqueue0;
2962                 }
2963               init_nat_k (&kv0, ip0->dst_address,
2964                           vnet_buffer (b0)->ip.reass.l4_dst_port, 0,
2965                           ip_proto_to_nat_proto (ip0->protocol));
2966               if (!clib_bihash_search_8_8 (&nm->static_mapping_by_external,
2967                                            &kv0, &value0))
2968                 {
2969                   m = pool_elt_at_index (nm->static_mappings, value0.value);
2970                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
2971                     next0 = NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2972                 }
2973             }
2974
2975         enqueue0:
2976           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2977                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
2978             {
2979               nat44_ei_classify_trace_t *t =
2980                 vlib_add_trace (vm, node, b0, sizeof (*t));
2981               t->cached = 0;
2982               t->next_in2out = next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
2983             }
2984
2985           next_in2out += next0 == NAT44_EI_CLASSIFY_NEXT_IN2OUT;
2986           next_out2in += next0 == NAT44_EI_CLASSIFY_NEXT_OUT2IN;
2987
2988           /* verify speculative enqueue, maybe switch current next frame */
2989           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2990                                            n_left_to_next, bi0, next0);
2991         }
2992
2993       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2994     }
2995
2996   vlib_node_increment_counter (
2997     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_IN2OUT, next_in2out);
2998   vlib_node_increment_counter (
2999     vm, node->node_index, NAT44_EI_CLASSIFY_ERROR_NEXT_OUT2IN, next_out2in);
3000   return frame->n_vectors;
3001 }
3002
3003 VLIB_REGISTER_NODE (nat44_ei_classify_node) = {
3004   .name = "nat44-ei-classify",
3005   .vector_size = sizeof (u32),
3006   .format_trace = format_nat44_ei_classify_trace,
3007   .type = VLIB_NODE_TYPE_INTERNAL,
3008   .n_errors = ARRAY_LEN(nat44_ei_classify_error_strings),
3009   .error_strings = nat44_ei_classify_error_strings,
3010   .n_next_nodes = NAT44_EI_CLASSIFY_N_NEXT,
3011   .next_nodes = {
3012     [NAT44_EI_CLASSIFY_NEXT_IN2OUT] = "nat44-ei-in2out",
3013     [NAT44_EI_CLASSIFY_NEXT_OUT2IN] = "nat44-ei-out2in",
3014     [NAT44_EI_CLASSIFY_NEXT_DROP] = "error-drop",
3015   },
3016 };
3017
3018 /*
3019  * fd.io coding-style-patch-verification: ON
3020  *
3021  * Local Variables:
3022  * eval: (c-set-style "gnu")
3023  * End:
3024  */